2024-09-13T14:10:15.171 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-13T14:10:15.197 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T14:10:15.299 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903565 branch: wip-aclamk-cbt-combined description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: akupczyk@redhat.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7903565' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-aclamk-cbt-combined ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(POOL_APP_NOT_ENABLED\) - OSD_DOWN - mons down - mon down - MON_DOWN - out of quorum - PG_DEGRADED - Reduced data availability - Degraded data redundancy - OSDMAP_FLAGS - PG_AVAILABILITY sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: wip-aclamk-cbt-combined sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 owner: scheduled_akupczyk@teuthology priority: 160 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 6545 sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 682/13280 suite: upgrade suite_branch: wip-aclamk-cbt-combined suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 targets: smithi096.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPDjk/sFcYIv6W+ibHDFSjzgHMiPpWUfC1Sw0BDdhc2knitNEoFBoKew+sTCwOP95lCWVP1myzA3C98oWcryAB8= smithi204.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHAdNZxmcnim9nzC9TYPnncpwgImRrcysmbU2BiWeG5jH7vrs6gyV5WexDK4QRvB07yVxxRW2IIGHci3y2CUdDs= tasks: - pexec: all: - sudo dnf remove nvme-cli -y - sudo dnf install nvmetcli nvme-cli -y - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-13_10:00:18 tube: smithi user: akupczyk verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-13T14:10:15.299 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa; will attempt to use it 2024-09-13T14:10:15.300 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks 2024-09-13T14:10:15.301 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-13T14:10:15.302 INFO:teuthology.task.internal:Checking packages... 2024-09-13T14:10:15.323 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd58f73069a8157a9ace12a0247ee3e8497707ea9' 2024-09-13T14:10:15.323 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:10:15.323 INFO:teuthology.packaging:ref: None 2024-09-13T14:10:15.323 INFO:teuthology.packaging:tag: None 2024-09-13T14:10:15.324 INFO:teuthology.packaging:branch: wip-aclamk-cbt-combined 2024-09-13T14:10:15.324 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:10:15.324 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-aclamk-cbt-combined 2024-09-13T14:10:15.477 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4697.gd58f7306 2024-09-13T14:10:15.479 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-13T14:10:15.484 INFO:teuthology.task.internal:no buildpackages task found 2024-09-13T14:10:15.484 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-13T14:10:15.493 INFO:teuthology.task.internal:Saving configuration 2024-09-13T14:10:15.515 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-13T14:10:15.522 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-13T14:10:15.555 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi096.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903565', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 14:05:01.992650', 'locked_by': 'scheduled_akupczyk@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPDjk/sFcYIv6W+ibHDFSjzgHMiPpWUfC1Sw0BDdhc2knitNEoFBoKew+sTCwOP95lCWVP1myzA3C98oWcryAB8='} 2024-09-13T14:10:15.586 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi204.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903565', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 14:05:01.994620', 'locked_by': 'scheduled_akupczyk@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHAdNZxmcnim9nzC9TYPnncpwgImRrcysmbU2BiWeG5jH7vrs6gyV5WexDK4QRvB07yVxxRW2IIGHci3y2CUdDs='} 2024-09-13T14:10:15.587 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-13T14:10:15.595 INFO:teuthology.task.internal:roles: ubuntu@smithi096.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-13T14:10:15.595 INFO:teuthology.task.internal:roles: ubuntu@smithi204.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-13T14:10:15.595 INFO:teuthology.run_tasks:Running task console_log... 2024-09-13T14:10:15.680 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f4083a33be0>, signals=[15]) 2024-09-13T14:10:15.681 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-13T14:10:15.691 INFO:teuthology.task.internal:Opening connections... 2024-09-13T14:10:15.692 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi096.front.sepia.ceph.com 2024-09-13T14:10:15.693 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi096.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:10:15.779 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi204.front.sepia.ceph.com 2024-09-13T14:10:15.780 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi204.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:10:15.872 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-13T14:10:15.881 DEBUG:teuthology.orchestra.run.smithi096:> uname -m 2024-09-13T14:10:15.900 INFO:teuthology.orchestra.run.smithi096.stdout:x86_64 2024-09-13T14:10:15.900 DEBUG:teuthology.orchestra.run.smithi096:> cat /etc/os-release 2024-09-13T14:10:15.958 INFO:teuthology.orchestra.run.smithi096.stdout:NAME="CentOS Stream" 2024-09-13T14:10:15.958 INFO:teuthology.orchestra.run.smithi096.stdout:VERSION="9" 2024-09-13T14:10:15.958 INFO:teuthology.orchestra.run.smithi096.stdout:ID="centos" 2024-09-13T14:10:15.958 INFO:teuthology.orchestra.run.smithi096.stdout:ID_LIKE="rhel fedora" 2024-09-13T14:10:15.958 INFO:teuthology.orchestra.run.smithi096.stdout:VERSION_ID="9" 2024-09-13T14:10:15.958 INFO:teuthology.orchestra.run.smithi096.stdout:PLATFORM_ID="platform:el9" 2024-09-13T14:10:15.958 INFO:teuthology.orchestra.run.smithi096.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T14:10:15.958 INFO:teuthology.orchestra.run.smithi096.stdout:ANSI_COLOR="0;31" 2024-09-13T14:10:15.958 INFO:teuthology.orchestra.run.smithi096.stdout:LOGO="fedora-logo-icon" 2024-09-13T14:10:15.959 INFO:teuthology.orchestra.run.smithi096.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T14:10:15.959 INFO:teuthology.orchestra.run.smithi096.stdout:HOME_URL="https://centos.org/" 2024-09-13T14:10:15.959 INFO:teuthology.orchestra.run.smithi096.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T14:10:15.959 INFO:teuthology.orchestra.run.smithi096.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T14:10:15.959 INFO:teuthology.orchestra.run.smithi096.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T14:10:15.959 INFO:teuthology.lock.ops:Updating smithi096.front.sepia.ceph.com on lock server 2024-09-13T14:10:15.992 DEBUG:teuthology.orchestra.run.smithi204:> uname -m 2024-09-13T14:10:16.010 INFO:teuthology.orchestra.run.smithi204.stdout:x86_64 2024-09-13T14:10:16.010 DEBUG:teuthology.orchestra.run.smithi204:> cat /etc/os-release 2024-09-13T14:10:16.069 INFO:teuthology.orchestra.run.smithi204.stdout:NAME="CentOS Stream" 2024-09-13T14:10:16.069 INFO:teuthology.orchestra.run.smithi204.stdout:VERSION="9" 2024-09-13T14:10:16.069 INFO:teuthology.orchestra.run.smithi204.stdout:ID="centos" 2024-09-13T14:10:16.069 INFO:teuthology.orchestra.run.smithi204.stdout:ID_LIKE="rhel fedora" 2024-09-13T14:10:16.069 INFO:teuthology.orchestra.run.smithi204.stdout:VERSION_ID="9" 2024-09-13T14:10:16.069 INFO:teuthology.orchestra.run.smithi204.stdout:PLATFORM_ID="platform:el9" 2024-09-13T14:10:16.069 INFO:teuthology.orchestra.run.smithi204.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T14:10:16.069 INFO:teuthology.orchestra.run.smithi204.stdout:ANSI_COLOR="0;31" 2024-09-13T14:10:16.069 INFO:teuthology.orchestra.run.smithi204.stdout:LOGO="fedora-logo-icon" 2024-09-13T14:10:16.070 INFO:teuthology.orchestra.run.smithi204.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T14:10:16.070 INFO:teuthology.orchestra.run.smithi204.stdout:HOME_URL="https://centos.org/" 2024-09-13T14:10:16.070 INFO:teuthology.orchestra.run.smithi204.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T14:10:16.070 INFO:teuthology.orchestra.run.smithi204.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T14:10:16.070 INFO:teuthology.orchestra.run.smithi204.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T14:10:16.070 INFO:teuthology.lock.ops:Updating smithi204.front.sepia.ceph.com on lock server 2024-09-13T14:10:16.101 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-13T14:10:16.114 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-13T14:10:16.125 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-13T14:10:16.125 DEBUG:teuthology.orchestra.run.smithi096:> test '!' -e /home/ubuntu/cephtest 2024-09-13T14:10:16.128 DEBUG:teuthology.orchestra.run.smithi204:> test '!' -e /home/ubuntu/cephtest 2024-09-13T14:10:16.144 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-13T14:10:16.155 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-13T14:10:16.155 DEBUG:teuthology.orchestra.run.smithi096:> test -z $(ls -A /var/lib/ceph) 2024-09-13T14:10:16.185 DEBUG:teuthology.orchestra.run.smithi204:> test -z $(ls -A /var/lib/ceph) 2024-09-13T14:10:16.210 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-13T14:10:16.286 INFO:teuthology.run_tasks:Running task kernel... 2024-09-13T14:10:16.303 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-13T14:10:16.304 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-13T14:10:16.304 DEBUG:teuthology.orchestra.run.smithi096:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T14:10:16.304 DEBUG:teuthology.orchestra.run.smithi204:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T14:10:16.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:10:16.325 DEBUG:teuthology.orchestra.run.smithi096:> uname -r 2024-09-13T14:10:16.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:10:16.326 DEBUG:teuthology.orchestra.run.smithi204:> uname -r 2024-09-13T14:10:16.382 INFO:teuthology.orchestra.run.smithi096.stdout:5.14.0-503.el9.x86_64 2024-09-13T14:10:16.382 INFO:teuthology.task.kernel:Running kernel on smithi096: 5.14.0-503.el9.x86_64 2024-09-13T14:10:16.382 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum install -y kernel 2024-09-13T14:10:16.383 INFO:teuthology.orchestra.run.smithi204.stdout:5.14.0-503.el9.x86_64 2024-09-13T14:10:16.383 INFO:teuthology.task.kernel:Running kernel on smithi204: 5.14.0-503.el9.x86_64 2024-09-13T14:10:16.384 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum install -y kernel 2024-09-13T14:10:19.297 INFO:teuthology.orchestra.run.smithi204.stdout:CentOS Stream 9 - BaseOS 7.3 MB/s | 8.2 MB 00:01 2024-09-13T14:10:19.955 INFO:teuthology.orchestra.run.smithi096.stdout:CentOS Stream 9 - BaseOS 4.5 MB/s | 8.2 MB 00:01 2024-09-13T14:10:21.856 INFO:teuthology.orchestra.run.smithi096.stdout:CentOS Stream 9 - AppStream 25 MB/s | 20 MB 00:00 2024-09-13T14:10:24.914 INFO:teuthology.orchestra.run.smithi204.stdout:CentOS Stream 9 - AppStream 4.6 MB/s | 20 MB 00:04 2024-09-13T14:10:27.723 INFO:teuthology.orchestra.run.smithi096.stdout:CentOS Stream 9 - CRB 3.1 MB/s | 6.5 MB 00:02 2024-09-13T14:10:29.615 INFO:teuthology.orchestra.run.smithi096.stdout:CentOS Stream 9 - Extras packages 27 kB/s | 19 kB 00:00 2024-09-13T14:10:30.423 INFO:teuthology.orchestra.run.smithi204.stdout:CentOS Stream 9 - CRB 4.0 MB/s | 6.5 MB 00:01 2024-09-13T14:10:31.135 INFO:teuthology.orchestra.run.smithi096.stdout:Extra Packages for Enterprise Linux 16 MB/s | 23 MB 00:01 2024-09-13T14:10:32.024 INFO:teuthology.orchestra.run.smithi204.stdout:CentOS Stream 9 - Extras packages 48 kB/s | 19 kB 00:00 2024-09-13T14:10:33.511 INFO:teuthology.orchestra.run.smithi204.stdout:Extra Packages for Enterprise Linux 17 MB/s | 23 MB 00:01 2024-09-13T14:10:36.574 INFO:teuthology.orchestra.run.smithi096.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-13T14:10:38.589 INFO:teuthology.orchestra.run.smithi096.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T14:10:38.589 INFO:teuthology.orchestra.run.smithi096.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T14:10:38.669 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:10:38.680 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:10:38.681 INFO:teuthology.orchestra.run.smithi096.stdout: Package Architecture Version Repository Size 2024-09-13T14:10:38.681 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:10:38.681 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:10:38.681 INFO:teuthology.orchestra.run.smithi096.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T14:10:38.681 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2024-09-13T14:10:38.681 INFO:teuthology.orchestra.run.smithi096.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T14:10:38.681 INFO:teuthology.orchestra.run.smithi096.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T14:10:38.681 INFO:teuthology.orchestra.run.smithi096.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T14:10:38.681 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:10:38.681 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:10:38.681 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:10:38.682 INFO:teuthology.orchestra.run.smithi096.stdout:Install 4 Packages 2024-09-13T14:10:38.682 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:10:38.682 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 79 M 2024-09-13T14:10:38.682 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 126 M 2024-09-13T14:10:38.682 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:10:39.009 INFO:teuthology.orchestra.run.smithi204.stdout:lab-extras 26 kB/s | 1.7 kB 00:00 2024-09-13T14:10:39.025 INFO:teuthology.orchestra.run.smithi096.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 251 kB/s | 23 kB 00:00 2024-09-13T14:10:39.207 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:00:01 ago on Fri 13 Sep 2024 02:10:38 PM UTC. 2024-09-13T14:10:40.125 INFO:teuthology.orchestra.run.smithi096.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 13 MB/s | 16 MB 00:01 2024-09-13T14:10:41.049 INFO:teuthology.orchestra.run.smithi204.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T14:10:41.049 INFO:teuthology.orchestra.run.smithi204.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T14:10:41.132 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:10:41.139 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:10:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: Package Architecture Version Repository Size 2024-09-13T14:10:41.139 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:10:41.139 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:10:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T14:10:41.139 INFO:teuthology.orchestra.run.smithi204.stdout:Installing dependencies: 2024-09-13T14:10:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T14:10:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T14:10:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T14:10:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:10:41.140 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:10:41.140 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:10:41.140 INFO:teuthology.orchestra.run.smithi204.stdout:Install 4 Packages 2024-09-13T14:10:41.140 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:10:41.142 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 79 M 2024-09-13T14:10:41.143 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 126 M 2024-09-13T14:10:41.143 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:10:41.252 INFO:teuthology.orchestra.run.smithi204.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 254 kB/s | 23 kB 00:00 2024-09-13T14:10:41.750 INFO:teuthology.orchestra.run.smithi096.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 12 MB/s | 35 MB 00:02 2024-09-13T14:10:42.436 INFO:teuthology.orchestra.run.smithi204.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 12 MB/s | 16 MB 00:01 2024-09-13T14:10:42.978 INFO:teuthology.orchestra.run.smithi204.stdout:(3/4): kernel-modules-core-5.14.0-505.el9.x86_6 17 MB/s | 29 MB 00:01 2024-09-13T14:10:43.303 INFO:teuthology.orchestra.run.smithi204.stdout:(4/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 16 MB/s | 35 MB 00:02 2024-09-13T14:10:43.303 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:10:43.303 INFO:teuthology.orchestra.run.smithi204.stdout:Total 36 MB/s | 79 MB 00:02 2024-09-13T14:10:43.987 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:10:44.346 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:10:44.346 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:10:45.336 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:10:45.336 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:10:46.424 INFO:teuthology.orchestra.run.smithi096.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 3.9 MB/s | 29 MB 00:07 2024-09-13T14:10:46.425 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:10:46.425 INFO:teuthology.orchestra.run.smithi096.stdout:Total 10 MB/s | 79 MB 00:07 2024-09-13T14:10:47.106 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:10:47.469 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:10:47.469 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:10:47.878 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:10:48.428 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:10:48.428 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:10:48.565 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T14:10:48.654 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:10:50.887 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:10:51.463 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:10:51.594 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T14:10:51.609 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:10:51.676 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:10:54.467 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:10:54.586 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:10:56.443 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:10:56.560 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:10:59.430 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:10:59.469 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:11:01.410 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:11:04.339 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:11:39.706 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:11:39.745 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:11:40.097 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:11:40.097 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T14:11:40.098 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:11:40.098 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:11:40.917 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:11:40.917 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:11:40.918 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:11:40.918 INFO:teuthology.orchestra.run.smithi204.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T14:11:40.918 INFO:teuthology.orchestra.run.smithi204.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T14:11:40.918 INFO:teuthology.orchestra.run.smithi204.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T14:11:40.918 INFO:teuthology.orchestra.run.smithi204.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T14:11:40.918 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:11:40.918 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:11:41.320 DEBUG:teuthology.orchestra.run.smithi204:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T14:11:41.454 INFO:teuthology.orchestra.run.smithi204.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T14:11:41.454 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T14:11:41.454 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T14:11:41.454 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T14:11:41.454 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T14:11:41.454 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-13T14:11:41.455 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi204.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T14:11:41.455 DEBUG:teuthology.orchestra.run.smithi204:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T14:11:41.513 INFO:teuthology.orchestra.run.smithi204.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T14:11:41.514 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T14:11:41.514 DEBUG:teuthology.orchestra.run.smithi204:> sudo rpm -qi grub2-tools 2024-09-13T14:11:41.553 INFO:teuthology.orchestra.run.smithi204.stdout:Name : grub2-tools 2024-09-13T14:11:41.553 INFO:teuthology.orchestra.run.smithi204.stdout:Epoch : 1 2024-09-13T14:11:41.554 INFO:teuthology.orchestra.run.smithi204.stdout:Version : 2.06 2024-09-13T14:11:41.554 INFO:teuthology.orchestra.run.smithi204.stdout:Release : 82.el9 2024-09-13T14:11:41.554 INFO:teuthology.orchestra.run.smithi204.stdout:Architecture: x86_64 2024-09-13T14:11:41.554 INFO:teuthology.orchestra.run.smithi204.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T14:11:41.554 INFO:teuthology.orchestra.run.smithi204.stdout:Group : Unspecified 2024-09-13T14:11:41.554 INFO:teuthology.orchestra.run.smithi204.stdout:Size : 8274670 2024-09-13T14:11:41.554 INFO:teuthology.orchestra.run.smithi204.stdout:License : GPLv3+ 2024-09-13T14:11:41.554 INFO:teuthology.orchestra.run.smithi204.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T14:11:41.554 INFO:teuthology.orchestra.run.smithi204.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T14:11:41.554 INFO:teuthology.orchestra.run.smithi204.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T14:11:41.554 INFO:teuthology.orchestra.run.smithi204.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T14:11:41.555 INFO:teuthology.orchestra.run.smithi204.stdout:Packager : builder@centos.org 2024-09-13T14:11:41.555 INFO:teuthology.orchestra.run.smithi204.stdout:Vendor : CentOS 2024-09-13T14:11:41.555 INFO:teuthology.orchestra.run.smithi204.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T14:11:41.555 INFO:teuthology.orchestra.run.smithi204.stdout:Summary : Support tools for GRUB. 2024-09-13T14:11:41.555 INFO:teuthology.orchestra.run.smithi204.stdout:Description : 2024-09-13T14:11:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:11:41.555 INFO:teuthology.orchestra.run.smithi204.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T14:11:41.555 INFO:teuthology.orchestra.run.smithi204.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T14:11:41.555 INFO:teuthology.orchestra.run.smithi204.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T14:11:41.555 INFO:teuthology.orchestra.run.smithi204.stdout:hardware devices. 2024-09-13T14:11:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:11:41.555 INFO:teuthology.orchestra.run.smithi204.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T14:11:41.556 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T14:11:41.556 INFO:teuthology.task.kernel:Updating grub on smithi204 to boot 5.14.0-505.el9.x86_64 2024-09-13T14:11:41.557 DEBUG:teuthology.orchestra.run.smithi204:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T14:11:42.194 INFO:teuthology.orchestra.run.smithi204.stderr:Generating grub configuration file ... 2024-09-13T14:11:42.681 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:11:42.720 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:11:43.144 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:11:43.144 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T14:11:43.144 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:11:43.144 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:11:43.537 INFO:teuthology.orchestra.run.smithi204.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T14:11:43.575 INFO:teuthology.orchestra.run.smithi204.stderr:done 2024-09-13T14:11:43.578 DEBUG:teuthology.orchestra.run.smithi204:> mktemp 2024-09-13T14:11:43.593 INFO:teuthology.orchestra.run.smithi204.stdout:/tmp/tmp.OnOrkANqWu 2024-09-13T14:11:43.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.OnOrkANqWu 2024-09-13T14:11:43.661 DEBUG:teuthology.orchestra.run.smithi204:> sudo chmod 0666 /tmp/tmp.OnOrkANqWu 2024-09-13T14:11:43.802 DEBUG:teuthology.orchestra.remote:smithi204:/tmp/tmp.OnOrkANqWu is 6KB 2024-09-13T14:11:43.851 DEBUG:teuthology.orchestra.run.smithi204:> rm -fr /tmp/tmp.OnOrkANqWu 2024-09-13T14:11:43.866 DEBUG:teuthology.orchestra.run.smithi204:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T14:11:43.930 INFO:teuthology.orchestra.run.smithi204.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T14:11:43.930 INFO:teuthology.orchestra.run.smithi204.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T14:11:43.930 INFO:teuthology.orchestra.run.smithi204.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T14:11:43.930 INFO:teuthology.orchestra.run.smithi204.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T14:11:43.931 DEBUG:teuthology.orchestra.run.smithi204:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T14:11:43.991 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:11:43.991 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:11:43.991 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:11:43.991 INFO:teuthology.orchestra.run.smithi096.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T14:11:43.991 INFO:teuthology.orchestra.run.smithi096.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T14:11:43.991 INFO:teuthology.orchestra.run.smithi096.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T14:11:43.992 INFO:teuthology.orchestra.run.smithi096.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T14:11:43.992 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:11:43.992 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:11:44.080 DEBUG:teuthology.orchestra.run.smithi204:> sudo shutdown -r now 2024-09-13T14:11:44.345 DEBUG:teuthology.orchestra.run.smithi096:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T14:11:44.428 INFO:teuthology.orchestra.run.smithi096.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T14:11:44.429 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T14:11:44.429 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T14:11:44.429 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T14:11:44.429 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T14:11:44.429 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-13T14:11:44.429 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi096.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T14:11:44.429 DEBUG:teuthology.orchestra.run.smithi096:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T14:11:44.487 INFO:teuthology.orchestra.run.smithi096.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T14:11:44.487 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T14:11:44.487 DEBUG:teuthology.orchestra.run.smithi096:> sudo rpm -qi grub2-tools 2024-09-13T14:11:44.533 INFO:teuthology.orchestra.run.smithi096.stdout:Name : grub2-tools 2024-09-13T14:11:44.533 INFO:teuthology.orchestra.run.smithi096.stdout:Epoch : 1 2024-09-13T14:11:44.533 INFO:teuthology.orchestra.run.smithi096.stdout:Version : 2.06 2024-09-13T14:11:44.533 INFO:teuthology.orchestra.run.smithi096.stdout:Release : 82.el9 2024-09-13T14:11:44.533 INFO:teuthology.orchestra.run.smithi096.stdout:Architecture: x86_64 2024-09-13T14:11:44.534 INFO:teuthology.orchestra.run.smithi096.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T14:11:44.534 INFO:teuthology.orchestra.run.smithi096.stdout:Group : Unspecified 2024-09-13T14:11:44.534 INFO:teuthology.orchestra.run.smithi096.stdout:Size : 8274670 2024-09-13T14:11:44.534 INFO:teuthology.orchestra.run.smithi096.stdout:License : GPLv3+ 2024-09-13T14:11:44.534 INFO:teuthology.orchestra.run.smithi096.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T14:11:44.534 INFO:teuthology.orchestra.run.smithi096.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T14:11:44.534 INFO:teuthology.orchestra.run.smithi096.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T14:11:44.534 INFO:teuthology.orchestra.run.smithi096.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T14:11:44.534 INFO:teuthology.orchestra.run.smithi096.stdout:Packager : builder@centos.org 2024-09-13T14:11:44.534 INFO:teuthology.orchestra.run.smithi096.stdout:Vendor : CentOS 2024-09-13T14:11:44.534 INFO:teuthology.orchestra.run.smithi096.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T14:11:44.534 INFO:teuthology.orchestra.run.smithi096.stdout:Summary : Support tools for GRUB. 2024-09-13T14:11:44.534 INFO:teuthology.orchestra.run.smithi096.stdout:Description : 2024-09-13T14:11:44.535 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:11:44.535 INFO:teuthology.orchestra.run.smithi096.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T14:11:44.535 INFO:teuthology.orchestra.run.smithi096.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T14:11:44.535 INFO:teuthology.orchestra.run.smithi096.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T14:11:44.535 INFO:teuthology.orchestra.run.smithi096.stdout:hardware devices. 2024-09-13T14:11:44.535 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:11:44.535 INFO:teuthology.orchestra.run.smithi096.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T14:11:44.536 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T14:11:44.536 INFO:teuthology.task.kernel:Updating grub on smithi096 to boot 5.14.0-505.el9.x86_64 2024-09-13T14:11:44.536 DEBUG:teuthology.orchestra.run.smithi096:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T14:11:45.093 INFO:teuthology.orchestra.run.smithi096.stderr:Generating grub configuration file ... 2024-09-13T14:11:46.467 INFO:teuthology.orchestra.run.smithi096.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T14:11:46.517 INFO:teuthology.orchestra.run.smithi096.stderr:done 2024-09-13T14:11:46.520 DEBUG:teuthology.orchestra.run.smithi096:> mktemp 2024-09-13T14:11:46.535 INFO:teuthology.orchestra.run.smithi096.stdout:/tmp/tmp.xaNYvlXmRb 2024-09-13T14:11:46.535 DEBUG:teuthology.orchestra.run.smithi096:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.xaNYvlXmRb 2024-09-13T14:11:46.605 DEBUG:teuthology.orchestra.run.smithi096:> sudo chmod 0666 /tmp/tmp.xaNYvlXmRb 2024-09-13T14:11:46.744 DEBUG:teuthology.orchestra.remote:smithi096:/tmp/tmp.xaNYvlXmRb is 6KB 2024-09-13T14:11:46.792 DEBUG:teuthology.orchestra.run.smithi096:> rm -fr /tmp/tmp.xaNYvlXmRb 2024-09-13T14:11:46.807 DEBUG:teuthology.orchestra.run.smithi096:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T14:11:46.870 INFO:teuthology.orchestra.run.smithi096.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T14:11:46.871 INFO:teuthology.orchestra.run.smithi096.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T14:11:46.871 INFO:teuthology.orchestra.run.smithi096.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T14:11:46.871 INFO:teuthology.orchestra.run.smithi096.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T14:11:46.872 DEBUG:teuthology.orchestra.run.smithi096:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T14:11:47.024 DEBUG:teuthology.orchestra.run.smithi096:> sudo shutdown -r now 2024-09-13T14:12:14.109 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-13T14:12:14.109 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi204.front.sepia.ceph.com' 2024-09-13T14:12:14.110 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi204.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:12:17.029 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-13T14:12:17.029 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi096.front.sepia.ceph.com' 2024-09-13T14:12:17.029 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi096.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:12:48.573 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.204 2024-09-13T14:12:51.645 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.96 2024-09-13T14:12:57.580 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi204.front.sepia.ceph.com' 2024-09-13T14:12:57.581 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi204.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:13:00.648 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi096.front.sepia.ceph.com' 2024-09-13T14:13:00.649 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi096.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:13:19.037 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.96 2024-09-13T14:13:31.050 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi096.front.sepia.ceph.com' 2024-09-13T14:13:31.051 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi096.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:13:31.554 DEBUG:teuthology.orchestra.run.smithi096:> true 2024-09-13T14:13:31.979 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi096.front.sepia.ceph.com' 2024-09-13T14:13:31.979 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-505.el9.x86_64"... 2024-09-13T14:13:31.979 DEBUG:teuthology.orchestra.run.smithi096:> uname -r 2024-09-13T14:13:31.993 INFO:teuthology.orchestra.run.smithi096.stdout:5.14.0-505.el9.x86_64 2024-09-13T14:13:31.993 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T14:13:31.993 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T14:13:31.994 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T14:13:32.995 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-13T14:13:32.995 DEBUG:teuthology.orchestra.run.smithi096:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T14:13:33.097 INFO:teuthology.orchestra.run.smithi096.stdout:ttyS1 2024-09-13T14:13:33.122 DEBUG:teuthology.parallel:result is None 2024-09-13T14:13:57.592 DEBUG:teuthology.orchestra.remote:timed out 2024-09-13T14:14:09.593 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi204.front.sepia.ceph.com' 2024-09-13T14:14:09.594 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi204.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:14:10.022 DEBUG:teuthology.orchestra.run.smithi204:> true 2024-09-13T14:14:10.367 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi204.front.sepia.ceph.com' 2024-09-13T14:14:10.367 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-505.el9.x86_64"... 2024-09-13T14:14:10.368 DEBUG:teuthology.orchestra.run.smithi204:> uname -r 2024-09-13T14:14:10.383 INFO:teuthology.orchestra.run.smithi204.stdout:5.14.0-505.el9.x86_64 2024-09-13T14:14:10.383 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T14:14:10.384 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T14:14:10.384 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T14:14:11.384 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-13T14:14:11.385 DEBUG:teuthology.orchestra.run.smithi204:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T14:14:11.484 INFO:teuthology.orchestra.run.smithi204.stdout:ttyS1 2024-09-13T14:14:11.506 DEBUG:teuthology.parallel:result is None 2024-09-13T14:14:11.506 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-13T14:14:11.514 INFO:teuthology.task.internal:Creating test directory... 2024-09-13T14:14:11.514 DEBUG:teuthology.orchestra.run.smithi096:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T14:14:11.517 DEBUG:teuthology.orchestra.run.smithi204:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T14:14:11.536 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-13T14:14:11.555 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-13T14:14:11.602 INFO:teuthology.task.internal:Creating archive directory... 2024-09-13T14:14:11.603 DEBUG:teuthology.orchestra.run.smithi096:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T14:14:11.605 DEBUG:teuthology.orchestra.run.smithi204:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T14:14:11.635 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-13T14:14:11.663 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-13T14:14:11.663 DEBUG:teuthology.orchestra.run.smithi096:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T14:14:11.673 DEBUG:teuthology.orchestra.run.smithi204:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T14:14:11.701 INFO:teuthology.orchestra.run.smithi096.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T14:14:11.712 INFO:teuthology.orchestra.run.smithi096.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T14:14:11.712 INFO:teuthology.orchestra.run.smithi204.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T14:14:11.724 INFO:teuthology.orchestra.run.smithi204.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T14:14:11.726 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-13T14:14:11.733 INFO:teuthology.task.internal:Configuring sudo... 2024-09-13T14:14:11.733 DEBUG:teuthology.orchestra.run.smithi096:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T14:14:11.757 DEBUG:teuthology.orchestra.run.smithi204:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T14:14:11.794 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-13T14:14:11.806 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-13T14:14:11.806 DEBUG:teuthology.orchestra.run.smithi096:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T14:14:11.826 DEBUG:teuthology.orchestra.run.smithi204:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T14:14:11.851 DEBUG:teuthology.orchestra.run.smithi096:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T14:14:11.933 DEBUG:teuthology.orchestra.run.smithi096:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T14:14:11.970 DEBUG:teuthology.orchestra.run.smithi096:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T14:14:12.060 DEBUG:teuthology.orchestra.run.smithi096:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T14:14:12.094 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:14:12.094 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T14:14:12.160 DEBUG:teuthology.orchestra.run.smithi204:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T14:14:12.213 DEBUG:teuthology.orchestra.run.smithi204:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T14:14:12.260 DEBUG:teuthology.orchestra.run.smithi204:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T14:14:12.352 DEBUG:teuthology.orchestra.run.smithi204:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T14:14:12.386 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-09-13T14:14:12.386 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T14:14:12.453 DEBUG:teuthology.orchestra.run.smithi096:> sudo service rsyslog restart 2024-09-13T14:14:12.456 DEBUG:teuthology.orchestra.run.smithi204:> sudo service rsyslog restart 2024-09-13T14:14:12.501 INFO:teuthology.orchestra.run.smithi096.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T14:14:12.539 INFO:teuthology.orchestra.run.smithi204.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T14:14:13.003 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-13T14:14:13.011 INFO:teuthology.task.internal:Starting timer... 2024-09-13T14:14:13.011 INFO:teuthology.run_tasks:Running task pcp... 2024-09-13T14:14:13.074 INFO:teuthology.run_tasks:Running task selinux... 2024-09-13T14:14:13.149 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-09-13T14:14:13.149 DEBUG:teuthology.orchestra.run.smithi096:> sudo service auditd rotate 2024-09-13T14:14:13.191 INFO:teuthology.orchestra.run.smithi096.stdout:Rotating logs: 2024-09-13T14:14:13.193 DEBUG:teuthology.orchestra.run.smithi204:> sudo service auditd rotate 2024-09-13T14:14:13.239 INFO:teuthology.orchestra.run.smithi204.stdout:Rotating logs: 2024-09-13T14:14:13.241 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-13T14:14:13.241 DEBUG:teuthology.orchestra.run.smithi096:> /usr/sbin/getenforce 2024-09-13T14:14:13.280 INFO:teuthology.orchestra.run.smithi096.stdout:Permissive 2024-09-13T14:14:13.280 DEBUG:teuthology.orchestra.run.smithi204:> /usr/sbin/getenforce 2024-09-13T14:14:13.307 INFO:teuthology.orchestra.run.smithi204.stdout:Permissive 2024-09-13T14:14:13.307 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi096.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi204.front.sepia.ceph.com': 'permissive'} 2024-09-13T14:14:13.308 DEBUG:teuthology.orchestra.run.smithi096:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-13T14:14:13.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:14:13.346 DEBUG:teuthology.orchestra.run.smithi204:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-13T14:14:13.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:14:13.377 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-13T14:14:13.377 DEBUG:teuthology.orchestra.run.smithi096:> sudo /usr/sbin/setenforce permissive 2024-09-13T14:14:13.427 DEBUG:teuthology.orchestra.run.smithi204:> sudo /usr/sbin/setenforce permissive 2024-09-13T14:14:13.466 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-13T14:14:13.476 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-13T14:14:13.487 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-13T14:14:13.488 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi096.front.sepia.ceph.com,smithi204.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-13T14:17:55.141 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi096.front.sepia.ceph.com'), Remote(name='ubuntu@smithi204.front.sepia.ceph.com')] 2024-09-13T14:17:55.142 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi096.front.sepia.ceph.com' 2024-09-13T14:17:55.143 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi096.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:17:55.224 DEBUG:teuthology.orchestra.run.smithi096:> true 2024-09-13T14:17:55.307 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi096.front.sepia.ceph.com' 2024-09-13T14:17:55.308 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi204.front.sepia.ceph.com' 2024-09-13T14:17:55.308 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi204.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:17:55.409 DEBUG:teuthology.orchestra.run.smithi204:> true 2024-09-13T14:17:55.488 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi204.front.sepia.ceph.com' 2024-09-13T14:17:55.488 INFO:teuthology.run_tasks:Running task clock... 2024-09-13T14:17:55.498 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-13T14:17:55.499 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T14:17:55.499 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T14:17:55.501 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T14:17:55.502 DEBUG:teuthology.orchestra.run.smithi204:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T14:17:55.528 INFO:teuthology.orchestra.run.smithi096.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T14:17:55.542 INFO:teuthology.orchestra.run.smithi096.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T14:17:55.557 INFO:teuthology.orchestra.run.smithi204.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T14:17:55.566 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: ntpd: command not found 2024-09-13T14:17:55.571 INFO:teuthology.orchestra.run.smithi204.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T14:17:55.578 INFO:teuthology.orchestra.run.smithi096.stdout:506 Cannot talk to daemon 2024-09-13T14:17:55.592 INFO:teuthology.orchestra.run.smithi096.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T14:17:55.595 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: ntpd: command not found 2024-09-13T14:17:55.606 INFO:teuthology.orchestra.run.smithi096.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T14:17:55.606 INFO:teuthology.orchestra.run.smithi204.stdout:506 Cannot talk to daemon 2024-09-13T14:17:55.620 INFO:teuthology.orchestra.run.smithi204.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T14:17:55.635 INFO:teuthology.orchestra.run.smithi204.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T14:17:55.669 INFO:teuthology.orchestra.run.smithi096.stderr:bash: line 1: ntpq: command not found 2024-09-13T14:17:55.673 INFO:teuthology.orchestra.run.smithi096.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T14:17:55.673 INFO:teuthology.orchestra.run.smithi096.stdout:=============================================================================== 2024-09-13T14:17:55.673 INFO:teuthology.orchestra.run.smithi096.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:17:55.673 INFO:teuthology.orchestra.run.smithi096.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:17:55.673 INFO:teuthology.orchestra.run.smithi096.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:17:55.673 INFO:teuthology.orchestra.run.smithi096.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:17:55.689 INFO:teuthology.orchestra.run.smithi204.stderr:bash: line 1: ntpq: command not found 2024-09-13T14:17:55.693 INFO:teuthology.orchestra.run.smithi204.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T14:17:55.693 INFO:teuthology.orchestra.run.smithi204.stdout:=============================================================================== 2024-09-13T14:17:55.693 INFO:teuthology.orchestra.run.smithi204.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:17:55.693 INFO:teuthology.orchestra.run.smithi204.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:17:55.694 INFO:teuthology.orchestra.run.smithi204.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:17:55.694 INFO:teuthology.orchestra.run.smithi204.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:17:55.694 INFO:teuthology.run_tasks:Running task pexec... 2024-09-13T14:17:55.703 INFO:teuthology.task.pexec:Executing custom commands... 2024-09-13T14:17:55.703 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi096.front.sepia.ceph.com 2024-09-13T14:17:55.704 DEBUG:teuthology.orchestra.run.smithi096:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-13T14:17:55.704 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi204.front.sepia.ceph.com 2024-09-13T14:17:55.704 DEBUG:teuthology.orchestra.run.smithi204:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-13T14:17:56.062 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:17:56.075 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:17:56.075 INFO:teuthology.orchestra.run.smithi096.stdout: Package Architecture Version Repository Size 2024-09-13T14:17:56.076 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:17:56.076 INFO:teuthology.orchestra.run.smithi096.stdout:Removing: 2024-09-13T14:17:56.076 INFO:teuthology.orchestra.run.smithi096.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-13T14:17:56.076 INFO:teuthology.orchestra.run.smithi096.stdout:Removing unused dependencies: 2024-09-13T14:17:56.076 INFO:teuthology.orchestra.run.smithi096.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-13T14:17:56.076 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:17:56.076 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:17:56.076 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:17:56.076 INFO:teuthology.orchestra.run.smithi096.stdout:Remove 2 Packages 2024-09-13T14:17:56.076 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:17:56.076 INFO:teuthology.orchestra.run.smithi096.stdout:Freed space: 5.5 M 2024-09-13T14:17:56.076 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:17:56.080 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:17:56.080 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:17:56.086 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:17:56.096 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:17:56.097 INFO:teuthology.orchestra.run.smithi204.stdout: Package Architecture Version Repository Size 2024-09-13T14:17:56.097 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:17:56.097 INFO:teuthology.orchestra.run.smithi204.stdout:Removing: 2024-09-13T14:17:56.097 INFO:teuthology.orchestra.run.smithi204.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-13T14:17:56.097 INFO:teuthology.orchestra.run.smithi204.stdout:Removing unused dependencies: 2024-09-13T14:17:56.097 INFO:teuthology.orchestra.run.smithi204.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-13T14:17:56.097 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:17:56.097 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:17:56.097 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:17:56.097 INFO:teuthology.orchestra.run.smithi204.stdout:Remove 2 Packages 2024-09-13T14:17:56.097 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:17:56.097 INFO:teuthology.orchestra.run.smithi204.stdout:Freed space: 5.5 M 2024-09-13T14:17:56.097 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:17:56.101 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:17:56.101 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:17:56.148 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:17:56.148 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:17:56.159 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:17:56.160 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:17:56.287 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:17:56.305 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:17:56.368 INFO:teuthology.orchestra.run.smithi096.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-13T14:17:56.374 INFO:teuthology.orchestra.run.smithi204.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-13T14:17:56.429 INFO:teuthology.orchestra.run.smithi096.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:17:56.436 INFO:teuthology.orchestra.run.smithi204.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:17:56.896 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:17:56.897 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-13T14:17:56.976 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:17:56.976 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-13T14:17:57.147 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-13T14:17:57.147 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:17:57.147 INFO:teuthology.orchestra.run.smithi096.stdout:Removed: 2024-09-13T14:17:57.147 INFO:teuthology.orchestra.run.smithi096.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:17:57.147 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:17:57.147 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:17:57.296 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-13T14:17:57.297 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:17:57.297 INFO:teuthology.orchestra.run.smithi204.stdout:Removed: 2024-09-13T14:17:57.297 INFO:teuthology.orchestra.run.smithi204.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:17:57.297 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:17:57.297 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:17:57.991 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:47 ago on Fri 13 Sep 2024 02:17:10 PM UTC. 2024-09-13T14:17:58.129 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:00:47 ago on Fri 13 Sep 2024 02:17:10 PM UTC. 2024-09-13T14:17:58.131 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:17:58.132 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:17:58.132 INFO:teuthology.orchestra.run.smithi096.stdout: Package Architecture Version Repository Size 2024-09-13T14:17:58.132 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:17:58.132 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:17:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-13T14:17:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-13T14:17:58.133 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2024-09-13T14:17:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-13T14:17:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-13T14:17:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-13T14:17:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-13T14:17:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:17:58.133 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:17:58.133 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:17:58.133 INFO:teuthology.orchestra.run.smithi096.stdout:Install 6 Packages 2024-09-13T14:17:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:17:58.143 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 2.0 M 2024-09-13T14:17:58.143 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 9.2 M 2024-09-13T14:17:58.143 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:17:58.266 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:17:58.266 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:17:58.266 INFO:teuthology.orchestra.run.smithi204.stdout: Package Architecture Version Repository Size 2024-09-13T14:17:58.267 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:17:58.267 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:17:58.267 INFO:teuthology.orchestra.run.smithi204.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-13T14:17:58.267 INFO:teuthology.orchestra.run.smithi204.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-13T14:17:58.267 INFO:teuthology.orchestra.run.smithi204.stdout:Installing dependencies: 2024-09-13T14:17:58.267 INFO:teuthology.orchestra.run.smithi204.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-13T14:17:58.267 INFO:teuthology.orchestra.run.smithi204.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-13T14:17:58.267 INFO:teuthology.orchestra.run.smithi204.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-13T14:17:58.267 INFO:teuthology.orchestra.run.smithi204.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-13T14:17:58.267 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:17:58.267 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:17:58.267 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:17:58.267 INFO:teuthology.orchestra.run.smithi204.stdout:Install 6 Packages 2024-09-13T14:17:58.268 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:17:58.269 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 2.0 M 2024-09-13T14:17:58.270 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 9.2 M 2024-09-13T14:17:58.270 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:17:58.519 INFO:teuthology.orchestra.run.smithi096.stdout:(1/6): libnvme-1.9-3.el9.x86_64.rpm 902 kB/s | 106 kB 00:00 2024-09-13T14:17:58.536 INFO:teuthology.orchestra.run.smithi096.stdout:(2/6): nvmetcli-0.7-3.el9.noarch.rpm 330 kB/s | 44 kB 00:00 2024-09-13T14:17:58.595 INFO:teuthology.orchestra.run.smithi096.stdout:(3/6): nvme-cli-2.9.1-6.el9.x86_64.rpm 4.6 MB/s | 906 kB 00:00 2024-09-13T14:17:58.621 INFO:teuthology.orchestra.run.smithi096.stdout:(4/6): python3-configshell-1.1.30-1.el9.noarch. 710 kB/s | 72 kB 00:00 2024-09-13T14:17:58.646 INFO:teuthology.orchestra.run.smithi096.stdout:(5/6): python3-kmod-0.9-32.el9.x86_64.rpm 771 kB/s | 84 kB 00:00 2024-09-13T14:17:58.705 INFO:teuthology.orchestra.run.smithi096.stdout:(6/6): python3-urwid-2.1.2-4.el9.x86_64.rpm 7.5 MB/s | 837 kB 00:00 2024-09-13T14:17:58.705 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:17:58.705 INFO:teuthology.orchestra.run.smithi096.stdout:Total 3.6 MB/s | 2.0 MB 00:00 2024-09-13T14:17:58.779 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:17:58.793 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:17:58.793 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:17:59.017 INFO:teuthology.orchestra.run.smithi204.stdout:(1/6): nvmetcli-0.7-3.el9.noarch.rpm 69 kB/s | 44 kB 00:00 2024-09-13T14:17:59.042 INFO:teuthology.orchestra.run.smithi204.stdout:(2/6): libnvme-1.9-3.el9.x86_64.rpm 160 kB/s | 106 kB 00:00 2024-09-13T14:17:59.070 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:17:59.070 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:17:59.284 INFO:teuthology.orchestra.run.smithi204.stdout:(3/6): nvme-cli-2.9.1-6.el9.x86_64.rpm 998 kB/s | 906 kB 00:00 2024-09-13T14:17:59.481 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:17:59.581 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/6 2024-09-13T14:17:59.616 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/6 2024-09-13T14:17:59.643 INFO:teuthology.orchestra.run.smithi204.stdout:(4/6): python3-configshell-1.1.30-1.el9.noarch. 115 kB/s | 72 kB 00:00 2024-09-13T14:17:59.645 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/6 2024-09-13T14:17:59.718 INFO:teuthology.orchestra.run.smithi204.stdout:(5/6): python3-kmod-0.9-32.el9.x86_64.rpm 125 kB/s | 84 kB 00:00 2024-09-13T14:17:59.848 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/6 2024-09-13T14:17:59.877 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-09-13T14:18:00.177 INFO:teuthology.orchestra.run.smithi204.stdout:(6/6): python3-urwid-2.1.2-4.el9.x86_64.rpm 946 kB/s | 837 kB 00:00 2024-09-13T14:18:00.177 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:18:00.177 INFO:teuthology.orchestra.run.smithi204.stdout:Total 1.0 MB/s | 2.0 MB 00:01 2024-09-13T14:18:00.257 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:18:00.271 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:18:00.271 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:18:00.319 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-09-13T14:18:00.349 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/6 2024-09-13T14:18:00.580 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:18:00.580 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:18:01.025 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:18:01.113 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/6 2024-09-13T14:18:01.148 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/6 2024-09-13T14:18:01.176 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/6 2024-09-13T14:18:01.385 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/6 2024-09-13T14:18:01.389 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/6 2024-09-13T14:18:01.389 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/6 2024-09-13T14:18:01.389 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/6 2024-09-13T14:18:01.389 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/6 2024-09-13T14:18:01.389 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/6 2024-09-13T14:18:01.389 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/6 2024-09-13T14:18:01.424 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-09-13T14:18:01.729 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/6 2024-09-13T14:18:01.729 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:18:01.729 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:18:01.729 INFO:teuthology.orchestra.run.smithi096.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:18:01.729 INFO:teuthology.orchestra.run.smithi096.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-13T14:18:01.729 INFO:teuthology.orchestra.run.smithi096.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-13T14:18:01.729 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:18:01.729 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:18:01.856 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-09-13T14:18:01.886 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/6 2024-09-13T14:18:02.083 DEBUG:teuthology.parallel:result is None 2024-09-13T14:18:02.863 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/6 2024-09-13T14:18:02.863 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/6 2024-09-13T14:18:02.863 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/6 2024-09-13T14:18:02.863 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/6 2024-09-13T14:18:02.864 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/6 2024-09-13T14:18:02.864 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/6 2024-09-13T14:18:03.237 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/6 2024-09-13T14:18:03.237 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:18:03.237 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:18:03.237 INFO:teuthology.orchestra.run.smithi204.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:18:03.237 INFO:teuthology.orchestra.run.smithi204.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-13T14:18:03.237 INFO:teuthology.orchestra.run.smithi204.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-13T14:18:03.237 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:18:03.237 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:18:03.549 DEBUG:teuthology.parallel:result is None 2024-09-13T14:18:03.550 INFO:teuthology.run_tasks:Running task install... 2024-09-13T14:18:03.558 DEBUG:teuthology.task.install:project ceph 2024-09-13T14:18:03.558 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}} 2024-09-13T14:18:03.559 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'} 2024-09-13T14:18:03.559 INFO:teuthology.task.install:Using flavor: default 2024-09-13T14:18:03.565 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-13T14:18:03.565 INFO:teuthology.task.install:extra packages: [] 2024-09-13T14:18:03.565 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9', 'tag': None, 'wait_for_package': False} 2024-09-13T14:18:03.566 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:18:03.566 INFO:teuthology.packaging:ref: None 2024-09-13T14:18:03.566 INFO:teuthology.packaging:tag: None 2024-09-13T14:18:03.566 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:18:03.566 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:18:03.566 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-13T14:18:03.568 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9', 'tag': None, 'wait_for_package': False} 2024-09-13T14:18:03.569 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:18:03.569 INFO:teuthology.packaging:ref: None 2024-09-13T14:18:03.569 INFO:teuthology.packaging:tag: None 2024-09-13T14:18:03.569 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:18:03.569 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:18:03.569 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-13T14:18:03.787 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/ 2024-09-13T14:18:03.788 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1627.gbbdadfa3 2024-09-13T14:18:03.841 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/ 2024-09-13T14:18:03.841 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1627.gbbdadfa3 2024-09-13T14:18:03.926 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-13T14:18:03.926 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-09-13T14:18:03.926 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-13T14:18:03.955 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-13T14:18:03.955 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:18:03.955 INFO:teuthology.packaging:ref: None 2024-09-13T14:18:03.955 INFO:teuthology.packaging:tag: None 2024-09-13T14:18:03.955 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:18:03.955 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:18:03.955 DEBUG:teuthology.orchestra.run.smithi204:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-13T14:18:03.961 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-13T14:18:03.961 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:18:03.962 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-13T14:18:03.995 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-13T14:18:03.995 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:18:03.995 INFO:teuthology.packaging:ref: None 2024-09-13T14:18:03.995 INFO:teuthology.packaging:tag: None 2024-09-13T14:18:03.996 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:18:03.996 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:18:03.996 DEBUG:teuthology.orchestra.run.smithi096:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-13T14:18:04.022 DEBUG:teuthology.orchestra.run.smithi204:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-13T14:18:04.066 DEBUG:teuthology.orchestra.run.smithi096:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-13T14:18:04.100 DEBUG:teuthology.orchestra.run.smithi204:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-13T14:18:04.145 DEBUG:teuthology.orchestra.run.smithi096:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-13T14:18:04.164 INFO:teuthology.orchestra.run.smithi204.stdout:check_obsoletes = 1 2024-09-13T14:18:04.165 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum clean all 2024-09-13T14:18:04.210 INFO:teuthology.orchestra.run.smithi096.stdout:check_obsoletes = 1 2024-09-13T14:18:04.212 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum clean all 2024-09-13T14:18:04.394 INFO:teuthology.orchestra.run.smithi204.stdout:45 files removed 2024-09-13T14:18:04.421 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install ceph-radosgw 2024-09-13T14:18:04.429 INFO:teuthology.orchestra.run.smithi096.stdout:45 files removed 2024-09-13T14:18:04.457 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-radosgw 2024-09-13T14:18:04.976 INFO:teuthology.orchestra.run.smithi204.stdout:ceph packages for x86_64 287 kB/s | 77 kB 00:00 2024-09-13T14:18:05.012 INFO:teuthology.orchestra.run.smithi096.stdout:ceph packages for x86_64 283 kB/s | 77 kB 00:00 2024-09-13T14:18:05.260 INFO:teuthology.orchestra.run.smithi204.stdout:ceph noarch packages 45 kB/s | 11 kB 00:00 2024-09-13T14:18:05.279 INFO:teuthology.orchestra.run.smithi096.stdout:ceph noarch packages 48 kB/s | 11 kB 00:00 2024-09-13T14:18:05.535 INFO:teuthology.orchestra.run.smithi204.stdout:ceph source packages 7.5 kB/s | 1.8 kB 00:00 2024-09-13T14:18:05.538 INFO:teuthology.orchestra.run.smithi096.stdout:ceph source packages 8.0 kB/s | 1.8 kB 00:00 2024-09-13T14:18:06.044 INFO:teuthology.orchestra.run.smithi204.stdout:CentOS Stream 9 - BaseOS 17 MB/s | 8.2 MB 00:00 2024-09-13T14:18:06.097 INFO:teuthology.orchestra.run.smithi096.stdout:CentOS Stream 9 - BaseOS 16 MB/s | 8.2 MB 00:00 2024-09-13T14:18:07.754 INFO:teuthology.orchestra.run.smithi204.stdout:CentOS Stream 9 - AppStream 33 MB/s | 20 MB 00:00 2024-09-13T14:18:14.346 INFO:teuthology.orchestra.run.smithi204.stdout:CentOS Stream 9 - CRB 2.4 MB/s | 6.5 MB 00:02 2024-09-13T14:18:15.405 INFO:teuthology.orchestra.run.smithi096.stdout:CentOS Stream 9 - AppStream 2.5 MB/s | 20 MB 00:08 2024-09-13T14:18:15.997 INFO:teuthology.orchestra.run.smithi204.stdout:CentOS Stream 9 - Extras packages 65 kB/s | 19 kB 00:00 2024-09-13T14:18:17.718 INFO:teuthology.orchestra.run.smithi204.stdout:Extra Packages for Enterprise Linux 14 MB/s | 23 MB 00:01 2024-09-13T14:18:20.964 INFO:teuthology.orchestra.run.smithi096.stdout:CentOS Stream 9 - CRB 3.8 MB/s | 6.5 MB 00:01 2024-09-13T14:18:22.617 INFO:teuthology.orchestra.run.smithi096.stdout:CentOS Stream 9 - Extras packages 66 kB/s | 19 kB 00:00 2024-09-13T14:18:23.199 INFO:teuthology.orchestra.run.smithi204.stdout:lab-extras 47 kB/s | 1.7 kB 00:00 2024-09-13T14:18:24.094 INFO:teuthology.orchestra.run.smithi096.stdout:Extra Packages for Enterprise Linux 17 MB/s | 23 MB 00:01 2024-09-13T14:18:25.616 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:18:25.617 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:18:25.617 INFO:teuthology.orchestra.run.smithi204.stdout: Package Arch Version Repository Size 2024-09-13T14:18:25.618 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:18:25.618 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:18:25.618 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-radosgw x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 14 M 2024-09-13T14:18:25.618 INFO:teuthology.orchestra.run.smithi204.stdout:Upgrading: 2024-09-13T14:18:25.618 INFO:teuthology.orchestra.run.smithi204.stdout: librados2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 3.1 M 2024-09-13T14:18:25.618 INFO:teuthology.orchestra.run.smithi204.stdout: librbd1 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.9 M 2024-09-13T14:18:25.618 INFO:teuthology.orchestra.run.smithi204.stdout:Installing dependencies: 2024-09-13T14:18:25.618 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-base x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 5.0 M 2024-09-13T14:18:25.618 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-common x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 22 M 2024-09-13T14:18:25.618 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-selinux x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 25 k 2024-09-13T14:18:25.618 INFO:teuthology.orchestra.run.smithi204.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-13T14:18:25.618 INFO:teuthology.orchestra.run.smithi204.stdout: libcephfs2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 637 k 2024-09-13T14:18:25.619 INFO:teuthology.orchestra.run.smithi204.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-13T14:18:25.619 INFO:teuthology.orchestra.run.smithi204.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-13T14:18:25.619 INFO:teuthology.orchestra.run.smithi204.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-13T14:18:25.619 INFO:teuthology.orchestra.run.smithi204.stdout: libradosstriper1 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 457 k 2024-09-13T14:18:25.619 INFO:teuthology.orchestra.run.smithi204.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-13T14:18:25.619 INFO:teuthology.orchestra.run.smithi204.stdout: librgw2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 4.9 M 2024-09-13T14:18:25.619 INFO:teuthology.orchestra.run.smithi204.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-13T14:18:25.619 INFO:teuthology.orchestra.run.smithi204.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-13T14:18:25.619 INFO:teuthology.orchestra.run.smithi204.stdout: python3-ceph-argparse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 45 k 2024-09-13T14:18:25.619 INFO:teuthology.orchestra.run.smithi204.stdout: python3-ceph-common x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 120 k 2024-09-13T14:18:25.619 INFO:teuthology.orchestra.run.smithi204.stdout: python3-cephfs x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 156 k 2024-09-13T14:18:25.620 INFO:teuthology.orchestra.run.smithi204.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-13T14:18:25.620 INFO:teuthology.orchestra.run.smithi204.stdout: python3-rados x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 320 k 2024-09-13T14:18:25.620 INFO:teuthology.orchestra.run.smithi204.stdout: python3-rbd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 294 k 2024-09-13T14:18:25.620 INFO:teuthology.orchestra.run.smithi204.stdout: python3-rgw x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 99 k 2024-09-13T14:18:25.620 INFO:teuthology.orchestra.run.smithi204.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-13T14:18:25.620 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:18:25.620 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:18:25.620 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:18:25.620 INFO:teuthology.orchestra.run.smithi204.stdout:Install 22 Packages 2024-09-13T14:18:25.620 INFO:teuthology.orchestra.run.smithi204.stdout:Upgrade 2 Packages 2024-09-13T14:18:25.620 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:18:25.621 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 58 M 2024-09-13T14:18:25.621 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:18:26.258 INFO:teuthology.orchestra.run.smithi204.stdout:(1/24): ceph-base-17.2.7-1627.gbbdadfa3.el9.x86 12 MB/s | 5.0 MB 00:00 2024-09-13T14:18:26.309 INFO:teuthology.orchestra.run.smithi204.stdout:(2/24): ceph-selinux-17.2.7-1627.gbbdadfa3.el9. 504 kB/s | 25 kB 00:00 2024-09-13T14:18:26.417 INFO:teuthology.orchestra.run.smithi204.stdout:(3/24): libcephfs2-17.2.7-1627.gbbdadfa3.el9.x8 5.7 MB/s | 637 kB 00:00 2024-09-13T14:18:26.492 INFO:teuthology.orchestra.run.smithi204.stdout:(4/24): libradosstriper1-17.2.7-1627.gbbdadfa3. 6.0 MB/s | 457 kB 00:00 2024-09-13T14:18:26.826 INFO:teuthology.orchestra.run.smithi204.stdout:(5/24): librgw2-17.2.7-1627.gbbdadfa3.el9.x86_6 15 MB/s | 4.9 MB 00:00 2024-09-13T14:18:27.060 INFO:teuthology.orchestra.run.smithi204.stdout:(6/24): ceph-common-17.2.7-1627.gbbdadfa3.el9.x 18 MB/s | 22 MB 00:01 2024-09-13T14:18:27.077 INFO:teuthology.orchestra.run.smithi204.stdout:(7/24): python3-ceph-argparse-17.2.7-1627.gbbda 180 kB/s | 45 kB 00:00 2024-09-13T14:18:27.111 INFO:teuthology.orchestra.run.smithi204.stdout:(8/24): python3-ceph-common-17.2.7-1627.gbbdadf 2.3 MB/s | 120 kB 00:00 2024-09-13T14:18:27.135 INFO:teuthology.orchestra.run.smithi204.stdout:(9/24): python3-cephfs-17.2.7-1627.gbbdadfa3.el 2.6 MB/s | 156 kB 00:00 2024-09-13T14:18:27.161 INFO:teuthology.orchestra.run.smithi204.stdout:(10/24): python3-rados-17.2.7-1627.gbbdadfa3.el 6.2 MB/s | 320 kB 00:00 2024-09-13T14:18:27.186 INFO:teuthology.orchestra.run.smithi204.stdout:(11/24): python3-rbd-17.2.7-1627.gbbdadfa3.el9. 5.7 MB/s | 294 kB 00:00 2024-09-13T14:18:27.211 INFO:teuthology.orchestra.run.smithi204.stdout:(12/24): python3-rgw-17.2.7-1627.gbbdadfa3.el9. 1.9 MB/s | 99 kB 00:00 2024-09-13T14:18:27.354 INFO:teuthology.orchestra.run.smithi204.stdout:(13/24): ceph-radosgw-17.2.7-1627.gbbdadfa3.el9 9.5 MB/s | 14 MB 00:01 2024-09-13T14:18:27.870 INFO:teuthology.orchestra.run.smithi204.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 69 kB/s | 45 kB 00:00 2024-09-13T14:18:28.004 INFO:teuthology.orchestra.run.smithi204.stdout:(15/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 196 kB/s | 160 kB 00:00 2024-09-13T14:18:28.288 INFO:teuthology.orchestra.run.smithi204.stdout:(16/24): librdkafka-1.6.1-102.el9.x86_64.rpm 710 kB/s | 662 kB 00:00 2024-09-13T14:18:28.346 INFO:teuthology.orchestra.run.smithi204.stdout:(17/24): gperftools-libs-2.9.1-3.el9.x86_64.rpm 5.2 MB/s | 308 kB 00:00 2024-09-13T14:18:28.380 INFO:teuthology.orchestra.run.smithi204.stdout:(18/24): liboath-2.6.7-2.el9.x86_64.rpm 1.4 MB/s | 49 kB 00:00 2024-09-13T14:18:28.405 INFO:teuthology.orchestra.run.smithi204.stdout:(19/24): libunwind-1.6.2-1.el9.x86_64.rpm 2.6 MB/s | 67 kB 00:00 2024-09-13T14:18:28.572 INFO:teuthology.orchestra.run.smithi204.stdout:(20/24): thrift-0.15.0-2.el9.x86_64.rpm 9.5 MB/s | 1.6 MB 00:00 2024-09-13T14:18:28.597 INFO:teuthology.orchestra.run.smithi204.stdout:(21/24): python3-prettytable-0.7.2-27.el9.noarc 71 kB/s | 42 kB 00:00 2024-09-13T14:18:28.647 INFO:teuthology.orchestra.run.smithi204.stdout:(22/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 377 kB/s | 292 kB 00:00 2024-09-13T14:18:28.823 INFO:teuthology.orchestra.run.smithi204.stdout:(23/24): librados2-17.2.7-1627.gbbdadfa3.el9.x8 12 MB/s | 3.1 MB 00:00 2024-09-13T14:18:28.906 INFO:teuthology.orchestra.run.smithi204.stdout:(24/24): librbd1-17.2.7-1627.gbbdadfa3.el9.x86_ 9.5 MB/s | 2.9 MB 00:00 2024-09-13T14:18:28.911 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:18:28.911 INFO:teuthology.orchestra.run.smithi204.stdout:Total 18 MB/s | 58 MB 00:03 2024-09-13T14:18:28.976 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:18:29.011 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:18:29.011 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:18:29.494 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:18:29.494 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:18:29.507 INFO:teuthology.orchestra.run.smithi096.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-09-13T14:18:30.691 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:18:31.082 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-09-13T14:18:31.145 INFO:teuthology.orchestra.run.smithi204.stdout: Upgrading : librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/26 2024-09-13T14:18:31.220 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/26 2024-09-13T14:18:31.308 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/26 2024-09-13T14:18:31.355 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/26 2024-09-13T14:18:31.444 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/26 2024-09-13T14:18:31.489 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-09-13T14:18:31.535 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-09-13T14:18:31.884 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:18:31.886 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:18:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2024-09-13T14:18:31.886 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:18:31.886 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:18:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-radosgw x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 14 M 2024-09-13T14:18:31.887 INFO:teuthology.orchestra.run.smithi096.stdout:Upgrading: 2024-09-13T14:18:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: librados2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 3.1 M 2024-09-13T14:18:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: librbd1 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.9 M 2024-09-13T14:18:31.887 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2024-09-13T14:18:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-base x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 5.0 M 2024-09-13T14:18:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-common x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 22 M 2024-09-13T14:18:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-selinux x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 25 k 2024-09-13T14:18:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-13T14:18:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: libcephfs2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 637 k 2024-09-13T14:18:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-13T14:18:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-13T14:18:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-13T14:18:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: libradosstriper1 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 457 k 2024-09-13T14:18:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-13T14:18:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: librgw2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 4.9 M 2024-09-13T14:18:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-13T14:18:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-13T14:18:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: python3-ceph-argparse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 45 k 2024-09-13T14:18:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: python3-ceph-common x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 120 k 2024-09-13T14:18:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cephfs x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 156 k 2024-09-13T14:18:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-13T14:18:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rados x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 320 k 2024-09-13T14:18:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rbd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 294 k 2024-09-13T14:18:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rgw x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 99 k 2024-09-13T14:18:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-13T14:18:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:18:31.889 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:18:31.889 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:18:31.889 INFO:teuthology.orchestra.run.smithi096.stdout:Install 22 Packages 2024-09-13T14:18:31.889 INFO:teuthology.orchestra.run.smithi096.stdout:Upgrade 2 Packages 2024-09-13T14:18:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:18:31.889 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 58 M 2024-09-13T14:18:31.890 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:18:31.975 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-09-13T14:18:32.008 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 8/26 2024-09-13T14:18:32.071 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 8/26 2024-09-13T14:18:32.099 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-09-13T14:18:32.143 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el9. 10/26 2024-09-13T14:18:32.179 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 11/26 2024-09-13T14:18:32.242 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 12/26 2024-09-13T14:18:32.269 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_6 13/26 2024-09-13T14:18:32.335 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_6 13/26 2024-09-13T14:18:32.374 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-09-13T14:18:32.420 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/26 2024-09-13T14:18:32.458 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-09-13T14:18:32.745 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-09-13T14:18:32.792 INFO:teuthology.orchestra.run.smithi096.stdout:(1/24): ceph-base-17.2.7-1627.gbbdadfa3.el9.x86 10 MB/s | 5.0 MB 00:00 2024-09-13T14:18:32.815 INFO:teuthology.orchestra.run.smithi204.stdout: Upgrading : librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 18/26 2024-09-13T14:18:32.867 INFO:teuthology.orchestra.run.smithi096.stdout:(2/24): ceph-selinux-17.2.7-1627.gbbdadfa3.el9. 336 kB/s | 25 kB 00:00 2024-09-13T14:18:32.888 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 18/26 2024-09-13T14:18:32.961 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 19/26 2024-09-13T14:18:32.967 INFO:teuthology.orchestra.run.smithi096.stdout:(3/24): libcephfs2-17.2.7-1627.gbbdadfa3.el9.x8 6.2 MB/s | 637 kB 00:00 2024-09-13T14:18:32.992 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x8 20/26 2024-09-13T14:18:33.084 INFO:teuthology.orchestra.run.smithi096.stdout:(4/24): libradosstriper1-17.2.7-1627.gbbdadfa3. 3.8 MB/s | 457 kB 00:00 2024-09-13T14:18:33.285 INFO:teuthology.orchestra.run.smithi096.stdout:(5/24): ceph-radosgw-17.2.7-1627.gbbdadfa3.el9. 15 MB/s | 14 MB 00:00 2024-09-13T14:18:33.318 INFO:teuthology.orchestra.run.smithi096.stdout:(6/24): python3-ceph-argparse-17.2.7-1627.gbbda 1.3 MB/s | 45 kB 00:00 2024-09-13T14:18:33.410 INFO:teuthology.orchestra.run.smithi096.stdout:(7/24): librgw2-17.2.7-1627.gbbdadfa3.el9.x86_6 15 MB/s | 4.9 MB 00:00 2024-09-13T14:18:33.435 INFO:teuthology.orchestra.run.smithi096.stdout:(8/24): python3-ceph-common-17.2.7-1627.gbbdadf 1.0 MB/s | 120 kB 00:00 2024-09-13T14:18:33.461 INFO:teuthology.orchestra.run.smithi096.stdout:(9/24): python3-cephfs-17.2.7-1627.gbbdadfa3.el 3.0 MB/s | 156 kB 00:00 2024-09-13T14:18:33.486 INFO:teuthology.orchestra.run.smithi096.stdout:(10/24): python3-rados-17.2.7-1627.gbbdadfa3.el 6.2 MB/s | 320 kB 00:00 2024-09-13T14:18:33.695 INFO:teuthology.orchestra.run.smithi096.stdout:(11/24): ceph-common-17.2.7-1627.gbbdadfa3.el9. 16 MB/s | 22 MB 00:01 2024-09-13T14:18:33.720 INFO:teuthology.orchestra.run.smithi096.stdout:(12/24): python3-rbd-17.2.7-1627.gbbdadfa3.el9. 1.1 MB/s | 294 kB 00:00 2024-09-13T14:18:33.753 INFO:teuthology.orchestra.run.smithi096.stdout:(13/24): python3-rgw-17.2.7-1627.gbbdadfa3.el9. 372 kB/s | 99 kB 00:00 2024-09-13T14:18:33.862 INFO:teuthology.orchestra.run.smithi096.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 418 kB/s | 45 kB 00:00 2024-09-13T14:18:33.912 INFO:teuthology.orchestra.run.smithi096.stdout:(15/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 738 kB/s | 160 kB 00:00 2024-09-13T14:18:33.963 INFO:teuthology.orchestra.run.smithi096.stdout:(16/24): python3-prettytable-0.7.2-27.el9.noarc 843 kB/s | 42 kB 00:00 2024-09-13T14:18:33.996 INFO:teuthology.orchestra.run.smithi096.stdout:(17/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.1 MB/s | 292 kB 00:00 2024-09-13T14:18:34.048 INFO:teuthology.orchestra.run.smithi096.stdout:(18/24): librdkafka-1.6.1-102.el9.x86_64.rpm 2.2 MB/s | 662 kB 00:00 2024-09-13T14:18:34.081 INFO:teuthology.orchestra.run.smithi096.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 634 kB/s | 49 kB 00:00 2024-09-13T14:18:34.106 INFO:teuthology.orchestra.run.smithi096.stdout:(20/24): gperftools-libs-2.9.1-3.el9.x86_64.rpm 2.1 MB/s | 308 kB 00:00 2024-09-13T14:18:34.131 INFO:teuthology.orchestra.run.smithi096.stdout:(21/24): libunwind-1.6.2-1.el9.x86_64.rpm 816 kB/s | 67 kB 00:00 2024-09-13T14:18:34.231 INFO:teuthology.orchestra.run.smithi096.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 11 MB/s | 1.6 MB 00:00 2024-09-13T14:18:34.281 INFO:teuthology.orchestra.run.smithi096.stdout:(23/24): librados2-17.2.7-1627.gbbdadfa3.el9.x8 18 MB/s | 3.1 MB 00:00 2024-09-13T14:18:34.357 INFO:teuthology.orchestra.run.smithi096.stdout:(24/24): librbd1-17.2.7-1627.gbbdadfa3.el9.x86_ 13 MB/s | 2.9 MB 00:00 2024-09-13T14:18:34.361 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:18:34.362 INFO:teuthology.orchestra.run.smithi096.stdout:Total 24 MB/s | 58 MB 00:02 2024-09-13T14:18:34.423 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:18:34.456 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:18:34.456 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:18:34.931 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:18:34.931 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:18:35.483 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 21/26 2024-09-13T14:18:35.531 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 21/26 2024-09-13T14:18:36.073 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 21/26 2024-09-13T14:18:36.109 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 22/26 2024-09-13T14:18:36.116 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:18:36.194 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 22/26 2024-09-13T14:18:36.194 INFO:teuthology.orchestra.run.smithi204.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-13T14:18:36.195 INFO:teuthology.orchestra.run.smithi204.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-13T14:18:36.195 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:18:36.236 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 23/26 2024-09-13T14:18:36.516 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-09-13T14:18:36.596 INFO:teuthology.orchestra.run.smithi096.stdout: Upgrading : librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/26 2024-09-13T14:18:36.691 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/26 2024-09-13T14:18:36.775 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/26 2024-09-13T14:18:36.831 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/26 2024-09-13T14:18:36.921 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/26 2024-09-13T14:18:36.973 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-09-13T14:18:37.011 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-09-13T14:18:37.451 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-09-13T14:18:37.476 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 8/26 2024-09-13T14:18:37.539 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 8/26 2024-09-13T14:18:37.567 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-09-13T14:18:37.610 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el9. 10/26 2024-09-13T14:18:37.647 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 11/26 2024-09-13T14:18:37.710 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 12/26 2024-09-13T14:18:37.737 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_6 13/26 2024-09-13T14:18:37.803 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_6 13/26 2024-09-13T14:18:37.842 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-09-13T14:18:37.879 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/26 2024-09-13T14:18:37.912 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-09-13T14:18:38.216 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-09-13T14:18:38.267 INFO:teuthology.orchestra.run.smithi096.stdout: Upgrading : librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 18/26 2024-09-13T14:18:38.357 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 18/26 2024-09-13T14:18:38.430 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 19/26 2024-09-13T14:18:38.461 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x8 20/26 2024-09-13T14:18:40.868 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 21/26 2024-09-13T14:18:40.908 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 21/26 2024-09-13T14:18:41.434 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 21/26 2024-09-13T14:18:41.602 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 22/26 2024-09-13T14:18:41.775 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 22/26 2024-09-13T14:18:41.776 INFO:teuthology.orchestra.run.smithi096.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-13T14:18:41.776 INFO:teuthology.orchestra.run.smithi096.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-13T14:18:41.776 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:18:41.820 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 23/26 2024-09-13T14:18:46.150 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 23/26 2024-09-13T14:18:46.150 INFO:teuthology.orchestra.run.smithi204.stdout:skipping the directory /sys 2024-09-13T14:18:46.150 INFO:teuthology.orchestra.run.smithi204.stdout:skipping the directory /proc 2024-09-13T14:18:46.150 INFO:teuthology.orchestra.run.smithi204.stdout:skipping the directory /mnt 2024-09-13T14:18:46.150 INFO:teuthology.orchestra.run.smithi204.stdout:skipping the directory /var/tmp 2024-09-13T14:18:46.150 INFO:teuthology.orchestra.run.smithi204.stdout:skipping the directory /home 2024-09-13T14:18:46.150 INFO:teuthology.orchestra.run.smithi204.stdout:skipping the directory /root 2024-09-13T14:18:46.151 INFO:teuthology.orchestra.run.smithi204.stdout:skipping the directory /tmp 2024-09-13T14:18:46.151 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:18:47.471 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 24/26 2024-09-13T14:18:47.549 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 24/26 2024-09-13T14:18:47.550 INFO:teuthology.orchestra.run.smithi204.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:18:47.550 INFO:teuthology.orchestra.run.smithi204.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-13T14:18:47.550 INFO:teuthology.orchestra.run.smithi204.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:18:47.550 INFO:teuthology.orchestra.run.smithi204.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:18:47.550 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:18:47.551 INFO:teuthology.orchestra.run.smithi204.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-09-13T14:18:47.649 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-09-13T14:18:47.649 INFO:teuthology.orchestra.run.smithi204.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-09-13T14:18:48.929 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-09-13T14:18:48.929 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/26 2024-09-13T14:18:48.929 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/26 2024-09-13T14:18:48.929 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/26 2024-09-13T14:18:48.929 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/26 2024-09-13T14:18:48.929 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 5/26 2024-09-13T14:18:48.929 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_6 6/26 2024-09-13T14:18:48.929 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 7/26 2024-09-13T14:18:48.929 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el9. 8/26 2024-09-13T14:18:48.930 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x8 9/26 2024-09-13T14:18:48.930 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 10/26 2024-09-13T14:18:48.930 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 11/26 2024-09-13T14:18:48.930 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 12/26 2024-09-13T14:18:48.930 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 13/26 2024-09-13T14:18:48.930 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-09-13T14:18:48.930 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-09-13T14:18:48.930 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-09-13T14:18:48.930 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-09-13T14:18:48.930 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-09-13T14:18:48.930 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 19/26 2024-09-13T14:18:48.930 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-09-13T14:18:48.930 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-09-13T14:18:48.931 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-09-13T14:18:48.931 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 23/26 2024-09-13T14:18:48.931 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-09-13T14:18:48.931 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 25/26 2024-09-13T14:18:49.700 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-09-13T14:18:49.700 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:18:49.700 INFO:teuthology.orchestra.run.smithi204.stdout:Upgraded: 2024-09-13T14:18:49.700 INFO:teuthology.orchestra.run.smithi204.stdout: librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:49.700 INFO:teuthology.orchestra.run.smithi204.stdout: librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:49.701 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:18:49.701 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:49.701 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:49.701 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:49.701 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:49.701 INFO:teuthology.orchestra.run.smithi204.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-13T14:18:49.701 INFO:teuthology.orchestra.run.smithi204.stdout: libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:49.701 INFO:teuthology.orchestra.run.smithi204.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-13T14:18:49.701 INFO:teuthology.orchestra.run.smithi204.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-13T14:18:49.701 INFO:teuthology.orchestra.run.smithi204.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-13T14:18:49.701 INFO:teuthology.orchestra.run.smithi204.stdout: libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:49.701 INFO:teuthology.orchestra.run.smithi204.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-13T14:18:49.701 INFO:teuthology.orchestra.run.smithi204.stdout: librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:49.702 INFO:teuthology.orchestra.run.smithi204.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-13T14:18:49.702 INFO:teuthology.orchestra.run.smithi204.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-13T14:18:49.702 INFO:teuthology.orchestra.run.smithi204.stdout: python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:49.702 INFO:teuthology.orchestra.run.smithi204.stdout: python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:49.702 INFO:teuthology.orchestra.run.smithi204.stdout: python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:49.702 INFO:teuthology.orchestra.run.smithi204.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-13T14:18:49.702 INFO:teuthology.orchestra.run.smithi204.stdout: python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:49.702 INFO:teuthology.orchestra.run.smithi204.stdout: python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:49.702 INFO:teuthology.orchestra.run.smithi204.stdout: python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:49.702 INFO:teuthology.orchestra.run.smithi204.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-13T14:18:49.702 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:18:49.702 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:18:50.066 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install ceph-test 2024-09-13T14:18:50.637 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:00:27 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:18:50.775 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:18:50.775 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:18:50.775 INFO:teuthology.orchestra.run.smithi204.stdout: Package Arch Version Repository Size 2024-09-13T14:18:50.775 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:18:50.776 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:18:50.776 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-test x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 48 M 2024-09-13T14:18:50.776 INFO:teuthology.orchestra.run.smithi204.stdout:Installing dependencies: 2024-09-13T14:18:50.776 INFO:teuthology.orchestra.run.smithi204.stdout: libcephsqlite x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 163 k 2024-09-13T14:18:50.776 INFO:teuthology.orchestra.run.smithi204.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-13T14:18:50.776 INFO:teuthology.orchestra.run.smithi204.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-13T14:18:50.776 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:18:50.776 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:18:50.776 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:18:50.776 INFO:teuthology.orchestra.run.smithi204.stdout:Install 4 Packages 2024-09-13T14:18:50.776 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:18:50.777 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 48 M 2024-09-13T14:18:50.777 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 193 M 2024-09-13T14:18:50.777 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:18:51.040 INFO:teuthology.orchestra.run.smithi204.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.4 MB/s | 304 kB 00:00 2024-09-13T14:18:51.065 INFO:teuthology.orchestra.run.smithi204.stdout:(2/4): libcephsqlite-17.2.7-1627.gbbdadfa3.el9. 686 kB/s | 163 kB 00:00 2024-09-13T14:18:51.082 INFO:teuthology.orchestra.run.smithi204.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.5 MB/s | 64 kB 00:00 2024-09-13T14:18:51.507 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 23/26 2024-09-13T14:18:51.507 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /sys 2024-09-13T14:18:51.507 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /proc 2024-09-13T14:18:51.508 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /mnt 2024-09-13T14:18:51.508 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /var/tmp 2024-09-13T14:18:51.508 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /home 2024-09-13T14:18:51.508 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /root 2024-09-13T14:18:51.508 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /tmp 2024-09-13T14:18:51.508 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:18:52.333 INFO:teuthology.orchestra.run.smithi204.stdout:(4/4): ceph-test-17.2.7-1627.gbbdadfa3.el9.x86_ 32 MB/s | 48 MB 00:01 2024-09-13T14:18:52.335 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:18:52.335 INFO:teuthology.orchestra.run.smithi204.stdout:Total 31 MB/s | 48 MB 00:01 2024-09-13T14:18:52.363 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:18:52.382 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:18:52.383 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:18:52.752 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:18:52.753 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:18:52.786 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 24/26 2024-09-13T14:18:52.862 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 24/26 2024-09-13T14:18:52.862 INFO:teuthology.orchestra.run.smithi096.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:18:52.862 INFO:teuthology.orchestra.run.smithi096.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-13T14:18:52.863 INFO:teuthology.orchestra.run.smithi096.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:18:52.863 INFO:teuthology.orchestra.run.smithi096.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:18:52.863 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:18:52.865 INFO:teuthology.orchestra.run.smithi096.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-09-13T14:18:52.947 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-09-13T14:18:52.948 INFO:teuthology.orchestra.run.smithi096.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-09-13T14:18:53.276 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:18:53.368 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-13T14:18:53.418 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-13T14:18:53.443 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/4 2024-09-13T14:18:54.192 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-09-13T14:18:54.192 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/26 2024-09-13T14:18:54.192 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/26 2024-09-13T14:18:54.193 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/26 2024-09-13T14:18:54.193 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/26 2024-09-13T14:18:54.193 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 5/26 2024-09-13T14:18:54.193 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_6 6/26 2024-09-13T14:18:54.193 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 7/26 2024-09-13T14:18:54.193 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el9. 8/26 2024-09-13T14:18:54.193 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x8 9/26 2024-09-13T14:18:54.193 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 10/26 2024-09-13T14:18:54.193 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 11/26 2024-09-13T14:18:54.193 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 12/26 2024-09-13T14:18:54.193 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 13/26 2024-09-13T14:18:54.194 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-09-13T14:18:54.194 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-09-13T14:18:54.194 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-09-13T14:18:54.194 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-09-13T14:18:54.194 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-09-13T14:18:54.194 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 19/26 2024-09-13T14:18:54.194 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-09-13T14:18:54.194 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-09-13T14:18:54.194 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-09-13T14:18:54.194 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 23/26 2024-09-13T14:18:54.194 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-09-13T14:18:54.194 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 25/26 2024-09-13T14:18:55.153 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-09-13T14:18:55.153 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:18:55.153 INFO:teuthology.orchestra.run.smithi096.stdout:Upgraded: 2024-09-13T14:18:55.153 INFO:teuthology.orchestra.run.smithi096.stdout: librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:55.153 INFO:teuthology.orchestra.run.smithi096.stdout: librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:55.153 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:18:55.153 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:55.153 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:55.153 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:55.153 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:55.153 INFO:teuthology.orchestra.run.smithi096.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-13T14:18:55.154 INFO:teuthology.orchestra.run.smithi096.stdout: libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:55.154 INFO:teuthology.orchestra.run.smithi096.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-13T14:18:55.154 INFO:teuthology.orchestra.run.smithi096.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-13T14:18:55.154 INFO:teuthology.orchestra.run.smithi096.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-13T14:18:55.154 INFO:teuthology.orchestra.run.smithi096.stdout: libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:55.154 INFO:teuthology.orchestra.run.smithi096.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-13T14:18:55.154 INFO:teuthology.orchestra.run.smithi096.stdout: librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:55.154 INFO:teuthology.orchestra.run.smithi096.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-13T14:18:55.154 INFO:teuthology.orchestra.run.smithi096.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-13T14:18:55.154 INFO:teuthology.orchestra.run.smithi096.stdout: python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:55.154 INFO:teuthology.orchestra.run.smithi096.stdout: python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:55.154 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:55.154 INFO:teuthology.orchestra.run.smithi096.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-13T14:18:55.154 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:55.155 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:55.155 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:18:55.155 INFO:teuthology.orchestra.run.smithi096.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-13T14:18:55.155 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:18:55.155 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:18:55.525 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-test 2024-09-13T14:18:56.093 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:27 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:18:56.225 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:18:56.226 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:18:56.226 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2024-09-13T14:18:56.226 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:18:56.226 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:18:56.226 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-test x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 48 M 2024-09-13T14:18:56.226 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2024-09-13T14:18:56.226 INFO:teuthology.orchestra.run.smithi096.stdout: libcephsqlite x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 163 k 2024-09-13T14:18:56.226 INFO:teuthology.orchestra.run.smithi096.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-13T14:18:56.226 INFO:teuthology.orchestra.run.smithi096.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-13T14:18:56.226 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:18:56.226 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:18:56.227 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:18:56.227 INFO:teuthology.orchestra.run.smithi096.stdout:Install 4 Packages 2024-09-13T14:18:56.227 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:18:56.227 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 48 M 2024-09-13T14:18:56.227 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 193 M 2024-09-13T14:18:56.227 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:18:56.476 INFO:teuthology.orchestra.run.smithi096.stdout:(1/4): libcephsqlite-17.2.7-1627.gbbdadfa3.el9. 720 kB/s | 163 kB 00:00 2024-09-13T14:18:56.509 INFO:teuthology.orchestra.run.smithi096.stdout:(2/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.1 MB/s | 304 kB 00:00 2024-09-13T14:18:56.551 INFO:teuthology.orchestra.run.smithi096.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 850 kB/s | 64 kB 00:00 2024-09-13T14:18:57.760 INFO:teuthology.orchestra.run.smithi096.stdout:(4/4): ceph-test-17.2.7-1627.gbbdadfa3.el9.x86_ 32 MB/s | 48 MB 00:01 2024-09-13T14:18:57.761 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:18:57.762 INFO:teuthology.orchestra.run.smithi096.stdout:Total 32 MB/s | 48 MB 00:01 2024-09-13T14:18:57.789 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:18:57.808 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:18:57.808 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:18:57.885 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/4 2024-09-13T14:18:57.976 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/4 2024-09-13T14:18:58.175 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:18:58.175 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:18:58.700 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:18:58.782 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-13T14:18:58.843 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-13T14:18:58.870 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/4 2024-09-13T14:19:00.312 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/4 2024-09-13T14:19:00.312 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/4 2024-09-13T14:19:00.312 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/4 2024-09-13T14:19:00.312 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-13T14:19:00.637 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-13T14:19:00.637 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:00.637 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:19:00.637 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:00.638 INFO:teuthology.orchestra.run.smithi204.stdout: libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:00.638 INFO:teuthology.orchestra.run.smithi204.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-13T14:19:00.638 INFO:teuthology.orchestra.run.smithi204.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-13T14:19:00.638 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:00.638 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:19:00.917 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install ceph 2024-09-13T14:19:01.493 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:00:38 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:19:01.636 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:19:01.637 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:19:01.637 INFO:teuthology.orchestra.run.smithi204.stdout: Package Arch Version Repository Size 2024-09-13T14:19:01.637 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:19:01.637 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:19:01.637 INFO:teuthology.orchestra.run.smithi204.stdout: ceph x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 6.5 k 2024-09-13T14:19:01.637 INFO:teuthology.orchestra.run.smithi204.stdout:Installing dependencies: 2024-09-13T14:19:01.638 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mds x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.1 M 2024-09-13T14:19:01.638 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mgr x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 1.4 M 2024-09-13T14:19:01.638 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 240 k 2024-09-13T14:19:01.638 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mon x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 3.9 M 2024-09-13T14:19:01.638 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-osd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 15 M 2024-09-13T14:19:01.638 INFO:teuthology.orchestra.run.smithi204.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-13T14:19:01.638 INFO:teuthology.orchestra.run.smithi204.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-13T14:19:01.638 INFO:teuthology.orchestra.run.smithi204.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-13T14:19:01.638 INFO:teuthology.orchestra.run.smithi204.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-13T14:19:01.638 INFO:teuthology.orchestra.run.smithi204.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-13T14:19:01.638 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-13T14:19:01.639 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-13T14:19:01.639 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jaraco-collections 2024-09-13T14:19:01.639 INFO:teuthology.orchestra.run.smithi204.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-09-13T14:19:01.639 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-13T14:19:01.639 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-13T14:19:01.639 INFO:teuthology.orchestra.run.smithi204.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-13T14:19:01.639 INFO:teuthology.orchestra.run.smithi204.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-13T14:19:01.639 INFO:teuthology.orchestra.run.smithi204.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-13T14:19:01.639 INFO:teuthology.orchestra.run.smithi204.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-13T14:19:01.639 INFO:teuthology.orchestra.run.smithi204.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-13T14:19:01.639 INFO:teuthology.orchestra.run.smithi204.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-13T14:19:01.639 INFO:teuthology.orchestra.run.smithi204.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-13T14:19:01.639 INFO:teuthology.orchestra.run.smithi204.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-13T14:19:01.639 INFO:teuthology.orchestra.run.smithi204.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-13T14:19:01.640 INFO:teuthology.orchestra.run.smithi204.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-13T14:19:01.640 INFO:teuthology.orchestra.run.smithi204.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-13T14:19:01.640 INFO:teuthology.orchestra.run.smithi204.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-13T14:19:01.640 INFO:teuthology.orchestra.run.smithi204.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-13T14:19:01.640 INFO:teuthology.orchestra.run.smithi204.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-13T14:19:01.640 INFO:teuthology.orchestra.run.smithi204.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-13T14:19:01.640 INFO:teuthology.orchestra.run.smithi204.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-13T14:19:01.640 INFO:teuthology.orchestra.run.smithi204.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-13T14:19:01.640 INFO:teuthology.orchestra.run.smithi204.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-13T14:19:01.640 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:01.640 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:19:01.640 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:19:01.640 INFO:teuthology.orchestra.run.smithi204.stdout:Install 34 Packages 2024-09-13T14:19:01.640 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:01.641 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 27 M 2024-09-13T14:19:01.641 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 96 M 2024-09-13T14:19:01.641 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:19:02.071 INFO:teuthology.orchestra.run.smithi204.stdout:(1/34): ceph-17.2.7-1627.gbbdadfa3.el9.x86_64.r 57 kB/s | 6.5 kB 00:00 2024-09-13T14:19:02.255 INFO:teuthology.orchestra.run.smithi204.stdout:(2/34): ceph-mds-17.2.7-1627.gbbdadfa3.el9.x86_ 6.9 MB/s | 2.1 MB 00:00 2024-09-13T14:19:02.297 INFO:teuthology.orchestra.run.smithi204.stdout:(3/34): ceph-mgr-17.2.7-1627.gbbdadfa3.el9.x86_ 4.1 MB/s | 1.4 MB 00:00 2024-09-13T14:19:02.423 INFO:teuthology.orchestra.run.smithi204.stdout:(4/34): ceph-mon-17.2.7-1627.gbbdadfa3.el9.x86_ 11 MB/s | 3.9 MB 00:00 2024-09-13T14:19:02.448 INFO:teuthology.orchestra.run.smithi204.stdout:(5/34): ceph-mgr-modules-core-17.2.7-1627.gbbda 1.6 MB/s | 240 kB 00:00 2024-09-13T14:19:02.548 INFO:teuthology.orchestra.run.smithi204.stdout:(6/34): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.0 MB/s | 253 kB 00:00 2024-09-13T14:19:02.590 INFO:teuthology.orchestra.run.smithi204.stdout:(7/34): python3-ply-3.11-14.el9.noarch.rpm 2.5 MB/s | 106 kB 00:00 2024-09-13T14:19:02.632 INFO:teuthology.orchestra.run.smithi204.stdout:(8/34): python3-cryptography-36.0.1-4.el9.x86_6 6.7 MB/s | 1.2 MB 00:00 2024-09-13T14:19:02.657 INFO:teuthology.orchestra.run.smithi204.stdout:(9/34): python3-pycparser-2.20-6.el9.noarch.rpm 2.0 MB/s | 135 kB 00:00 2024-09-13T14:19:02.682 INFO:teuthology.orchestra.run.smithi204.stdout:(10/34): python3-requests-2.25.1-8.el9.noarch.r 2.4 MB/s | 125 kB 00:00 2024-09-13T14:19:02.708 INFO:teuthology.orchestra.run.smithi204.stdout:(11/34): python3-urllib3-1.26.5-6.el9.noarch.rp 4.2 MB/s | 215 kB 00:00 2024-09-13T14:19:02.868 INFO:teuthology.orchestra.run.smithi204.stdout:(12/34): ceph-osd-17.2.7-1627.gbbdadfa3.el9.x86 25 MB/s | 15 MB 00:00 2024-09-13T14:19:02.942 INFO:teuthology.orchestra.run.smithi204.stdout:(13/34): python3-markupsafe-1.1.1-12.el9.x86_64 148 kB/s | 35 kB 00:00 2024-09-13T14:19:02.967 INFO:teuthology.orchestra.run.smithi204.stdout:(14/34): python3-mako-1.1.4-6.el9.noarch.rpm 606 kB/s | 172 kB 00:00 2024-09-13T14:19:03.003 INFO:teuthology.orchestra.run.smithi204.stdout:(15/34): python3-packaging-20.9-5.el9.noarch.rp 568 kB/s | 77 kB 00:00 2024-09-13T14:19:03.028 INFO:teuthology.orchestra.run.smithi204.stdout:(16/34): python3-pytz-2021.1-5.el9.noarch.rpm 594 kB/s | 51 kB 00:00 2024-09-13T14:19:03.053 INFO:teuthology.orchestra.run.smithi204.stdout:(17/34): python3-toml-0.10.2-6.el9.noarch.rpm 483 kB/s | 42 kB 00:00 2024-09-13T14:19:03.103 INFO:teuthology.orchestra.run.smithi204.stdout:(18/34): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 433 kB/s | 43 kB 00:00 2024-09-13T14:19:03.129 INFO:teuthology.orchestra.run.smithi204.stdout:(19/34): python3-jaraco-8.2.1-3.el9.noarch.rpm 459 kB/s | 11 kB 00:00 2024-09-13T14:19:03.154 INFO:teuthology.orchestra.run.smithi204.stdout:(20/34): python3-jaraco-classes-3.2.1-5.el9.noa 710 kB/s | 18 kB 00:00 2024-09-13T14:19:03.179 INFO:teuthology.orchestra.run.smithi204.stdout:(21/34): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.1 MB/s | 172 kB 00:00 2024-09-13T14:19:03.204 INFO:teuthology.orchestra.run.smithi204.stdout:(22/34): python3-jaraco-collections-3.0.0-8.el9 463 kB/s | 23 kB 00:00 2024-09-13T14:19:03.230 INFO:teuthology.orchestra.run.smithi204.stdout:(23/34): python3-jaraco-functools-3.5.0-2.el9.n 387 kB/s | 19 kB 00:00 2024-09-13T14:19:03.240 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/4 2024-09-13T14:19:03.255 INFO:teuthology.orchestra.run.smithi204.stdout:(24/34): python3-jaraco-text-3.2.0-6.el9.noarch 391 kB/s | 20 kB 00:00 2024-09-13T14:19:03.281 INFO:teuthology.orchestra.run.smithi204.stdout:(25/34): python3-logutils-0.3.5-21.el9.noarch.r 924 kB/s | 46 kB 00:00 2024-09-13T14:19:03.306 INFO:teuthology.orchestra.run.smithi204.stdout:(26/34): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-09-13T14:19:03.325 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/4 2024-09-13T14:19:03.339 INFO:teuthology.orchestra.run.smithi204.stdout:(27/34): python3-cherrypy-18.6.1-2.el9.noarch.r 1.3 MB/s | 358 kB 00:00 2024-09-13T14:19:03.365 INFO:teuthology.orchestra.run.smithi204.stdout:(28/34): python3-portend-3.1.0-2.el9.noarch.rpm 280 kB/s | 16 kB 00:00 2024-09-13T14:19:03.390 INFO:teuthology.orchestra.run.smithi204.stdout:(29/34): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.7 MB/s | 90 kB 00:00 2024-09-13T14:19:03.415 INFO:teuthology.orchestra.run.smithi204.stdout:(30/34): python3-tempora-5.0.0-2.el9.noarch.rpm 713 kB/s | 36 kB 00:00 2024-09-13T14:19:03.440 INFO:teuthology.orchestra.run.smithi204.stdout:(31/34): python3-pecan-1.4.2-3.el9.noarch.rpm 1.7 MB/s | 272 kB 00:00 2024-09-13T14:19:03.466 INFO:teuthology.orchestra.run.smithi204.stdout:(32/34): python3-webob-1.8.8-2.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-09-13T14:19:03.483 INFO:teuthology.orchestra.run.smithi204.stdout:(33/34): python3-zc-lockfile-2.0-10.el9.noarch. 477 kB/s | 20 kB 00:00 2024-09-13T14:19:03.516 INFO:teuthology.orchestra.run.smithi204.stdout:(34/34): python3-werkzeug-2.0.3-3.el9.1.noarch. 4.2 MB/s | 427 kB 00:00 2024-09-13T14:19:03.521 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:03.521 INFO:teuthology.orchestra.run.smithi204.stdout:Total 14 MB/s | 27 MB 00:01 2024-09-13T14:19:03.649 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:19:03.686 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:19:03.686 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:19:04.081 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:19:04.082 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:19:05.297 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:19:05.361 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/34 2024-09-13T14:19:05.393 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/34 2024-09-13T14:19:05.426 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/34 2024-09-13T14:19:05.518 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/34 2024-09-13T14:19:05.597 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/34 2024-09-13T14:19:05.646 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/34 2024-09-13T14:19:05.660 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/4 2024-09-13T14:19:05.660 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/4 2024-09-13T14:19:05.660 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/4 2024-09-13T14:19:05.660 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-13T14:19:05.681 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/34 2024-09-13T14:19:05.711 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/34 2024-09-13T14:19:05.745 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/34 2024-09-13T14:19:05.778 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/34 2024-09-13T14:19:05.819 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/34 2024-09-13T14:19:05.854 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/34 2024-09-13T14:19:05.888 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/34 2024-09-13T14:19:05.937 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/34 2024-09-13T14:19:05.977 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-packaging-20.9-5.el9.noarch 15/34 2024-09-13T14:19:05.988 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-13T14:19:05.988 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:05.988 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:19:05.988 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:05.989 INFO:teuthology.orchestra.run.smithi096.stdout: libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:05.989 INFO:teuthology.orchestra.run.smithi096.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-13T14:19:05.989 INFO:teuthology.orchestra.run.smithi096.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-13T14:19:05.989 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:05.989 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:19:06.059 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 16/34 2024-09-13T14:19:06.239 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 17/34 2024-09-13T14:19:06.276 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph 2024-09-13T14:19:06.319 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 18/34 2024-09-13T14:19:06.372 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 19/34 2024-09-13T14:19:06.411 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 20/34 2024-09-13T14:19:06.468 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-ply-3.11-14.el9.noarch 21/34 2024-09-13T14:19:06.531 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 22/34 2024-09-13T14:19:06.718 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 23/34 2024-09-13T14:19:06.765 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 24/34 2024-09-13T14:19:06.843 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 25/34 2024-09-13T14:19:06.844 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:37 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:19:06.939 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 26/34 2024-09-13T14:19:06.977 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:19:06.979 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:06.979 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2024-09-13T14:19:06.980 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:06.980 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:19:06.980 INFO:teuthology.orchestra.run.smithi096.stdout: ceph x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 6.5 k 2024-09-13T14:19:06.980 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2024-09-13T14:19:06.980 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mds x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.1 M 2024-09-13T14:19:06.980 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 1.4 M 2024-09-13T14:19:06.980 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 240 k 2024-09-13T14:19:06.980 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mon x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 3.9 M 2024-09-13T14:19:06.980 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-osd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 15 M 2024-09-13T14:19:06.980 INFO:teuthology.orchestra.run.smithi096.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-13T14:19:06.980 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-13T14:19:06.980 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-13T14:19:06.981 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-13T14:19:06.981 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-13T14:19:06.981 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-13T14:19:06.981 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-13T14:19:06.981 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jaraco-collections 2024-09-13T14:19:06.981 INFO:teuthology.orchestra.run.smithi096.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-09-13T14:19:06.981 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-13T14:19:06.981 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-13T14:19:06.981 INFO:teuthology.orchestra.run.smithi096.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-13T14:19:06.981 INFO:teuthology.orchestra.run.smithi096.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-13T14:19:06.981 INFO:teuthology.orchestra.run.smithi096.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-13T14:19:06.981 INFO:teuthology.orchestra.run.smithi096.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-13T14:19:06.981 INFO:teuthology.orchestra.run.smithi096.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-13T14:19:06.982 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-13T14:19:06.982 INFO:teuthology.orchestra.run.smithi096.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-13T14:19:06.982 INFO:teuthology.orchestra.run.smithi096.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-13T14:19:06.982 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-13T14:19:06.982 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-13T14:19:06.982 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-13T14:19:06.982 INFO:teuthology.orchestra.run.smithi096.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-13T14:19:06.982 INFO:teuthology.orchestra.run.smithi096.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-13T14:19:06.982 INFO:teuthology.orchestra.run.smithi096.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-13T14:19:06.982 INFO:teuthology.orchestra.run.smithi096.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-13T14:19:06.982 INFO:teuthology.orchestra.run.smithi096.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-13T14:19:06.982 INFO:teuthology.orchestra.run.smithi096.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-13T14:19:06.982 INFO:teuthology.orchestra.run.smithi096.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-13T14:19:06.983 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:06.983 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:19:06.983 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:06.983 INFO:teuthology.orchestra.run.smithi096.stdout:Install 34 Packages 2024-09-13T14:19:06.983 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:06.983 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 27/34 2024-09-13T14:19:06.984 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 27 M 2024-09-13T14:19:06.984 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 96 M 2024-09-13T14:19:06.984 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:19:07.114 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 28/34 2024-09-13T14:19:07.260 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el9. 29/34 2024-09-13T14:19:07.291 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 30/34 2024-09-13T14:19:07.317 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 30/34 2024-09-13T14:19:07.317 INFO:teuthology.orchestra.run.smithi204.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:19:07.317 INFO:teuthology.orchestra.run.smithi204.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-13T14:19:07.317 INFO:teuthology.orchestra.run.smithi204.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:19:07.317 INFO:teuthology.orchestra.run.smithi204.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:19:07.317 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:07.542 INFO:teuthology.orchestra.run.smithi096.stdout:(1/34): ceph-17.2.7-1627.gbbdadfa3.el9.x86_64.r 47 kB/s | 6.5 kB 00:00 2024-09-13T14:19:07.717 INFO:teuthology.orchestra.run.smithi096.stdout:(2/34): ceph-mgr-17.2.7-1627.gbbdadfa3.el9.x86_ 4.5 MB/s | 1.4 MB 00:00 2024-09-13T14:19:07.768 INFO:teuthology.orchestra.run.smithi096.stdout:(3/34): ceph-mds-17.2.7-1627.gbbdadfa3.el9.x86_ 5.7 MB/s | 2.1 MB 00:00 2024-09-13T14:19:07.834 INFO:teuthology.orchestra.run.smithi096.stdout:(4/34): ceph-mgr-modules-core-17.2.7-1627.gbbda 3.5 MB/s | 240 kB 00:00 2024-09-13T14:19:08.043 INFO:teuthology.orchestra.run.smithi096.stdout:(5/34): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.2 MB/s | 253 kB 00:00 2024-09-13T14:19:08.227 INFO:teuthology.orchestra.run.smithi096.stdout:(6/34): ceph-osd-17.2.7-1627.gbbdadfa3.el9.x86_ 30 MB/s | 15 MB 00:00 2024-09-13T14:19:08.285 INFO:teuthology.orchestra.run.smithi096.stdout:(7/34): ceph-mon-17.2.7-1627.gbbdadfa3.el9.x86_ 5.3 MB/s | 3.9 MB 00:00 2024-09-13T14:19:08.394 INFO:teuthology.orchestra.run.smithi096.stdout:(8/34): python3-cryptography-36.0.1-4.el9.x86_6 3.5 MB/s | 1.2 MB 00:00 2024-09-13T14:19:08.428 INFO:teuthology.orchestra.run.smithi096.stdout:(9/34): python3-ply-3.11-14.el9.noarch.rpm 532 kB/s | 106 kB 00:00 2024-09-13T14:19:08.453 INFO:teuthology.orchestra.run.smithi096.stdout:(10/34): python3-requests-2.25.1-8.el9.noarch.r 2.1 MB/s | 125 kB 00:00 2024-09-13T14:19:08.486 INFO:teuthology.orchestra.run.smithi096.stdout:(11/34): python3-pycparser-2.20-6.el9.noarch.rp 673 kB/s | 135 kB 00:00 2024-09-13T14:19:08.520 INFO:teuthology.orchestra.run.smithi096.stdout:(12/34): python3-mako-1.1.4-6.el9.noarch.rpm 2.9 MB/s | 172 kB 00:00 2024-09-13T14:19:08.537 INFO:teuthology.orchestra.run.smithi096.stdout:(13/34): python3-urllib3-1.26.5-6.el9.noarch.rp 1.9 MB/s | 215 kB 00:00 2024-09-13T14:19:08.554 INFO:teuthology.orchestra.run.smithi096.stdout:(14/34): python3-markupsafe-1.1.1-12.el9.x86_64 514 kB/s | 35 kB 00:00 2024-09-13T14:19:08.579 INFO:teuthology.orchestra.run.smithi096.stdout:(15/34): python3-packaging-20.9-5.el9.noarch.rp 1.1 MB/s | 77 kB 00:00 2024-09-13T14:19:08.605 INFO:teuthology.orchestra.run.smithi096.stdout:(16/34): python3-pytz-2021.1-5.el9.noarch.rpm 761 kB/s | 51 kB 00:00 2024-09-13T14:19:08.630 INFO:teuthology.orchestra.run.smithi096.stdout:(17/34): python3-toml-0.10.2-6.el9.noarch.rpm 551 kB/s | 42 kB 00:00 2024-09-13T14:19:08.676 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 31/34 2024-09-13T14:19:08.681 INFO:teuthology.orchestra.run.smithi096.stdout:(18/34): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 430 kB/s | 43 kB 00:00 2024-09-13T14:19:08.705 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 31/34 2024-09-13T14:19:08.705 INFO:teuthology.orchestra.run.smithi204.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:19:08.705 INFO:teuthology.orchestra.run.smithi204.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-13T14:19:08.705 INFO:teuthology.orchestra.run.smithi204.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:19:08.706 INFO:teuthology.orchestra.run.smithi204.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:19:08.706 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:08.706 INFO:teuthology.orchestra.run.smithi096.stdout:(19/34): python3-jaraco-8.2.1-3.el9.noarch.rpm 463 kB/s | 11 kB 00:00 2024-09-13T14:19:08.739 INFO:teuthology.orchestra.run.smithi096.stdout:(20/34): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.2 MB/s | 172 kB 00:00 2024-09-13T14:19:08.765 INFO:teuthology.orchestra.run.smithi096.stdout:(21/34): python3-jaraco-classes-3.2.1-5.el9.noa 301 kB/s | 18 kB 00:00 2024-09-13T14:19:08.792 INFO:teuthology.orchestra.run.smithi096.stdout:(22/34): python3-jaraco-collections-3.0.0-8.el9 461 kB/s | 23 kB 00:00 2024-09-13T14:19:08.815 INFO:teuthology.orchestra.run.smithi096.stdout:(23/34): python3-jaraco-functools-3.5.0-2.el9.n 386 kB/s | 19 kB 00:00 2024-09-13T14:19:08.841 INFO:teuthology.orchestra.run.smithi096.stdout:(24/34): python3-jaraco-text-3.2.0-6.el9.noarch 392 kB/s | 20 kB 00:00 2024-09-13T14:19:08.866 INFO:teuthology.orchestra.run.smithi096.stdout:(25/34): python3-logutils-0.3.5-21.el9.noarch.r 921 kB/s | 46 kB 00:00 2024-09-13T14:19:08.891 INFO:teuthology.orchestra.run.smithi096.stdout:(26/34): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-09-13T14:19:08.925 INFO:teuthology.orchestra.run.smithi096.stdout:(27/34): python3-cherrypy-18.6.1-2.el9.noarch.r 1.2 MB/s | 358 kB 00:00 2024-09-13T14:19:08.950 INFO:teuthology.orchestra.run.smithi096.stdout:(28/34): python3-portend-3.1.0-2.el9.noarch.rpm 280 kB/s | 16 kB 00:00 2024-09-13T14:19:08.975 INFO:teuthology.orchestra.run.smithi096.stdout:(29/34): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.7 MB/s | 90 kB 00:00 2024-09-13T14:19:09.001 INFO:teuthology.orchestra.run.smithi096.stdout:(30/34): python3-tempora-5.0.0-2.el9.noarch.rpm 709 kB/s | 36 kB 00:00 2024-09-13T14:19:09.026 INFO:teuthology.orchestra.run.smithi096.stdout:(31/34): python3-pecan-1.4.2-3.el9.noarch.rpm 1.7 MB/s | 272 kB 00:00 2024-09-13T14:19:09.051 INFO:teuthology.orchestra.run.smithi096.stdout:(32/34): python3-webob-1.8.8-2.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-09-13T14:19:09.076 INFO:teuthology.orchestra.run.smithi096.stdout:(33/34): python3-zc-lockfile-2.0-10.el9.noarch. 398 kB/s | 20 kB 00:00 2024-09-13T14:19:09.077 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 32/34 2024-09-13T14:19:09.101 INFO:teuthology.orchestra.run.smithi096.stdout:(34/34): python3-werkzeug-2.0.3-3.el9.1.noarch. 4.2 MB/s | 427 kB 00:00 2024-09-13T14:19:09.102 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 32/34 2024-09-13T14:19:09.102 INFO:teuthology.orchestra.run.smithi204.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:19:09.102 INFO:teuthology.orchestra.run.smithi204.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-13T14:19:09.102 INFO:teuthology.orchestra.run.smithi204.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:19:09.102 INFO:teuthology.orchestra.run.smithi204.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:19:09.102 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:09.105 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:09.106 INFO:teuthology.orchestra.run.smithi096.stdout:Total 13 MB/s | 27 MB 00:02 2024-09-13T14:19:09.232 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:19:09.269 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:19:09.270 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:19:09.311 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 33/34 2024-09-13T14:19:09.335 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 33/34 2024-09-13T14:19:09.335 INFO:teuthology.orchestra.run.smithi204.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:19:09.335 INFO:teuthology.orchestra.run.smithi204.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-13T14:19:09.335 INFO:teuthology.orchestra.run.smithi204.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:19:09.335 INFO:teuthology.orchestra.run.smithi204.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:19:09.335 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:09.399 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 34/34 2024-09-13T14:19:09.651 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:19:09.651 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:19:10.923 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:19:10.996 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/34 2024-09-13T14:19:11.028 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/34 2024-09-13T14:19:11.061 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/34 2024-09-13T14:19:11.139 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 34/34 2024-09-13T14:19:11.139 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/34 2024-09-13T14:19:11.139 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/34 2024-09-13T14:19:11.139 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/34 2024-09-13T14:19:11.139 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/34 2024-09-13T14:19:11.139 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 5/34 2024-09-13T14:19:11.139 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el9. 6/34 2024-09-13T14:19:11.139 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/34 2024-09-13T14:19:11.139 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/34 2024-09-13T14:19:11.139 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/34 2024-09-13T14:19:11.139 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/34 2024-09-13T14:19:11.139 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/34 2024-09-13T14:19:11.140 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/34 2024-09-13T14:19:11.140 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/34 2024-09-13T14:19:11.140 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/34 2024-09-13T14:19:11.140 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 15/34 2024-09-13T14:19:11.140 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 16/34 2024-09-13T14:19:11.140 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 17/34 2024-09-13T14:19:11.140 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 18/34 2024-09-13T14:19:11.140 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 19/34 2024-09-13T14:19:11.140 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 20/34 2024-09-13T14:19:11.140 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 21/34 2024-09-13T14:19:11.140 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 22/34 2024-09-13T14:19:11.140 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 23/34 2024-09-13T14:19:11.140 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 24/34 2024-09-13T14:19:11.140 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 25/34 2024-09-13T14:19:11.141 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 26/34 2024-09-13T14:19:11.141 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 27/34 2024-09-13T14:19:11.142 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 28/34 2024-09-13T14:19:11.142 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 29/34 2024-09-13T14:19:11.142 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 30/34 2024-09-13T14:19:11.142 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 31/34 2024-09-13T14:19:11.142 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 32/34 2024-09-13T14:19:11.142 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 33/34 2024-09-13T14:19:11.154 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/34 2024-09-13T14:19:11.241 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/34 2024-09-13T14:19:11.289 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/34 2024-09-13T14:19:11.333 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/34 2024-09-13T14:19:11.362 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/34 2024-09-13T14:19:11.397 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/34 2024-09-13T14:19:11.431 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/34 2024-09-13T14:19:11.471 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/34 2024-09-13T14:19:11.514 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/34 2024-09-13T14:19:11.549 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/34 2024-09-13T14:19:11.606 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/34 2024-09-13T14:19:11.638 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-packaging-20.9-5.el9.noarch 15/34 2024-09-13T14:19:11.720 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 16/34 2024-09-13T14:19:11.882 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 17/34 2024-09-13T14:19:11.971 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 18/34 2024-09-13T14:19:12.032 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 19/34 2024-09-13T14:19:12.038 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 34/34 2024-09-13T14:19:12.038 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:12.038 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:19:12.038 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:12.038 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:12.038 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:12.038 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:19:12.038 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:12.039 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:12.039 INFO:teuthology.orchestra.run.smithi204.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-13T14:19:12.039 INFO:teuthology.orchestra.run.smithi204.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-13T14:19:12.039 INFO:teuthology.orchestra.run.smithi204.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-13T14:19:12.039 INFO:teuthology.orchestra.run.smithi204.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-13T14:19:12.039 INFO:teuthology.orchestra.run.smithi204.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-13T14:19:12.039 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-13T14:19:12.039 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-13T14:19:12.039 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-13T14:19:12.039 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-13T14:19:12.039 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-13T14:19:12.039 INFO:teuthology.orchestra.run.smithi204.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-13T14:19:12.039 INFO:teuthology.orchestra.run.smithi204.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-13T14:19:12.039 INFO:teuthology.orchestra.run.smithi204.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-13T14:19:12.039 INFO:teuthology.orchestra.run.smithi204.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-13T14:19:12.040 INFO:teuthology.orchestra.run.smithi204.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-13T14:19:12.040 INFO:teuthology.orchestra.run.smithi204.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-13T14:19:12.040 INFO:teuthology.orchestra.run.smithi204.stdout: python3-ply-3.11-14.el9.noarch 2024-09-13T14:19:12.040 INFO:teuthology.orchestra.run.smithi204.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-13T14:19:12.040 INFO:teuthology.orchestra.run.smithi204.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-13T14:19:12.040 INFO:teuthology.orchestra.run.smithi204.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-13T14:19:12.040 INFO:teuthology.orchestra.run.smithi204.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-13T14:19:12.040 INFO:teuthology.orchestra.run.smithi204.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-13T14:19:12.040 INFO:teuthology.orchestra.run.smithi204.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-13T14:19:12.040 INFO:teuthology.orchestra.run.smithi204.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-13T14:19:12.040 INFO:teuthology.orchestra.run.smithi204.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-13T14:19:12.040 INFO:teuthology.orchestra.run.smithi204.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-13T14:19:12.040 INFO:teuthology.orchestra.run.smithi204.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-13T14:19:12.040 INFO:teuthology.orchestra.run.smithi204.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-13T14:19:12.041 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:12.041 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:19:12.071 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 20/34 2024-09-13T14:19:12.128 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-ply-3.11-14.el9.noarch 21/34 2024-09-13T14:19:12.184 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 22/34 2024-09-13T14:19:12.380 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 23/34 2024-09-13T14:19:12.409 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install ceph-base 2024-09-13T14:19:12.425 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 24/34 2024-09-13T14:19:12.496 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 25/34 2024-09-13T14:19:12.584 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 26/34 2024-09-13T14:19:12.624 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 27/34 2024-09-13T14:19:12.748 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 28/34 2024-09-13T14:19:12.897 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el9. 29/34 2024-09-13T14:19:12.927 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 30/34 2024-09-13T14:19:12.953 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 30/34 2024-09-13T14:19:12.953 INFO:teuthology.orchestra.run.smithi096.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:19:12.953 INFO:teuthology.orchestra.run.smithi096.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-13T14:19:12.953 INFO:teuthology.orchestra.run.smithi096.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:19:12.953 INFO:teuthology.orchestra.run.smithi096.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:19:12.953 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:12.983 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:00:49 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:19:13.078 INFO:teuthology.orchestra.run.smithi204.stdout:Package ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:19:13.120 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:19:13.121 INFO:teuthology.orchestra.run.smithi204.stdout:Nothing to do. 2024-09-13T14:19:13.121 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:19:13.189 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install cephadm 2024-09-13T14:19:13.754 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:00:50 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:19:13.887 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:19:13.888 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:19:13.888 INFO:teuthology.orchestra.run.smithi204.stdout: Package Arch Version Repository Size 2024-09-13T14:19:13.888 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:19:13.888 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:19:13.888 INFO:teuthology.orchestra.run.smithi204.stdout: cephadm noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 83 k 2024-09-13T14:19:13.888 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:13.888 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:19:13.889 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:19:13.889 INFO:teuthology.orchestra.run.smithi204.stdout:Install 1 Package 2024-09-13T14:19:13.889 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:13.889 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 83 k 2024-09-13T14:19:13.889 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 375 k 2024-09-13T14:19:13.889 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:19:14.095 INFO:teuthology.orchestra.run.smithi204.stdout:cephadm-17.2.7-1627.gbbdadfa3.el9.noarch.rpm 402 kB/s | 83 kB 00:00 2024-09-13T14:19:14.095 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:14.096 INFO:teuthology.orchestra.run.smithi204.stdout:Total 399 kB/s | 83 kB 00:00 2024-09-13T14:19:14.096 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:19:14.101 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:19:14.101 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:19:14.103 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:19:14.103 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:19:14.168 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:19:14.328 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 31/34 2024-09-13T14:19:14.357 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 31/34 2024-09-13T14:19:14.358 INFO:teuthology.orchestra.run.smithi096.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:19:14.358 INFO:teuthology.orchestra.run.smithi096.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-13T14:19:14.358 INFO:teuthology.orchestra.run.smithi096.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:19:14.358 INFO:teuthology.orchestra.run.smithi096.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:19:14.358 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:14.619 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:19:14.724 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:19:14.738 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 32/34 2024-09-13T14:19:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 32/34 2024-09-13T14:19:14.763 INFO:teuthology.orchestra.run.smithi096.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:19:14.763 INFO:teuthology.orchestra.run.smithi096.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-13T14:19:14.763 INFO:teuthology.orchestra.run.smithi096.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:19:14.763 INFO:teuthology.orchestra.run.smithi096.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:19:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:14.989 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 33/34 2024-09-13T14:19:15.013 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 33/34 2024-09-13T14:19:15.013 INFO:teuthology.orchestra.run.smithi096.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:19:15.013 INFO:teuthology.orchestra.run.smithi096.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-13T14:19:15.013 INFO:teuthology.orchestra.run.smithi096.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:19:15.013 INFO:teuthology.orchestra.run.smithi096.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:19:15.013 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:15.074 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:19:15.078 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 34/34 2024-09-13T14:19:15.308 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:19:15.308 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:15.308 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:19:15.308 INFO:teuthology.orchestra.run.smithi204.stdout: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:19:15.308 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:15.309 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:19:15.524 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install ceph-immutable-object-cache 2024-09-13T14:19:16.102 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:00:53 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:19:16.237 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:19:16.237 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:19:16.238 INFO:teuthology.orchestra.run.smithi204.stdout: Package Arch Version Repo Size 2024-09-13T14:19:16.238 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:19:16.238 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:19:16.238 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 137 k 2024-09-13T14:19:16.238 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:16.238 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:19:16.238 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:19:16.238 INFO:teuthology.orchestra.run.smithi204.stdout:Install 1 Package 2024-09-13T14:19:16.238 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:16.238 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 137 k 2024-09-13T14:19:16.238 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 379 k 2024-09-13T14:19:16.238 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:19:16.444 INFO:teuthology.orchestra.run.smithi204.stdout:ceph-immutable-object-cache-17.2.7-1627.gbbdadf 670 kB/s | 137 kB 00:00 2024-09-13T14:19:16.444 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:16.445 INFO:teuthology.orchestra.run.smithi204.stdout:Total 664 kB/s | 137 kB 00:00 2024-09-13T14:19:16.445 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:19:16.453 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:19:16.453 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:19:16.510 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:19:16.510 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:19:16.656 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:19:16.737 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3. 1/1 2024-09-13T14:19:16.757 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 34/34 2024-09-13T14:19:16.757 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/34 2024-09-13T14:19:16.757 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/34 2024-09-13T14:19:16.757 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/34 2024-09-13T14:19:16.757 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/34 2024-09-13T14:19:16.757 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 5/34 2024-09-13T14:19:16.757 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el9. 6/34 2024-09-13T14:19:16.757 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/34 2024-09-13T14:19:16.757 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/34 2024-09-13T14:19:16.757 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/34 2024-09-13T14:19:16.757 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/34 2024-09-13T14:19:16.758 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/34 2024-09-13T14:19:16.758 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/34 2024-09-13T14:19:16.758 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/34 2024-09-13T14:19:16.758 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/34 2024-09-13T14:19:16.758 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 15/34 2024-09-13T14:19:16.758 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 16/34 2024-09-13T14:19:16.758 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 17/34 2024-09-13T14:19:16.758 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 18/34 2024-09-13T14:19:16.758 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 19/34 2024-09-13T14:19:16.758 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 20/34 2024-09-13T14:19:16.758 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 21/34 2024-09-13T14:19:16.758 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 22/34 2024-09-13T14:19:16.758 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 23/34 2024-09-13T14:19:16.758 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 24/34 2024-09-13T14:19:16.759 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 25/34 2024-09-13T14:19:16.759 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 26/34 2024-09-13T14:19:16.760 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 27/34 2024-09-13T14:19:16.760 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 28/34 2024-09-13T14:19:16.760 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 29/34 2024-09-13T14:19:16.760 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 30/34 2024-09-13T14:19:16.760 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 31/34 2024-09-13T14:19:16.760 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 32/34 2024-09-13T14:19:16.760 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 33/34 2024-09-13T14:19:16.762 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3. 1/1 2024-09-13T14:19:16.762 INFO:teuthology.orchestra.run.smithi204.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:19:16.762 INFO:teuthology.orchestra.run.smithi204.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-13T14:19:16.762 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:17.683 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3. 1/1 2024-09-13T14:19:17.684 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:17.684 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:19:17.684 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:17.684 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:17.684 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:19:17.882 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 34/34 2024-09-13T14:19:17.882 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:17.882 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:19:17.882 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:17.882 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:17.883 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:17.883 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:19:17.883 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:17.883 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:17.883 INFO:teuthology.orchestra.run.smithi096.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-13T14:19:17.883 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-13T14:19:17.883 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-13T14:19:17.883 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-13T14:19:17.883 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-13T14:19:17.883 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-13T14:19:17.884 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-13T14:19:17.884 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-13T14:19:17.884 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-13T14:19:17.884 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-13T14:19:17.884 INFO:teuthology.orchestra.run.smithi096.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-13T14:19:17.884 INFO:teuthology.orchestra.run.smithi096.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-13T14:19:17.884 INFO:teuthology.orchestra.run.smithi096.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-13T14:19:17.884 INFO:teuthology.orchestra.run.smithi096.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-13T14:19:17.884 INFO:teuthology.orchestra.run.smithi096.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-13T14:19:17.884 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-13T14:19:17.884 INFO:teuthology.orchestra.run.smithi096.stdout: python3-ply-3.11-14.el9.noarch 2024-09-13T14:19:17.884 INFO:teuthology.orchestra.run.smithi096.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-13T14:19:17.884 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-13T14:19:17.885 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-13T14:19:17.885 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-13T14:19:17.885 INFO:teuthology.orchestra.run.smithi096.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-13T14:19:17.885 INFO:teuthology.orchestra.run.smithi096.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-13T14:19:17.885 INFO:teuthology.orchestra.run.smithi096.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-13T14:19:17.885 INFO:teuthology.orchestra.run.smithi096.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-13T14:19:17.885 INFO:teuthology.orchestra.run.smithi096.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-13T14:19:17.885 INFO:teuthology.orchestra.run.smithi096.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-13T14:19:17.885 INFO:teuthology.orchestra.run.smithi096.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-13T14:19:17.885 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:17.885 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:19:17.914 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install ceph-mgr 2024-09-13T14:19:18.235 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-base 2024-09-13T14:19:18.498 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:00:55 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:19:18.589 INFO:teuthology.orchestra.run.smithi204.stdout:Package ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:19:18.631 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:19:18.632 INFO:teuthology.orchestra.run.smithi204.stdout:Nothing to do. 2024-09-13T14:19:18.632 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:19:18.705 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install ceph-mgr-dashboard 2024-09-13T14:19:18.798 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:49 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:19:18.889 INFO:teuthology.orchestra.run.smithi096.stdout:Package ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:19:18.930 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:19:18.931 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2024-09-13T14:19:18.931 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:19:18.992 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install cephadm 2024-09-13T14:19:19.276 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:00:56 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:19:19.408 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:19:19.409 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:19:19.409 INFO:teuthology.orchestra.run.smithi204.stdout: Package Arch Version Repository Size 2024-09-13T14:19:19.409 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:19:19.409 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:19:19.409 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 3.4 M 2024-09-13T14:19:19.409 INFO:teuthology.orchestra.run.smithi204.stdout:Installing dependencies: 2024-09-13T14:19:19.409 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 24 k 2024-09-13T14:19:19.410 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 15 k 2024-09-13T14:19:19.410 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-13T14:19:19.410 INFO:teuthology.orchestra.run.smithi204.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-13T14:19:19.410 INFO:teuthology.orchestra.run.smithi204.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-13T14:19:19.410 INFO:teuthology.orchestra.run.smithi204.stdout:Installing weak dependencies: 2024-09-13T14:19:19.410 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-13T14:19:19.410 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:19.410 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:19:19.410 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:19:19.410 INFO:teuthology.orchestra.run.smithi204.stdout:Install 7 Packages 2024-09-13T14:19:19.411 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:19.411 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 3.7 M 2024-09-13T14:19:19.411 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 80 M 2024-09-13T14:19:19.411 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:19:19.551 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:50 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:19:19.682 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:19:19.683 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:19.683 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2024-09-13T14:19:19.683 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:19.683 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:19:19.684 INFO:teuthology.orchestra.run.smithi096.stdout: cephadm noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 83 k 2024-09-13T14:19:19.684 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:19.684 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:19:19.684 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:19.684 INFO:teuthology.orchestra.run.smithi096.stdout:Install 1 Package 2024-09-13T14:19:19.684 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:19.684 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 83 k 2024-09-13T14:19:19.684 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 375 k 2024-09-13T14:19:19.684 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:19:19.880 INFO:teuthology.orchestra.run.smithi096.stdout:cephadm-17.2.7-1627.gbbdadfa3.el9.noarch.rpm 422 kB/s | 83 kB 00:00 2024-09-13T14:19:19.881 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:19.881 INFO:teuthology.orchestra.run.smithi096.stdout:Total 419 kB/s | 83 kB 00:00 2024-09-13T14:19:19.881 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:19:19.886 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:19:19.886 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:19:19.888 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:19:19.888 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:19:19.898 INFO:teuthology.orchestra.run.smithi204.stdout:(1/7): ceph-grafana-dashboards-17.2.7-1627.gbbd 157 kB/s | 24 kB 00:00 2024-09-13T14:19:19.924 INFO:teuthology.orchestra.run.smithi204.stdout:(2/7): ceph-prometheus-alerts-17.2.7-1627.gbbda 81 kB/s | 15 kB 00:00 2024-09-13T14:19:19.966 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:19:19.974 INFO:teuthology.orchestra.run.smithi204.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 120 kB/s | 9.0 kB 00:00 2024-09-13T14:19:20.007 INFO:teuthology.orchestra.run.smithi204.stdout:(4/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 976 kB/s | 31 kB 00:00 2024-09-13T14:19:20.033 INFO:teuthology.orchestra.run.smithi204.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 375 kB/s | 41 kB 00:00 2024-09-13T14:19:20.066 INFO:teuthology.orchestra.run.smithi204.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 3.2 MB/s | 188 kB 00:00 2024-09-13T14:19:20.267 INFO:teuthology.orchestra.run.smithi204.stdout:(7/7): ceph-mgr-dashboard-17.2.7-1627.gbbdadfa3 6.4 MB/s | 3.4 MB 00:00 2024-09-13T14:19:20.270 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:20.270 INFO:teuthology.orchestra.run.smithi204.stdout:Total 4.2 MB/s | 3.7 MB 00:00 2024-09-13T14:19:20.270 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:19:20.279 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:19:20.279 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:19:20.367 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:19:20.367 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:19:20.421 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:19:20.516 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:19:20.673 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:19:20.743 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-09-13T14:19:20.777 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-09-13T14:19:20.823 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-09-13T14:19:20.864 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-09-13T14:19:20.882 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:19:20.893 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.el9.n 5/7 2024-09-13T14:19:21.102 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:19:21.102 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:21.102 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:19:21.102 INFO:teuthology.orchestra.run.smithi096.stdout: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:19:21.102 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:21.103 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:19:21.323 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-immutable-object-cache 2024-09-13T14:19:21.884 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3.el9. 6/7 2024-09-13T14:19:21.895 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:52 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:19:21.930 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarc 7/7 2024-09-13T14:19:22.029 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:19:22.029 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:22.030 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repo Size 2024-09-13T14:19:22.030 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:22.030 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:19:22.030 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 137 k 2024-09-13T14:19:22.030 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:22.030 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:19:22.030 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:22.030 INFO:teuthology.orchestra.run.smithi096.stdout:Install 1 Package 2024-09-13T14:19:22.030 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:22.031 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 137 k 2024-09-13T14:19:22.031 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 379 k 2024-09-13T14:19:22.031 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:19:22.254 INFO:teuthology.orchestra.run.smithi096.stdout:ceph-immutable-object-cache-17.2.7-1627.gbbdadf 617 kB/s | 137 kB 00:00 2024-09-13T14:19:22.254 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:22.255 INFO:teuthology.orchestra.run.smithi096.stdout:Total 613 kB/s | 137 kB 00:00 2024-09-13T14:19:22.255 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:19:22.262 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:19:22.262 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:19:22.318 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:19:22.319 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:19:22.471 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:19:22.572 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3. 1/1 2024-09-13T14:19:22.596 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3. 1/1 2024-09-13T14:19:22.597 INFO:teuthology.orchestra.run.smithi096.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:19:22.597 INFO:teuthology.orchestra.run.smithi096.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-13T14:19:22.597 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:22.969 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarc 7/7 2024-09-13T14:19:22.969 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3.el9. 1/7 2024-09-13T14:19:22.969 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarc 2/7 2024-09-13T14:19:22.969 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.el9.n 3/7 2024-09-13T14:19:22.969 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-09-13T14:19:22.969 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-09-13T14:19:22.969 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-09-13T14:19:23.258 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-09-13T14:19:23.258 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:23.258 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:19:23.258 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:19:23.258 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:19:23.258 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:19:23.258 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-13T14:19:23.259 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-13T14:19:23.259 INFO:teuthology.orchestra.run.smithi204.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-13T14:19:23.259 INFO:teuthology.orchestra.run.smithi204.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-13T14:19:23.259 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:23.259 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:19:23.506 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-13T14:19:23.588 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3. 1/1 2024-09-13T14:19:23.588 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:23.588 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:19:23.588 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:23.588 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:23.588 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:19:23.782 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-mgr 2024-09-13T14:19:24.060 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:01 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:19:24.191 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:19:24.193 INFO:teuthology.orchestra.run.smithi204.stdout:======================================================================================= 2024-09-13T14:19:24.193 INFO:teuthology.orchestra.run.smithi204.stdout: Package Arch Version Repository Size 2024-09-13T14:19:24.193 INFO:teuthology.orchestra.run.smithi204.stdout:======================================================================================= 2024-09-13T14:19:24.193 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:19:24.193 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 7.4 M 2024-09-13T14:19:24.193 INFO:teuthology.orchestra.run.smithi204.stdout:Installing dependencies: 2024-09-13T14:19:24.193 INFO:teuthology.orchestra.run.smithi204.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-13T14:19:24.193 INFO:teuthology.orchestra.run.smithi204.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-13T14:19:24.193 INFO:teuthology.orchestra.run.smithi204.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-13T14:19:24.193 INFO:teuthology.orchestra.run.smithi204.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-13T14:19:24.194 INFO:teuthology.orchestra.run.smithi204.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-13T14:19:24.194 INFO:teuthology.orchestra.run.smithi204.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-13T14:19:24.194 INFO:teuthology.orchestra.run.smithi204.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-13T14:19:24.194 INFO:teuthology.orchestra.run.smithi204.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-13T14:19:24.194 INFO:teuthology.orchestra.run.smithi204.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-13T14:19:24.194 INFO:teuthology.orchestra.run.smithi204.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-13T14:19:24.194 INFO:teuthology.orchestra.run.smithi204.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-13T14:19:24.194 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:24.194 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:19:24.194 INFO:teuthology.orchestra.run.smithi204.stdout:======================================================================================= 2024-09-13T14:19:24.194 INFO:teuthology.orchestra.run.smithi204.stdout:Install 12 Packages 2024-09-13T14:19:24.194 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:24.195 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 38 M 2024-09-13T14:19:24.195 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 212 M 2024-09-13T14:19:24.195 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:19:24.356 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:55 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:19:24.450 INFO:teuthology.orchestra.run.smithi096.stdout:Package ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:19:24.491 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:19:24.492 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2024-09-13T14:19:24.493 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:19:24.554 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-mgr-dashboard 2024-09-13T14:19:24.600 INFO:teuthology.orchestra.run.smithi204.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 1.1 MB/s | 191 kB 00:00 2024-09-13T14:19:24.750 INFO:teuthology.orchestra.run.smithi204.stdout:(2/12): libgfortran-11.5.0-2.el9.x86_64.rpm 2.4 MB/s | 801 kB 00:00 2024-09-13T14:19:24.884 INFO:teuthology.orchestra.run.smithi204.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 115 kB/s | 33 kB 00:00 2024-09-13T14:19:25.001 INFO:teuthology.orchestra.run.smithi204.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 156 kB/s | 18 kB 00:00 2024-09-13T14:19:25.110 INFO:teuthology.orchestra.run.smithi204.stdout:(5/12): ceph-mgr-diskprediction-local-17.2.7-16 11 MB/s | 7.4 MB 00:00 2024-09-13T14:19:25.124 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:56 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:19:25.135 INFO:teuthology.orchestra.run.smithi204.stdout:(6/12): openblas-0.3.26-2.el9.x86_64.rpm 288 kB/s | 38 kB 00:00 2024-09-13T14:19:25.257 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:19:25.258 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:25.258 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2024-09-13T14:19:25.258 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:25.258 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:19:25.258 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 3.4 M 2024-09-13T14:19:25.258 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2024-09-13T14:19:25.259 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 24 k 2024-09-13T14:19:25.259 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 15 k 2024-09-13T14:19:25.259 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-13T14:19:25.259 INFO:teuthology.orchestra.run.smithi096.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-13T14:19:25.259 INFO:teuthology.orchestra.run.smithi096.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-13T14:19:25.259 INFO:teuthology.orchestra.run.smithi096.stdout:Installing weak dependencies: 2024-09-13T14:19:25.259 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-13T14:19:25.259 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:25.259 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:19:25.259 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:25.259 INFO:teuthology.orchestra.run.smithi096.stdout:Install 7 Packages 2024-09-13T14:19:25.259 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:25.260 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 3.7 M 2024-09-13T14:19:25.260 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 80 M 2024-09-13T14:19:25.260 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:19:25.427 INFO:teuthology.orchestra.run.smithi204.stdout:(7/12): python3-devel-3.9.19-8.el9.x86_64.rpm 839 kB/s | 245 kB 00:00 2024-09-13T14:19:25.467 INFO:teuthology.orchestra.run.smithi096.stdout:(1/7): ceph-grafana-dashboards-17.2.7-1627.gbbd 159 kB/s | 24 kB 00:00 2024-09-13T14:19:25.492 INFO:teuthology.orchestra.run.smithi096.stdout:(2/7): ceph-prometheus-alerts-17.2.7-1627.gbbda 81 kB/s | 15 kB 00:00 2024-09-13T14:19:25.543 INFO:teuthology.orchestra.run.smithi096.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 120 kB/s | 9.0 kB 00:00 2024-09-13T14:19:25.576 INFO:teuthology.orchestra.run.smithi096.stdout:(4/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 977 kB/s | 31 kB 00:00 2024-09-13T14:19:25.601 INFO:teuthology.orchestra.run.smithi096.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 376 kB/s | 41 kB 00:00 2024-09-13T14:19:25.635 INFO:teuthology.orchestra.run.smithi096.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 3.2 MB/s | 188 kB 00:00 2024-09-13T14:19:25.786 INFO:teuthology.orchestra.run.smithi204.stdout:(8/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 2.9 MB/s | 3.0 MB 00:01 2024-09-13T14:19:25.819 INFO:teuthology.orchestra.run.smithi096.stdout:(7/7): ceph-mgr-dashboard-17.2.7-1627.gbbdadfa3 6.6 MB/s | 3.4 MB 00:00 2024-09-13T14:19:25.822 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:25.822 INFO:teuthology.orchestra.run.smithi096.stdout:Total 6.5 MB/s | 3.7 MB 00:00 2024-09-13T14:19:25.822 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:19:25.831 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:19:25.831 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:19:25.911 INFO:teuthology.orchestra.run.smithi204.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.0 MB/s | 250 kB 00:00 2024-09-13T14:19:25.912 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:19:25.913 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:19:26.220 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:19:26.286 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-09-13T14:19:26.320 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-09-13T14:19:26.375 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-09-13T14:19:26.395 INFO:teuthology.orchestra.run.smithi204.stdout:(10/12): openblas-openmp-0.3.26-2.el9.x86_64.rp 3.8 MB/s | 4.9 MB 00:01 2024-09-13T14:19:26.407 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-09-13T14:19:26.436 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.el9.n 5/7 2024-09-13T14:19:27.430 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3.el9. 6/7 2024-09-13T14:19:27.473 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarc 7/7 2024-09-13T14:19:27.645 INFO:teuthology.orchestra.run.smithi204.stdout:(11/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 2.3 MB/s | 5.1 MB 00:02 2024-09-13T14:19:28.530 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarc 7/7 2024-09-13T14:19:28.530 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3.el9. 1/7 2024-09-13T14:19:28.530 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarc 2/7 2024-09-13T14:19:28.530 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.el9.n 3/7 2024-09-13T14:19:28.530 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-09-13T14:19:28.530 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-09-13T14:19:28.530 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-09-13T14:19:28.811 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-09-13T14:19:28.811 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:28.811 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:19:28.812 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:19:28.812 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:19:28.812 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:19:28.812 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-13T14:19:28.812 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-13T14:19:28.812 INFO:teuthology.orchestra.run.smithi096.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-13T14:19:28.812 INFO:teuthology.orchestra.run.smithi096.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-13T14:19:28.812 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:28.812 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:19:29.037 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-13T14:19:29.595 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:00 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:19:29.727 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:19:29.728 INFO:teuthology.orchestra.run.smithi096.stdout:======================================================================================= 2024-09-13T14:19:29.728 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2024-09-13T14:19:29.728 INFO:teuthology.orchestra.run.smithi096.stdout:======================================================================================= 2024-09-13T14:19:29.729 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:19:29.729 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 7.4 M 2024-09-13T14:19:29.729 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2024-09-13T14:19:29.729 INFO:teuthology.orchestra.run.smithi096.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-13T14:19:29.729 INFO:teuthology.orchestra.run.smithi096.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-13T14:19:29.729 INFO:teuthology.orchestra.run.smithi096.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-13T14:19:29.729 INFO:teuthology.orchestra.run.smithi096.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-13T14:19:29.729 INFO:teuthology.orchestra.run.smithi096.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-13T14:19:29.729 INFO:teuthology.orchestra.run.smithi096.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-13T14:19:29.729 INFO:teuthology.orchestra.run.smithi096.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-13T14:19:29.729 INFO:teuthology.orchestra.run.smithi096.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-13T14:19:29.729 INFO:teuthology.orchestra.run.smithi096.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-13T14:19:29.729 INFO:teuthology.orchestra.run.smithi096.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-13T14:19:29.729 INFO:teuthology.orchestra.run.smithi096.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-13T14:19:29.729 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:29.730 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:19:29.730 INFO:teuthology.orchestra.run.smithi096.stdout:======================================================================================= 2024-09-13T14:19:29.730 INFO:teuthology.orchestra.run.smithi096.stdout:Install 12 Packages 2024-09-13T14:19:29.730 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:29.730 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 38 M 2024-09-13T14:19:29.730 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 212 M 2024-09-13T14:19:29.731 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:19:30.710 INFO:teuthology.orchestra.run.smithi096.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 337 kB/s | 191 kB 00:00 2024-09-13T14:19:30.827 INFO:teuthology.orchestra.run.smithi096.stdout:(2/12): ceph-mgr-diskprediction-local-17.2.7-16 11 MB/s | 7.4 MB 00:00 2024-09-13T14:19:30.852 INFO:teuthology.orchestra.run.smithi096.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 231 kB/s | 33 kB 00:00 2024-09-13T14:19:30.911 INFO:teuthology.orchestra.run.smithi096.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 312 kB/s | 18 kB 00:00 2024-09-13T14:19:30.969 INFO:teuthology.orchestra.run.smithi096.stdout:(5/12): openblas-0.3.26-2.el9.x86_64.rpm 660 kB/s | 38 kB 00:00 2024-09-13T14:19:31.036 INFO:teuthology.orchestra.run.smithi096.stdout:(6/12): libgfortran-11.5.0-2.el9.x86_64.rpm 897 kB/s | 801 kB 00:00 2024-09-13T14:19:31.120 INFO:teuthology.orchestra.run.smithi096.stdout:(7/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 10 MB/s | 3.0 MB 00:00 2024-09-13T14:19:31.187 INFO:teuthology.orchestra.run.smithi096.stdout:(8/12): python3-devel-3.9.19-8.el9.x86_64.rpm 1.6 MB/s | 245 kB 00:00 2024-09-13T14:19:31.279 INFO:teuthology.orchestra.run.smithi096.stdout:(9/12): openblas-openmp-0.3.26-2.el9.x86_64.rpm 16 MB/s | 4.9 MB 00:00 2024-09-13T14:19:31.330 INFO:teuthology.orchestra.run.smithi096.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 1.7 MB/s | 250 kB 00:00 2024-09-13T14:19:31.405 INFO:teuthology.orchestra.run.smithi096.stdout:(11/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 18 MB/s | 5.1 MB 00:00 2024-09-13T14:19:31.971 INFO:teuthology.orchestra.run.smithi096.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 23 MB/s | 16 MB 00:00 2024-09-13T14:19:31.972 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:31.973 INFO:teuthology.orchestra.run.smithi096.stdout:Total 17 MB/s | 38 MB 00:02 2024-09-13T14:19:32.327 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:19:32.352 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:19:32.354 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:19:32.884 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:19:32.885 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:19:33.641 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:19:33.731 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-13T14:19:33.769 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-13T14:19:33.869 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-13T14:19:33.921 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-13T14:19:34.104 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-13T14:19:34.128 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-13T14:19:34.215 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-13T14:19:34.863 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-13T14:19:34.975 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-13T14:19:36.224 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-13T14:19:37.266 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-13T14:19:37.295 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdad 12/12 2024-09-13T14:19:39.987 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdad 12/12 2024-09-13T14:19:39.988 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdad 1/12 2024-09-13T14:19:39.988 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-13T14:19:39.988 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-13T14:19:39.988 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-13T14:19:39.988 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-13T14:19:39.988 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-13T14:19:39.988 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-13T14:19:39.988 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-13T14:19:39.988 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-13T14:19:39.988 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-13T14:19:39.988 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-13T14:19:40.428 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-13T14:19:40.428 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:40.428 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:19:40.429 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:19:40.429 INFO:teuthology.orchestra.run.smithi096.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-13T14:19:40.429 INFO:teuthology.orchestra.run.smithi096.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-13T14:19:40.429 INFO:teuthology.orchestra.run.smithi096.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-13T14:19:40.429 INFO:teuthology.orchestra.run.smithi096.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-13T14:19:40.429 INFO:teuthology.orchestra.run.smithi096.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-13T14:19:40.429 INFO:teuthology.orchestra.run.smithi096.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-13T14:19:40.429 INFO:teuthology.orchestra.run.smithi096.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-13T14:19:40.429 INFO:teuthology.orchestra.run.smithi096.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-13T14:19:40.429 INFO:teuthology.orchestra.run.smithi096.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-13T14:19:40.429 INFO:teuthology.orchestra.run.smithi096.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-13T14:19:40.429 INFO:teuthology.orchestra.run.smithi096.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-13T14:19:40.429 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:40.429 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:19:40.814 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-mgr-rook 2024-09-13T14:19:41.389 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:12 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:19:41.522 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:19:41.523 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:41.523 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2024-09-13T14:19:41.523 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:41.523 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:19:41.524 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-rook noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 47 k 2024-09-13T14:19:41.524 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2024-09-13T14:19:41.524 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-13T14:19:41.524 INFO:teuthology.orchestra.run.smithi096.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-13T14:19:41.524 INFO:teuthology.orchestra.run.smithi096.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-13T14:19:41.524 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-13T14:19:41.524 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-13T14:19:41.524 INFO:teuthology.orchestra.run.smithi096.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-13T14:19:41.524 INFO:teuthology.orchestra.run.smithi096.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-13T14:19:41.524 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-13T14:19:41.524 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-13T14:19:41.524 INFO:teuthology.orchestra.run.smithi096.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-13T14:19:41.524 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-13T14:19:41.524 INFO:teuthology.orchestra.run.smithi096.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-13T14:19:41.525 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:41.525 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:19:41.525 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:41.525 INFO:teuthology.orchestra.run.smithi096.stdout:Install 13 Packages 2024-09-13T14:19:41.525 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:41.525 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 2.2 M 2024-09-13T14:19:41.525 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 26 M 2024-09-13T14:19:41.526 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:19:42.213 INFO:teuthology.orchestra.run.smithi096.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 189 kB/s | 19 kB 00:00 2024-09-13T14:19:42.238 INFO:teuthology.orchestra.run.smithi096.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 202 kB/s | 26 kB 00:00 2024-09-13T14:19:42.297 INFO:teuthology.orchestra.run.smithi096.stdout:(3/13): ceph-mgr-rook-17.2.7-1627.gbbdadfa3.el9 256 kB/s | 47 kB 00:00 2024-09-13T14:19:42.347 INFO:teuthology.orchestra.run.smithi096.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.6 MB/s | 222 kB 00:00 2024-09-13T14:19:42.372 INFO:teuthology.orchestra.run.smithi096.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.2 MB/s | 159 kB 00:00 2024-09-13T14:19:42.398 INFO:teuthology.orchestra.run.smithi096.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 1.0 MB/s | 54 kB 00:00 2024-09-13T14:19:42.448 INFO:teuthology.orchestra.run.smithi096.stdout:(7/13): python3-cachetools-4.2.4-1.el9.noarch.r 428 kB/s | 32 kB 00:00 2024-09-13T14:19:42.473 INFO:teuthology.orchestra.run.smithi096.stdout:(8/13): python3-certifi-2023.05.07-4.el9.noarch 188 kB/s | 14 kB 00:00 2024-09-13T14:19:42.523 INFO:teuthology.orchestra.run.smithi096.stdout:(9/13): python3-google-auth-2.34.0-1.el9.noarch 2.9 MB/s | 220 kB 00:00 2024-09-13T14:19:42.549 INFO:teuthology.orchestra.run.smithi096.stdout:(10/13): python3-pyasn1-modules-0.4.8-6.el9.noa 1.1 MB/s | 279 kB 00:00 2024-09-13T14:19:42.574 INFO:teuthology.orchestra.run.smithi096.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 1.1 MB/s | 59 kB 00:00 2024-09-13T14:19:42.633 INFO:teuthology.orchestra.run.smithi096.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.0 MB/s | 90 kB 00:00 2024-09-13T14:19:42.683 INFO:teuthology.orchestra.run.smithi096.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 4.9 MB/s | 1.0 MB 00:00 2024-09-13T14:19:42.686 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:42.686 INFO:teuthology.orchestra.run.smithi096.stdout:Total 1.9 MB/s | 2.2 MB 00:01 2024-09-13T14:19:42.752 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:19:42.768 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:19:42.768 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:19:42.857 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:19:42.859 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:19:43.360 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:19:43.538 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-13T14:19:43.592 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-13T14:19:43.648 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-13T14:19:43.685 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-13T14:19:43.727 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-13T14:19:43.828 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-13T14:19:43.931 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-13T14:19:43.978 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-13T14:19:44.701 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-13T14:19:44.751 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-13T14:19:44.785 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-13T14:19:44.831 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-13T14:19:44.855 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 13/13 2024-09-13T14:19:45.694 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 13/13 2024-09-13T14:19:45.694 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/13 2024-09-13T14:19:45.694 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-13T14:19:45.694 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-13T14:19:45.694 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-13T14:19:45.694 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-13T14:19:45.694 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-13T14:19:45.694 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-13T14:19:45.694 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-13T14:19:45.694 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-13T14:19:45.694 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-13T14:19:45.695 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-13T14:19:45.695 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-13T14:19:45.968 INFO:teuthology.orchestra.run.smithi204.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 830 kB/s | 16 MB 00:20 2024-09-13T14:19:45.969 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:45.969 INFO:teuthology.orchestra.run.smithi204.stdout:Total 1.8 MB/s | 38 MB 00:21 2024-09-13T14:19:46.026 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-13T14:19:46.027 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:46.027 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:19:46.027 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:19:46.027 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-13T14:19:46.027 INFO:teuthology.orchestra.run.smithi096.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-13T14:19:46.027 INFO:teuthology.orchestra.run.smithi096.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-13T14:19:46.027 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-13T14:19:46.027 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-13T14:19:46.027 INFO:teuthology.orchestra.run.smithi096.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-13T14:19:46.027 INFO:teuthology.orchestra.run.smithi096.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-13T14:19:46.027 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-13T14:19:46.027 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-13T14:19:46.027 INFO:teuthology.orchestra.run.smithi096.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-13T14:19:46.027 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-13T14:19:46.028 INFO:teuthology.orchestra.run.smithi096.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-13T14:19:46.028 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:46.028 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:19:46.266 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-mgr-cephadm 2024-09-13T14:19:46.325 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:19:46.351 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:19:46.351 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:19:46.836 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:17 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:19:46.875 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:19:46.876 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:19:46.968 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:19:46.969 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:46.969 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2024-09-13T14:19:46.969 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:46.969 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:19:46.969 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 124 k 2024-09-13T14:19:46.970 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2024-09-13T14:19:46.970 INFO:teuthology.orchestra.run.smithi096.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-13T14:19:46.970 INFO:teuthology.orchestra.run.smithi096.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-13T14:19:46.970 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-13T14:19:46.970 INFO:teuthology.orchestra.run.smithi096.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-13T14:19:46.970 INFO:teuthology.orchestra.run.smithi096.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-13T14:19:46.970 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:46.970 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:19:46.970 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:46.970 INFO:teuthology.orchestra.run.smithi096.stdout:Install 6 Packages 2024-09-13T14:19:46.970 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:46.971 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 7.0 M 2024-09-13T14:19:46.971 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 33 M 2024-09-13T14:19:46.971 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:19:47.471 INFO:teuthology.orchestra.run.smithi096.stdout:(1/6): ceph-mgr-cephadm-17.2.7-1627.gbbdadfa3.e 599 kB/s | 124 kB 00:00 2024-09-13T14:19:47.529 INFO:teuthology.orchestra.run.smithi096.stdout:(2/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 9.2 MB/s | 548 kB 00:00 2024-09-13T14:19:47.555 INFO:teuthology.orchestra.run.smithi096.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 2.2 MB/s | 58 kB 00:00 2024-09-13T14:19:47.580 INFO:teuthology.orchestra.run.smithi096.stdout:(4/6): python3-typing-extensions-4.12.2-2.el9.n 2.9 MB/s | 74 kB 00:00 2024-09-13T14:19:47.652 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:19:47.737 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-13T14:19:47.774 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-13T14:19:47.873 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-13T14:19:47.897 INFO:teuthology.orchestra.run.smithi096.stdout:(5/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 392 kB/s | 248 kB 00:00 2024-09-13T14:19:47.909 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-13T14:19:48.096 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-13T14:19:48.141 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-13T14:19:48.223 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-13T14:19:48.896 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-13T14:19:49.013 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-13T14:19:49.630 INFO:teuthology.orchestra.run.smithi096.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 2.5 MB/s | 6.0 MB 00:02 2024-09-13T14:19:49.632 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:49.633 INFO:teuthology.orchestra.run.smithi096.stdout:Total 2.6 MB/s | 7.0 MB 00:02 2024-09-13T14:19:49.706 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:19:49.718 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:19:49.718 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:19:49.818 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:19:49.819 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:19:50.124 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:19:50.263 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-13T14:19:50.294 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-13T14:19:50.349 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-13T14:19:50.877 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-13T14:19:50.968 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-13T14:19:51.024 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-13T14:19:51.053 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 6/6 2024-09-13T14:19:51.295 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-13T14:19:51.425 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdad 12/12 2024-09-13T14:19:51.875 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 6/6 2024-09-13T14:19:51.875 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/6 2024-09-13T14:19:51.876 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-13T14:19:51.876 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-13T14:19:51.876 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-13T14:19:51.876 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-13T14:19:52.148 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-13T14:19:52.148 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:52.148 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:19:52.148 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:19:52.148 INFO:teuthology.orchestra.run.smithi096.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-13T14:19:52.148 INFO:teuthology.orchestra.run.smithi096.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-13T14:19:52.148 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-13T14:19:52.148 INFO:teuthology.orchestra.run.smithi096.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-13T14:19:52.148 INFO:teuthology.orchestra.run.smithi096.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-13T14:19:52.148 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:52.149 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:19:52.411 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-fuse 2024-09-13T14:19:52.977 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:23 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:19:53.109 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:19:53.110 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:53.110 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2024-09-13T14:19:53.110 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:53.111 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:19:53.111 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-fuse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 813 k 2024-09-13T14:19:53.111 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2024-09-13T14:19:53.111 INFO:teuthology.orchestra.run.smithi096.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-13T14:19:53.111 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:53.111 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:19:53.111 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:53.111 INFO:teuthology.orchestra.run.smithi096.stdout:Install 2 Packages 2024-09-13T14:19:53.111 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:53.111 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 893 k 2024-09-13T14:19:53.111 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 2.6 M 2024-09-13T14:19:53.111 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:19:53.416 INFO:teuthology.orchestra.run.smithi096.stdout:(1/2): ceph-fuse-17.2.7-1627.gbbdadfa3.el9.x86_ 3.1 MB/s | 813 kB 00:00 2024-09-13T14:19:53.791 INFO:teuthology.orchestra.run.smithi096.stdout:(2/2): fuse-2.9.9-16.el9.x86_64.rpm 126 kB/s | 80 kB 00:00 2024-09-13T14:19:53.791 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:53.792 INFO:teuthology.orchestra.run.smithi096.stdout:Total 1.3 MB/s | 893 kB 00:00 2024-09-13T14:19:53.803 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:19:53.824 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:19:53.824 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:19:53.877 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdad 12/12 2024-09-13T14:19:53.877 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdad 1/12 2024-09-13T14:19:53.877 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-13T14:19:53.877 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-13T14:19:53.877 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-13T14:19:53.878 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-13T14:19:53.878 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-13T14:19:53.878 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-13T14:19:53.878 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-13T14:19:53.878 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-13T14:19:53.878 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-13T14:19:53.878 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-13T14:19:53.905 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:19:53.905 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:19:54.089 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:19:54.228 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-13T14:19:54.292 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-13T14:19:54.292 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:54.293 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:19:54.293 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:19:54.293 INFO:teuthology.orchestra.run.smithi204.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-13T14:19:54.293 INFO:teuthology.orchestra.run.smithi204.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-13T14:19:54.293 INFO:teuthology.orchestra.run.smithi204.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-13T14:19:54.293 INFO:teuthology.orchestra.run.smithi204.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-13T14:19:54.293 INFO:teuthology.orchestra.run.smithi204.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-13T14:19:54.293 INFO:teuthology.orchestra.run.smithi204.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-13T14:19:54.293 INFO:teuthology.orchestra.run.smithi204.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-13T14:19:54.293 INFO:teuthology.orchestra.run.smithi204.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-13T14:19:54.293 INFO:teuthology.orchestra.run.smithi204.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-13T14:19:54.294 INFO:teuthology.orchestra.run.smithi204.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-13T14:19:54.294 INFO:teuthology.orchestra.run.smithi204.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-13T14:19:54.294 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:54.294 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:19:54.314 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/2 2024-09-13T14:19:54.691 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install ceph-mgr-rook 2024-09-13T14:19:54.986 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/2 2024-09-13T14:19:54.986 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/2 2024-09-13T14:19:55.264 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-13T14:19:55.264 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:55.264 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:19:55.264 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-13T14:19:55.264 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:55.264 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:19:55.270 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:32 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:19:55.405 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:19:55.406 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:19:55.406 INFO:teuthology.orchestra.run.smithi204.stdout: Package Arch Version Repository Size 2024-09-13T14:19:55.406 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:19:55.406 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:19:55.406 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mgr-rook noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 47 k 2024-09-13T14:19:55.407 INFO:teuthology.orchestra.run.smithi204.stdout:Installing dependencies: 2024-09-13T14:19:55.407 INFO:teuthology.orchestra.run.smithi204.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-13T14:19:55.407 INFO:teuthology.orchestra.run.smithi204.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-13T14:19:55.407 INFO:teuthology.orchestra.run.smithi204.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-13T14:19:55.407 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-13T14:19:55.408 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-13T14:19:55.408 INFO:teuthology.orchestra.run.smithi204.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-13T14:19:55.408 INFO:teuthology.orchestra.run.smithi204.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-13T14:19:55.408 INFO:teuthology.orchestra.run.smithi204.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-13T14:19:55.408 INFO:teuthology.orchestra.run.smithi204.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-13T14:19:55.409 INFO:teuthology.orchestra.run.smithi204.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-13T14:19:55.409 INFO:teuthology.orchestra.run.smithi204.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-13T14:19:55.409 INFO:teuthology.orchestra.run.smithi204.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-13T14:19:55.409 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:55.409 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:19:55.409 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:19:55.410 INFO:teuthology.orchestra.run.smithi204.stdout:Install 13 Packages 2024-09-13T14:19:55.410 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:19:55.411 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 2.2 M 2024-09-13T14:19:55.411 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 26 M 2024-09-13T14:19:55.411 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:19:55.544 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install librados-devel 2024-09-13T14:19:55.885 INFO:teuthology.orchestra.run.smithi204.stdout:(1/13): ceph-mgr-rook-17.2.7-1627.gbbdadfa3.el9 283 kB/s | 47 kB 00:00 2024-09-13T14:19:56.010 INFO:teuthology.orchestra.run.smithi204.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 87 kB/s | 26 kB 00:00 2024-09-13T14:19:56.035 INFO:teuthology.orchestra.run.smithi204.stdout:(3/13): python3-jsonpointer-2.0-4.el9.noarch.rp 60 kB/s | 19 kB 00:00 2024-09-13T14:19:56.127 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:27 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:19:56.263 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:19:56.264 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:56.264 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repo Size 2024-09-13T14:19:56.264 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:56.264 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:19:56.264 INFO:teuthology.orchestra.run.smithi096.stdout: librados-devel x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 126 k 2024-09-13T14:19:56.264 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:56.264 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:19:56.264 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:56.264 INFO:teuthology.orchestra.run.smithi096.stdout:Install 1 Package 2024-09-13T14:19:56.264 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:56.264 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 126 k 2024-09-13T14:19:56.264 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 448 k 2024-09-13T14:19:56.264 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:19:56.410 INFO:teuthology.orchestra.run.smithi204.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 423 kB/s | 159 kB 00:00 2024-09-13T14:19:56.436 INFO:teuthology.orchestra.run.smithi204.stdout:(5/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 403 kB/s | 222 kB 00:00 2024-09-13T14:19:56.472 INFO:teuthology.orchestra.run.smithi096.stdout:librados-devel-17.2.7-1627.gbbdadfa3.el9.x86_64 609 kB/s | 126 kB 00:00 2024-09-13T14:19:56.472 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:56.473 INFO:teuthology.orchestra.run.smithi096.stdout:Total 604 kB/s | 126 kB 00:00 2024-09-13T14:19:56.473 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:19:56.478 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:19:56.479 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:19:56.486 INFO:teuthology.orchestra.run.smithi204.stdout:(6/13): python3-cachetools-4.2.4-1.el9.noarch.r 645 kB/s | 32 kB 00:00 2024-09-13T14:19:56.519 INFO:teuthology.orchestra.run.smithi204.stdout:(7/13): python3-pyasn1-modules-0.4.8-6.el9.noar 576 kB/s | 279 kB 00:00 2024-09-13T14:19:56.543 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:19:56.543 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:19:56.545 INFO:teuthology.orchestra.run.smithi204.stdout:(8/13): python3-certifi-2023.05.07-4.el9.noarch 240 kB/s | 14 kB 00:00 2024-09-13T14:19:56.580 INFO:teuthology.orchestra.run.smithi204.stdout:(9/13): python3-requests-oauthlib-1.3.0-12.el9. 319 kB/s | 54 kB 00:00 2024-09-13T14:19:56.620 INFO:teuthology.orchestra.run.smithi204.stdout:(10/13): python3-google-auth-2.34.0-1.el9.noarc 2.1 MB/s | 220 kB 00:00 2024-09-13T14:19:56.654 INFO:teuthology.orchestra.run.smithi204.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 805 kB/s | 59 kB 00:00 2024-09-13T14:19:56.679 INFO:teuthology.orchestra.run.smithi204.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.5 MB/s | 90 kB 00:00 2024-09-13T14:19:56.691 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:19:56.721 INFO:teuthology.orchestra.run.smithi204.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 6.0 MB/s | 1.0 MB 00:00 2024-09-13T14:19:56.724 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:56.724 INFO:teuthology.orchestra.run.smithi204.stdout:Total 1.7 MB/s | 2.2 MB 00:01 2024-09-13T14:19:56.789 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:19:56.804 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:19:56.804 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:19:56.811 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:19:56.893 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:19:56.894 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:19:57.234 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:19:57.400 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:19:57.486 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:19:57.486 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:57.486 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:19:57.486 INFO:teuthology.orchestra.run.smithi096.stdout: librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:19:57.487 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:57.487 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:19:57.568 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-13T14:19:57.630 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-13T14:19:57.686 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-13T14:19:57.723 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install libcephfs2 2024-09-13T14:19:57.732 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-13T14:19:57.782 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-13T14:19:57.892 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-13T14:19:57.994 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-13T14:19:58.042 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-13T14:19:58.289 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:29 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:19:58.379 INFO:teuthology.orchestra.run.smithi096.stdout:Package libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:19:58.422 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:19:58.424 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2024-09-13T14:19:58.424 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:19:58.514 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install libcephfs-devel 2024-09-13T14:19:58.774 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-13T14:19:58.823 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-13T14:19:58.857 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-13T14:19:58.902 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-13T14:19:58.927 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 13/13 2024-09-13T14:19:59.070 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:30 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:19:59.202 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:19:59.203 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:59.203 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repo Size 2024-09-13T14:19:59.204 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:59.204 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:19:59.204 INFO:teuthology.orchestra.run.smithi096.stdout: libcephfs-devel x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 25 k 2024-09-13T14:19:59.204 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:59.204 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:19:59.204 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:19:59.204 INFO:teuthology.orchestra.run.smithi096.stdout:Install 1 Package 2024-09-13T14:19:59.204 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:19:59.204 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 25 k 2024-09-13T14:19:59.205 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 110 k 2024-09-13T14:19:59.205 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:19:59.368 INFO:teuthology.orchestra.run.smithi096.stdout:libcephfs-devel-17.2.7-1627.gbbdadfa3.el9.x86_6 152 kB/s | 25 kB 00:00 2024-09-13T14:19:59.368 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:19:59.369 INFO:teuthology.orchestra.run.smithi096.stdout:Total 151 kB/s | 25 kB 00:00 2024-09-13T14:19:59.369 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:19:59.372 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:19:59.372 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:19:59.374 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:19:59.374 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:19:59.459 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:19:59.565 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:19:59.715 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 13/13 2024-09-13T14:19:59.716 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/13 2024-09-13T14:19:59.716 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-13T14:19:59.716 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-13T14:19:59.716 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-13T14:19:59.716 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-13T14:19:59.716 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-13T14:19:59.716 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-13T14:19:59.716 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-13T14:19:59.716 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-13T14:19:59.716 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-13T14:19:59.716 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-13T14:19:59.716 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-13T14:19:59.917 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:00.069 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-13T14:20:00.070 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:00.070 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:20:00.070 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:20:00.070 INFO:teuthology.orchestra.run.smithi204.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-13T14:20:00.070 INFO:teuthology.orchestra.run.smithi204.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-13T14:20:00.070 INFO:teuthology.orchestra.run.smithi204.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-13T14:20:00.070 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-13T14:20:00.070 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-13T14:20:00.070 INFO:teuthology.orchestra.run.smithi204.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-13T14:20:00.070 INFO:teuthology.orchestra.run.smithi204.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-13T14:20:00.070 INFO:teuthology.orchestra.run.smithi204.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-13T14:20:00.071 INFO:teuthology.orchestra.run.smithi204.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-13T14:20:00.071 INFO:teuthology.orchestra.run.smithi204.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-13T14:20:00.071 INFO:teuthology.orchestra.run.smithi204.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-13T14:20:00.071 INFO:teuthology.orchestra.run.smithi204.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-13T14:20:00.071 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:00.071 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:20:00.153 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:00.153 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:20:00.153 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:20:00.153 INFO:teuthology.orchestra.run.smithi096.stdout: libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:20:00.153 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:20:00.153 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:20:00.327 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install ceph-mgr-cephadm 2024-09-13T14:20:00.348 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install librados2 2024-09-13T14:20:00.892 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:37 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:20:00.909 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:31 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:20:01.000 INFO:teuthology.orchestra.run.smithi096.stdout:Package librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:20:01.030 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:20:01.030 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:01.031 INFO:teuthology.orchestra.run.smithi204.stdout: Package Arch Version Repository Size 2024-09-13T14:20:01.031 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:01.031 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:20:01.031 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 124 k 2024-09-13T14:20:01.031 INFO:teuthology.orchestra.run.smithi204.stdout:Installing dependencies: 2024-09-13T14:20:01.031 INFO:teuthology.orchestra.run.smithi204.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-13T14:20:01.031 INFO:teuthology.orchestra.run.smithi204.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-13T14:20:01.031 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-13T14:20:01.031 INFO:teuthology.orchestra.run.smithi204.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-13T14:20:01.031 INFO:teuthology.orchestra.run.smithi204.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-13T14:20:01.031 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:01.031 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:20:01.031 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:01.032 INFO:teuthology.orchestra.run.smithi204.stdout:Install 6 Packages 2024-09-13T14:20:01.032 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:01.032 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 7.0 M 2024-09-13T14:20:01.032 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 33 M 2024-09-13T14:20:01.032 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:20:01.044 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:20:01.045 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2024-09-13T14:20:01.045 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:20:01.108 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install librbd1 2024-09-13T14:20:01.284 INFO:teuthology.orchestra.run.smithi204.stdout:(1/6): ceph-mgr-cephadm-17.2.7-1627.gbbdadfa3.e 611 kB/s | 124 kB 00:00 2024-09-13T14:20:01.309 INFO:teuthology.orchestra.run.smithi204.stdout:(2/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.1 MB/s | 248 kB 00:00 2024-09-13T14:20:01.367 INFO:teuthology.orchestra.run.smithi204.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 995 kB/s | 58 kB 00:00 2024-09-13T14:20:01.417 INFO:teuthology.orchestra.run.smithi204.stdout:(4/6): python3-typing-extensions-4.12.2-2.el9.n 1.5 MB/s | 74 kB 00:00 2024-09-13T14:20:01.451 INFO:teuthology.orchestra.run.smithi204.stdout:(5/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 3.2 MB/s | 548 kB 00:00 2024-09-13T14:20:01.643 INFO:teuthology.orchestra.run.smithi204.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 11 MB/s | 6.0 MB 00:00 2024-09-13T14:20:01.645 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:20:01.645 INFO:teuthology.orchestra.run.smithi204.stdout:Total 11 MB/s | 7.0 MB 00:00 2024-09-13T14:20:01.673 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:32 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:20:01.715 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:20:01.726 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:20:01.726 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:20:01.764 INFO:teuthology.orchestra.run.smithi096.stdout:Package librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:20:01.807 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:20:01.808 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2024-09-13T14:20:01.808 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:20:01.824 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:20:01.826 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:20:01.870 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install python3-rados 2024-09-13T14:20:02.134 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:20:02.301 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-13T14:20:02.353 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-13T14:20:02.432 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:33 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:20:02.522 INFO:teuthology.orchestra.run.smithi096.stdout:Package python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:20:02.565 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:20:02.566 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2024-09-13T14:20:02.566 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:20:02.635 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install python3-rgw 2024-09-13T14:20:02.882 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-13T14:20:02.964 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-13T14:20:03.019 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-13T14:20:03.058 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 6/6 2024-09-13T14:20:03.206 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:34 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:20:03.296 INFO:teuthology.orchestra.run.smithi096.stdout:Package python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:20:03.339 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:20:03.340 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2024-09-13T14:20:03.340 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:20:03.399 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install python3-cephfs 2024-09-13T14:20:03.881 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 6/6 2024-09-13T14:20:03.881 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/6 2024-09-13T14:20:03.881 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-13T14:20:03.881 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-13T14:20:03.881 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-13T14:20:03.881 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-13T14:20:03.964 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:34 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:20:04.053 INFO:teuthology.orchestra.run.smithi096.stdout:Package python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:20:04.096 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:20:04.097 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2024-09-13T14:20:04.098 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:20:04.153 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-13T14:20:04.153 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:04.153 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:20:04.153 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:20:04.153 INFO:teuthology.orchestra.run.smithi204.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-13T14:20:04.153 INFO:teuthology.orchestra.run.smithi204.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-13T14:20:04.153 INFO:teuthology.orchestra.run.smithi204.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-13T14:20:04.153 INFO:teuthology.orchestra.run.smithi204.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-13T14:20:04.153 INFO:teuthology.orchestra.run.smithi204.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-13T14:20:04.153 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:04.153 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:20:04.161 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install python3-rbd 2024-09-13T14:20:04.401 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install ceph-fuse 2024-09-13T14:20:04.715 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:35 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:20:04.806 INFO:teuthology.orchestra.run.smithi096.stdout:Package python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:20:04.849 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:20:04.850 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2024-09-13T14:20:04.850 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:20:04.907 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install rbd-fuse 2024-09-13T14:20:04.967 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:41 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:20:05.102 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:20:05.103 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:05.103 INFO:teuthology.orchestra.run.smithi204.stdout: Package Arch Version Repository Size 2024-09-13T14:20:05.103 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:05.103 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:20:05.103 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-fuse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 813 k 2024-09-13T14:20:05.103 INFO:teuthology.orchestra.run.smithi204.stdout:Installing dependencies: 2024-09-13T14:20:05.103 INFO:teuthology.orchestra.run.smithi204.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-13T14:20:05.103 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:05.103 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:20:05.103 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:05.103 INFO:teuthology.orchestra.run.smithi204.stdout:Install 2 Packages 2024-09-13T14:20:05.104 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:05.104 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 893 k 2024-09-13T14:20:05.104 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 2.6 M 2024-09-13T14:20:05.104 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:20:05.412 INFO:teuthology.orchestra.run.smithi204.stdout:(1/2): ceph-fuse-17.2.7-1627.gbbdadfa3.el9.x86_ 2.8 MB/s | 813 kB 00:00 2024-09-13T14:20:05.469 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:36 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:20:05.601 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:20:05.601 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:20:05.601 INFO:teuthology.orchestra.run.smithi096.stdout: Package Architecture Version Repository Size 2024-09-13T14:20:05.601 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:20:05.601 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:20:05.602 INFO:teuthology.orchestra.run.smithi096.stdout: rbd-fuse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 82 k 2024-09-13T14:20:05.602 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:20:05.602 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:20:05.602 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:20:05.602 INFO:teuthology.orchestra.run.smithi096.stdout:Install 1 Package 2024-09-13T14:20:05.602 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:20:05.602 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 82 k 2024-09-13T14:20:05.603 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 222 k 2024-09-13T14:20:05.603 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:20:05.796 INFO:teuthology.orchestra.run.smithi096.stdout:rbd-fuse-17.2.7-1627.gbbdadfa3.el9.x86_64.rpm 422 kB/s | 82 kB 00:00 2024-09-13T14:20:05.797 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:20:05.797 INFO:teuthology.orchestra.run.smithi096.stdout:Total 419 kB/s | 82 kB 00:00 2024-09-13T14:20:05.797 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:20:05.804 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:20:05.804 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:20:05.837 INFO:teuthology.orchestra.run.smithi204.stdout:(2/2): fuse-2.9.9-16.el9.x86_64.rpm 113 kB/s | 80 kB 00:00 2024-09-13T14:20:05.838 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:20:05.839 INFO:teuthology.orchestra.run.smithi204.stdout:Total 1.2 MB/s | 893 kB 00:00 2024-09-13T14:20:05.851 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:20:05.867 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:20:05.867 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:20:05.875 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:20:05.876 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:20:05.958 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:20:05.958 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:20:06.015 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:20:06.134 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:06.145 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:20:06.282 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-13T14:20:06.354 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/2 2024-09-13T14:20:06.576 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:06.853 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:06.853 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:20:06.853 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:20:06.853 INFO:teuthology.orchestra.run.smithi096.stdout: rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:20:06.853 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:20:06.854 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:20:07.021 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/2 2024-09-13T14:20:07.022 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/2 2024-09-13T14:20:07.081 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install rbd-mirror 2024-09-13T14:20:07.285 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-13T14:20:07.286 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:07.286 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:20:07.286 INFO:teuthology.orchestra.run.smithi204.stdout: ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-13T14:20:07.286 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:07.286 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:20:07.577 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install librados-devel 2024-09-13T14:20:07.657 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:38 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:20:07.794 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:20:07.794 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:20:07.794 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repo Size 2024-09-13T14:20:07.794 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:20:07.794 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:20:07.794 INFO:teuthology.orchestra.run.smithi096.stdout: rbd-mirror x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.9 M 2024-09-13T14:20:07.794 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:20:07.795 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:20:07.795 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:20:07.795 INFO:teuthology.orchestra.run.smithi096.stdout:Install 1 Package 2024-09-13T14:20:07.795 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:20:07.795 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 2.9 M 2024-09-13T14:20:07.795 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 12 M 2024-09-13T14:20:07.795 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:20:08.158 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:45 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:20:08.184 INFO:teuthology.orchestra.run.smithi096.stdout:rbd-mirror-17.2.7-1627.gbbdadfa3.el9.x86_64.rpm 7.6 MB/s | 2.9 MB 00:00 2024-09-13T14:20:08.185 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:20:08.185 INFO:teuthology.orchestra.run.smithi096.stdout:Total 7.5 MB/s | 2.9 MB 00:00 2024-09-13T14:20:08.186 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:20:08.194 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:20:08.194 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:20:08.274 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:20:08.275 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:20:08.294 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:20:08.294 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:08.295 INFO:teuthology.orchestra.run.smithi204.stdout: Package Arch Version Repo Size 2024-09-13T14:20:08.295 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:08.295 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:20:08.295 INFO:teuthology.orchestra.run.smithi204.stdout: librados-devel x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 126 k 2024-09-13T14:20:08.295 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:08.295 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:20:08.295 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:08.295 INFO:teuthology.orchestra.run.smithi204.stdout:Install 1 Package 2024-09-13T14:20:08.295 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:08.296 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 126 k 2024-09-13T14:20:08.296 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 448 k 2024-09-13T14:20:08.296 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:20:08.502 INFO:teuthology.orchestra.run.smithi204.stdout:librados-devel-17.2.7-1627.gbbdadfa3.el9.x86_64 614 kB/s | 126 kB 00:00 2024-09-13T14:20:08.502 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:20:08.503 INFO:teuthology.orchestra.run.smithi204.stdout:Total 609 kB/s | 126 kB 00:00 2024-09-13T14:20:08.503 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:20:08.508 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:20:08.508 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:20:08.575 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:20:08.575 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:20:08.692 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:20:08.735 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:20:08.777 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:08.802 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:08.802 INFO:teuthology.orchestra.run.smithi096.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:20:08.802 INFO:teuthology.orchestra.run.smithi096.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-13T14:20:08.802 INFO:teuthology.orchestra.run.smithi096.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:20:08.802 INFO:teuthology.orchestra.run.smithi096.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:20:08.802 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:20:08.874 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:09.279 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:09.528 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:09.528 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:09.528 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:20:09.528 INFO:teuthology.orchestra.run.smithi204.stdout: librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:20:09.528 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:09.528 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:20:09.804 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install libcephfs2 2024-09-13T14:20:09.936 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:09.936 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:20:09.936 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:20:09.936 INFO:teuthology.orchestra.run.smithi096.stdout: rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:20:09.936 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:20:09.936 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:20:10.154 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install rbd-nbd 2024-09-13T14:20:10.385 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:47 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:20:10.478 INFO:teuthology.orchestra.run.smithi204.stdout:Package libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:20:10.526 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:20:10.527 INFO:teuthology.orchestra.run.smithi204.stdout:Nothing to do. 2024-09-13T14:20:10.527 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:20:10.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install libcephfs-devel 2024-09-13T14:20:10.734 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:01:41 ago on Fri 13 Sep 2024 02:18:29 PM UTC. 2024-09-13T14:20:10.870 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-13T14:20:10.871 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:20:10.871 INFO:teuthology.orchestra.run.smithi096.stdout: Package Architecture Version Repository Size 2024-09-13T14:20:10.871 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:20:10.871 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-13T14:20:10.871 INFO:teuthology.orchestra.run.smithi096.stdout: rbd-nbd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 165 k 2024-09-13T14:20:10.871 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:20:10.871 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-13T14:20:10.871 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-13T14:20:10.871 INFO:teuthology.orchestra.run.smithi096.stdout:Install 1 Package 2024-09-13T14:20:10.871 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:20:10.871 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 165 k 2024-09-13T14:20:10.871 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 477 k 2024-09-13T14:20:10.871 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-13T14:20:11.096 INFO:teuthology.orchestra.run.smithi096.stdout:rbd-nbd-17.2.7-1627.gbbdadfa3.el9.x86_64.rpm 734 kB/s | 165 kB 00:00 2024-09-13T14:20:11.097 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:20:11.098 INFO:teuthology.orchestra.run.smithi096.stdout:Total 729 kB/s | 165 kB 00:00 2024-09-13T14:20:11.098 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-13T14:20:11.105 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-13T14:20:11.105 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-13T14:20:11.156 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:48 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:20:11.168 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-13T14:20:11.169 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-13T14:20:11.289 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:20:11.293 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:11.293 INFO:teuthology.orchestra.run.smithi204.stdout: Package Arch Version Repo Size 2024-09-13T14:20:11.293 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:11.293 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:20:11.293 INFO:teuthology.orchestra.run.smithi204.stdout: libcephfs-devel x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 25 k 2024-09-13T14:20:11.293 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:11.294 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:20:11.294 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:11.294 INFO:teuthology.orchestra.run.smithi204.stdout:Install 1 Package 2024-09-13T14:20:11.294 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:11.294 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 25 k 2024-09-13T14:20:11.294 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 110 k 2024-09-13T14:20:11.294 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:20:11.335 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-13T14:20:11.440 INFO:teuthology.orchestra.run.smithi204.stdout:libcephfs-devel-17.2.7-1627.gbbdadfa3.el9.x86_6 168 kB/s | 25 kB 00:00 2024-09-13T14:20:11.440 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:20:11.440 INFO:teuthology.orchestra.run.smithi204.stdout:Total 166 kB/s | 25 kB 00:00 2024-09-13T14:20:11.441 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:20:11.444 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:20:11.444 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:20:11.446 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:20:11.446 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:20:11.454 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:11.529 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:20:11.636 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:11.886 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:11.992 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:12.159 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:12.159 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:20:12.159 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-13T14:20:12.159 INFO:teuthology.orchestra.run.smithi096.stdout: rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:20:12.159 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:20:12.159 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-13T14:20:12.225 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:12.225 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:12.225 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:20:12.225 INFO:teuthology.orchestra.run.smithi204.stdout: libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:20:12.225 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:12.225 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:20:12.395 DEBUG:teuthology.parallel:result is None 2024-09-13T14:20:12.412 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install librados2 2024-09-13T14:20:12.979 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:49 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:20:13.071 INFO:teuthology.orchestra.run.smithi204.stdout:Package librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:20:13.114 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:20:13.115 INFO:teuthology.orchestra.run.smithi204.stdout:Nothing to do. 2024-09-13T14:20:13.115 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:20:13.185 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install librbd1 2024-09-13T14:20:13.758 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:50 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:20:13.850 INFO:teuthology.orchestra.run.smithi204.stdout:Package librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:20:13.893 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:20:13.894 INFO:teuthology.orchestra.run.smithi204.stdout:Nothing to do. 2024-09-13T14:20:13.894 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:20:13.958 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install python3-rados 2024-09-13T14:20:14.524 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:51 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:20:14.614 INFO:teuthology.orchestra.run.smithi204.stdout:Package python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:20:14.657 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:20:14.658 INFO:teuthology.orchestra.run.smithi204.stdout:Nothing to do. 2024-09-13T14:20:14.659 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:20:14.727 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install python3-rgw 2024-09-13T14:20:15.292 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:52 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:20:15.384 INFO:teuthology.orchestra.run.smithi204.stdout:Package python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:20:15.427 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:20:15.428 INFO:teuthology.orchestra.run.smithi204.stdout:Nothing to do. 2024-09-13T14:20:15.429 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:20:15.502 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install python3-cephfs 2024-09-13T14:20:16.064 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:53 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:20:16.153 INFO:teuthology.orchestra.run.smithi204.stdout:Package python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:20:16.196 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:20:16.197 INFO:teuthology.orchestra.run.smithi204.stdout:Nothing to do. 2024-09-13T14:20:16.197 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:20:16.271 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install python3-rbd 2024-09-13T14:20:16.832 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:53 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:20:16.922 INFO:teuthology.orchestra.run.smithi204.stdout:Package python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:20:16.965 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:20:16.966 INFO:teuthology.orchestra.run.smithi204.stdout:Nothing to do. 2024-09-13T14:20:16.966 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:20:17.034 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install rbd-fuse 2024-09-13T14:20:17.602 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:54 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:20:17.736 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:20:17.737 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:17.737 INFO:teuthology.orchestra.run.smithi204.stdout: Package Architecture Version Repository Size 2024-09-13T14:20:17.737 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:17.737 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:20:17.737 INFO:teuthology.orchestra.run.smithi204.stdout: rbd-fuse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 82 k 2024-09-13T14:20:17.738 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:17.738 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:20:17.738 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:17.738 INFO:teuthology.orchestra.run.smithi204.stdout:Install 1 Package 2024-09-13T14:20:17.738 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:17.738 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 82 k 2024-09-13T14:20:17.738 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 222 k 2024-09-13T14:20:17.738 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:20:17.944 INFO:teuthology.orchestra.run.smithi204.stdout:rbd-fuse-17.2.7-1627.gbbdadfa3.el9.x86_64.rpm 398 kB/s | 82 kB 00:00 2024-09-13T14:20:17.944 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:20:17.944 INFO:teuthology.orchestra.run.smithi204.stdout:Total 396 kB/s | 82 kB 00:00 2024-09-13T14:20:17.945 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:20:17.951 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:20:17.952 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:20:18.015 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:20:18.015 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:20:18.160 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:20:18.265 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:18.634 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:18.901 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:18.901 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:18.901 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:20:18.901 INFO:teuthology.orchestra.run.smithi204.stdout: rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:20:18.901 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:18.902 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:20:19.170 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install rbd-mirror 2024-09-13T14:20:19.749 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:56 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:20:19.887 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:20:19.888 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:19.888 INFO:teuthology.orchestra.run.smithi204.stdout: Package Arch Version Repo Size 2024-09-13T14:20:19.888 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:19.888 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:20:19.888 INFO:teuthology.orchestra.run.smithi204.stdout: rbd-mirror x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.9 M 2024-09-13T14:20:19.888 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:19.888 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:20:19.888 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:19.888 INFO:teuthology.orchestra.run.smithi204.stdout:Install 1 Package 2024-09-13T14:20:19.888 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:19.888 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 2.9 M 2024-09-13T14:20:19.888 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 12 M 2024-09-13T14:20:19.888 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:20:20.282 INFO:teuthology.orchestra.run.smithi204.stdout:rbd-mirror-17.2.7-1627.gbbdadfa3.el9.x86_64.rpm 7.5 MB/s | 2.9 MB 00:00 2024-09-13T14:20:20.282 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:20:20.283 INFO:teuthology.orchestra.run.smithi204.stdout:Total 7.4 MB/s | 2.9 MB 00:00 2024-09-13T14:20:20.283 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:20:20.292 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:20:20.292 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:20:20.371 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:20:20.371 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:20:20.776 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:20:20.849 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:20.874 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:20.874 INFO:teuthology.orchestra.run.smithi204.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:20:20.874 INFO:teuthology.orchestra.run.smithi204.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-13T14:20:20.874 INFO:teuthology.orchestra.run.smithi204.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:20:20.874 INFO:teuthology.orchestra.run.smithi204.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:20:20.875 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:22.086 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:22.086 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:22.086 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:20:22.086 INFO:teuthology.orchestra.run.smithi204.stdout: rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:20:22.086 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:22.086 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:20:22.330 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum -y install rbd-nbd 2024-09-13T14:20:22.910 INFO:teuthology.orchestra.run.smithi204.stdout:Last metadata expiration check: 0:01:59 ago on Fri 13 Sep 2024 02:18:23 PM UTC. 2024-09-13T14:20:23.047 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-09-13T14:20:23.047 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:23.047 INFO:teuthology.orchestra.run.smithi204.stdout: Package Architecture Version Repository Size 2024-09-13T14:20:23.048 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:23.048 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-09-13T14:20:23.048 INFO:teuthology.orchestra.run.smithi204.stdout: rbd-nbd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 165 k 2024-09-13T14:20:23.048 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:23.048 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-09-13T14:20:23.048 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-09-13T14:20:23.048 INFO:teuthology.orchestra.run.smithi204.stdout:Install 1 Package 2024-09-13T14:20:23.048 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:23.048 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 165 k 2024-09-13T14:20:23.048 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 477 k 2024-09-13T14:20:23.048 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-09-13T14:20:23.263 INFO:teuthology.orchestra.run.smithi204.stdout:rbd-nbd-17.2.7-1627.gbbdadfa3.el9.x86_64.rpm 772 kB/s | 165 kB 00:00 2024-09-13T14:20:23.263 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:20:23.264 INFO:teuthology.orchestra.run.smithi204.stdout:Total 765 kB/s | 165 kB 00:00 2024-09-13T14:20:23.264 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-09-13T14:20:23.272 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-09-13T14:20:23.272 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-09-13T14:20:23.335 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-09-13T14:20:23.335 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-09-13T14:20:23.494 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-09-13T14:20:23.592 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:24.018 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:24.273 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:20:24.274 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:24.274 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-09-13T14:20:24.274 INFO:teuthology.orchestra.run.smithi204.stdout: rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:20:24.274 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:20:24.274 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-09-13T14:20:24.526 DEBUG:teuthology.parallel:result is None 2024-09-13T14:20:24.527 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:20:24.527 INFO:teuthology.packaging:ref: None 2024-09-13T14:20:24.527 INFO:teuthology.packaging:tag: None 2024-09-13T14:20:24.527 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:20:24.527 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:20:24.527 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-13T14:20:24.743 DEBUG:teuthology.orchestra.run.smithi096:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-13T14:20:24.768 INFO:teuthology.orchestra.run.smithi096.stdout:17.2.7-1627.gbbdadfa3.el9 2024-09-13T14:20:24.769 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1627.gbbdadfa3.el9 2024-09-13T14:20:24.769 INFO:teuthology.task.install:The correct ceph version 17.2.7-1627.gbbdadfa3 is installed. 2024-09-13T14:20:24.771 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:20:24.771 INFO:teuthology.packaging:ref: None 2024-09-13T14:20:24.771 INFO:teuthology.packaging:tag: None 2024-09-13T14:20:24.771 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:20:24.771 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:20:24.771 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-13T14:20:24.978 DEBUG:teuthology.orchestra.run.smithi204:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-13T14:20:25.001 INFO:teuthology.orchestra.run.smithi204.stdout:17.2.7-1627.gbbdadfa3.el9 2024-09-13T14:20:25.001 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1627.gbbdadfa3.el9 2024-09-13T14:20:25.001 INFO:teuthology.task.install:The correct ceph version 17.2.7-1627.gbbdadfa3 is installed. 2024-09-13T14:20:25.003 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-13T14:20:25.003 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:20:25.003 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-13T14:20:25.034 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-09-13T14:20:25.034 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-13T14:20:25.069 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-13T14:20:25.069 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:20:25.069 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/usr/bin/daemon-helper 2024-09-13T14:20:25.103 DEBUG:teuthology.orchestra.run.smithi096:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-13T14:20:25.171 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-09-13T14:20:25.171 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd of=/usr/bin/daemon-helper 2024-09-13T14:20:25.199 DEBUG:teuthology.orchestra.run.smithi204:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-13T14:20:25.264 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-13T14:20:25.264 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:20:25.264 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-13T14:20:25.313 DEBUG:teuthology.orchestra.run.smithi096:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-13T14:20:25.380 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-09-13T14:20:25.380 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-13T14:20:25.407 DEBUG:teuthology.orchestra.run.smithi204:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-13T14:20:25.472 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-13T14:20:25.472 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:20:25.472 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/usr/bin/stdin-killer 2024-09-13T14:20:25.497 DEBUG:teuthology.orchestra.run.smithi096:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-13T14:20:25.562 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-09-13T14:20:25.562 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd of=/usr/bin/stdin-killer 2024-09-13T14:20:25.588 DEBUG:teuthology.orchestra.run.smithi204:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-13T14:20:25.653 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-13T14:20:25.751 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'PG_AVAILABILITY'], 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'} 2024-09-13T14:20:25.751 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-09-13T14:20:25.751 INFO:tasks.cephadm:Cluster fsid is 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:20:25.751 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-13T14:20:25.752 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.96', 'mon.c': '[v2:172.21.15.96:3301,v1:172.21.15.96:6790]', 'mon.b': '172.21.15.204'} 2024-09-13T14:20:25.752 INFO:tasks.cephadm:First mon is mon.a on smithi096 2024-09-13T14:20:25.752 INFO:tasks.cephadm:First mgr is y 2024-09-13T14:20:25.752 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-13T14:20:25.752 DEBUG:teuthology.orchestra.run.smithi096:> sudo hostname $(hostname -s) 2024-09-13T14:20:25.778 DEBUG:teuthology.orchestra.run.smithi204:> sudo hostname $(hostname -s) 2024-09-13T14:20:25.803 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-09-13T14:20:25.803 DEBUG:teuthology.orchestra.run.smithi096:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T14:20:25.951 INFO:teuthology.orchestra.run.smithi096.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 13 14:20 /home/ubuntu/cephtest/cephadm 2024-09-13T14:20:25.951 DEBUG:teuthology.orchestra.run.smithi204:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T14:20:26.094 INFO:teuthology.orchestra.run.smithi204.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 13 14:20 /home/ubuntu/cephtest/cephadm 2024-09-13T14:20:26.094 DEBUG:teuthology.orchestra.run.smithi096:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T14:20:26.110 DEBUG:teuthology.orchestra.run.smithi204:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T14:20:26.136 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-09-13T14:20:26.136 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-13T14:20:26.152 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-13T14:20:27.676 INFO:teuthology.orchestra.run.smithi096.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T14:20:27.795 INFO:teuthology.orchestra.run.smithi204.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T14:21:02.342 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:21:02.342 INFO:teuthology.orchestra.run.smithi096.stdout: "ceph_version": "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)", 2024-09-13T14:21:02.342 INFO:teuthology.orchestra.run.smithi096.stdout: "image_id": "a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c", 2024-09-13T14:21:02.342 INFO:teuthology.orchestra.run.smithi096.stdout: "repo_digests": [ 2024-09-13T14:21:02.343 INFO:teuthology.orchestra.run.smithi096.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be" 2024-09-13T14:21:02.343 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:21:02.343 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:21:07.364 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:21:07.364 INFO:teuthology.orchestra.run.smithi204.stdout: "ceph_version": "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)", 2024-09-13T14:21:07.364 INFO:teuthology.orchestra.run.smithi204.stdout: "image_id": "a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c", 2024-09-13T14:21:07.364 INFO:teuthology.orchestra.run.smithi204.stdout: "repo_digests": [ 2024-09-13T14:21:07.364 INFO:teuthology.orchestra.run.smithi204.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be" 2024-09-13T14:21:07.364 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:21:07.364 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:21:07.383 DEBUG:teuthology.orchestra.run.smithi096:> sudo mkdir -p /etc/ceph 2024-09-13T14:21:07.417 DEBUG:teuthology.orchestra.run.smithi204:> sudo mkdir -p /etc/ceph 2024-09-13T14:21:07.444 DEBUG:teuthology.orchestra.run.smithi096:> sudo chmod 777 /etc/ceph 2024-09-13T14:21:07.485 DEBUG:teuthology.orchestra.run.smithi204:> sudo chmod 777 /etc/ceph 2024-09-13T14:21:07.511 INFO:tasks.cephadm:Writing seed config... 2024-09-13T14:21:07.512 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-13T14:21:07.513 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-13T14:21:07.513 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-13T14:21:07.513 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-13T14:21:07.513 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-13T14:21:07.513 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-09-13T14:21:07.513 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-13T14:21:07.513 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-13T14:21:07.513 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-13T14:21:07.513 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-13T14:21:07.513 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-13T14:21:07.514 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:21:07.514 DEBUG:teuthology.orchestra.run.smithi096:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-13T14:21:07.543 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 51e50ed6-71db-11ef-bceb-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-13T14:21:07.544 DEBUG:teuthology.orchestra.run.smithi096:mon.a> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.a.service 2024-09-13T14:21:07.587 DEBUG:teuthology.orchestra.run.smithi096:mgr.y> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.y.service 2024-09-13T14:21:07.629 INFO:tasks.cephadm:Bootstrapping... 2024-09-13T14:21:07.630 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.96 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:21:07.830 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:21:07.830 INFO:teuthology.orchestra.run.smithi096.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '51e50ed6-71db-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.96', '--skip-admin-label'] 2024-09-13T14:21:07.854 INFO:teuthology.orchestra.run.smithi096.stdout:/bin/podman: stdout 5.2.2 2024-09-13T14:21:07.854 INFO:teuthology.orchestra.run.smithi096.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-13T14:21:07.854 INFO:teuthology.orchestra.run.smithi096.stdout:Verifying podman|docker is present... 2024-09-13T14:21:07.876 INFO:teuthology.orchestra.run.smithi096.stdout:/bin/podman: stdout 5.2.2 2024-09-13T14:21:07.877 INFO:teuthology.orchestra.run.smithi096.stdout:Verifying lvm2 is present... 2024-09-13T14:21:07.877 INFO:teuthology.orchestra.run.smithi096.stdout:Verifying time synchronization is in place... 2024-09-13T14:21:07.885 INFO:teuthology.orchestra.run.smithi096.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-13T14:21:07.885 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T14:21:07.891 INFO:teuthology.orchestra.run.smithi096.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-13T14:21:07.891 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stdout inactive 2024-09-13T14:21:07.898 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stdout enabled 2024-09-13T14:21:07.905 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stdout active 2024-09-13T14:21:07.905 INFO:teuthology.orchestra.run.smithi096.stdout:Unit chronyd.service is enabled and running 2024-09-13T14:21:07.905 INFO:teuthology.orchestra.run.smithi096.stdout:Repeating the final host check... 2024-09-13T14:21:07.926 INFO:teuthology.orchestra.run.smithi096.stdout:/bin/podman: stdout 5.2.2 2024-09-13T14:21:07.926 INFO:teuthology.orchestra.run.smithi096.stdout:podman (/bin/podman) version 5.2.2 is present 2024-09-13T14:21:07.927 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl is present 2024-09-13T14:21:07.927 INFO:teuthology.orchestra.run.smithi096.stdout:lvcreate is present 2024-09-13T14:21:07.933 INFO:teuthology.orchestra.run.smithi096.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-13T14:21:07.934 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T14:21:07.940 INFO:teuthology.orchestra.run.smithi096.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-13T14:21:07.940 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stdout inactive 2024-09-13T14:21:07.948 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stdout enabled 2024-09-13T14:21:07.954 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stdout active 2024-09-13T14:21:07.954 INFO:teuthology.orchestra.run.smithi096.stdout:Unit chronyd.service is enabled and running 2024-09-13T14:21:07.954 INFO:teuthology.orchestra.run.smithi096.stdout:Host looks OK 2024-09-13T14:21:07.955 INFO:teuthology.orchestra.run.smithi096.stdout:Cluster fsid: 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:21:07.955 INFO:teuthology.orchestra.run.smithi096.stdout:Acquiring lock 139727473697696 on /run/cephadm/51e50ed6-71db-11ef-bceb-c7b262605968.lock 2024-09-13T14:21:07.955 INFO:teuthology.orchestra.run.smithi096.stdout:Lock 139727473697696 acquired on /run/cephadm/51e50ed6-71db-11ef-bceb-c7b262605968.lock 2024-09-13T14:21:07.955 INFO:teuthology.orchestra.run.smithi096.stdout:Verifying IP 172.21.15.96 port 3300 ... 2024-09-13T14:21:07.955 INFO:teuthology.orchestra.run.smithi096.stdout:Verifying IP 172.21.15.96 port 6789 ... 2024-09-13T14:21:07.955 INFO:teuthology.orchestra.run.smithi096.stdout:Base mon IP(s) is [172.21.15.96:3300, 172.21.15.96:6789], mon addrv is [v2:172.21.15.96:3300,v1:172.21.15.96:6789] 2024-09-13T14:21:07.959 INFO:teuthology.orchestra.run.smithi096.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.96 metric 100 2024-09-13T14:21:07.959 INFO:teuthology.orchestra.run.smithi096.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.96 metric 100 2024-09-13T14:21:07.962 INFO:teuthology.orchestra.run.smithi096.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-13T14:21:07.962 INFO:teuthology.orchestra.run.smithi096.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-13T14:21:07.962 INFO:teuthology.orchestra.run.smithi096.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-13T14:21:07.964 INFO:teuthology.orchestra.run.smithi096.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-13T14:21:07.964 INFO:teuthology.orchestra.run.smithi096.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-13T14:21:07.964 INFO:teuthology.orchestra.run.smithi096.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-13T14:21:07.965 INFO:teuthology.orchestra.run.smithi096.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-13T14:21:07.965 INFO:teuthology.orchestra.run.smithi096.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe8f:cc0d/64 scope link noprefixroute 2024-09-13T14:21:07.965 INFO:teuthology.orchestra.run.smithi096.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-13T14:21:07.965 INFO:teuthology.orchestra.run.smithi096.stdout:Mon IP `172.21.15.96` is in CIDR network `172.21.0.0/20` 2024-09-13T14:21:07.965 INFO:teuthology.orchestra.run.smithi096.stdout:Mon IP `172.21.15.96` is in CIDR network `172.21.0.0/20` 2024-09-13T14:21:07.966 INFO:teuthology.orchestra.run.smithi096.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-09-13T14:21:07.966 INFO:teuthology.orchestra.run.smithi096.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-13T14:21:07.966 INFO:teuthology.orchestra.run.smithi096.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T14:21:08.525 INFO:teuthology.orchestra.run.smithi096.stdout:/bin/podman: stdout a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c 2024-09-13T14:21:08.526 INFO:teuthology.orchestra.run.smithi096.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T14:21:08.526 INFO:teuthology.orchestra.run.smithi096.stdout:/bin/podman: stderr Getting image source signatures 2024-09-13T14:21:08.526 INFO:teuthology.orchestra.run.smithi096.stdout:/bin/podman: stderr Copying blob sha256:ca308b3f556da579f99cd79db684ebbf678db4c29a649b66aaf6f99bc8124cd6 2024-09-13T14:21:08.526 INFO:teuthology.orchestra.run.smithi096.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-09-13T14:21:08.526 INFO:teuthology.orchestra.run.smithi096.stdout:/bin/podman: stderr Copying config sha256:a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c 2024-09-13T14:21:08.527 INFO:teuthology.orchestra.run.smithi096.stdout:/bin/podman: stderr Writing manifest to image destination 2024-09-13T14:21:09.371 INFO:teuthology.orchestra.run.smithi096.stdout:ceph: stdout ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable) 2024-09-13T14:21:09.371 INFO:teuthology.orchestra.run.smithi096.stdout:Ceph version: ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable) 2024-09-13T14:21:09.371 INFO:teuthology.orchestra.run.smithi096.stdout:Extracting ceph user uid/gid from container image... 2024-09-13T14:21:10.309 INFO:teuthology.orchestra.run.smithi096.stdout:stat: stdout 167 167 2024-09-13T14:21:10.310 INFO:teuthology.orchestra.run.smithi096.stdout:Creating initial keys... 2024-09-13T14:21:11.215 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph-authtool: stdout AQBWSuRmc6oRJhAAQIizRh8SM4T72ruhU9UBWA== 2024-09-13T14:21:12.213 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph-authtool: stdout AQBXSuRm3JLyHhAAQDT4gvYG4ZduItTi1ofH2Q== 2024-09-13T14:21:13.127 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph-authtool: stdout AQBYSuRmjE3cHRAAJ0IDXbr5cZTWXoz/DBQRNQ== 2024-09-13T14:21:13.128 INFO:teuthology.orchestra.run.smithi096.stdout:Creating initial monmap... 2024-09-13T14:21:14.084 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T14:21:14.084 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-09-13T14:21:14.085 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:21:14.085 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T14:21:14.085 INFO:teuthology.orchestra.run.smithi096.stdout:monmaptool for a [v2:172.21.15.96:3300,v1:172.21.15.96:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T14:21:14.085 INFO:teuthology.orchestra.run.smithi096.stdout:setting min_mon_release = octopus 2024-09-13T14:21:14.085 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/monmaptool: set fsid to 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:21:14.085 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T14:21:14.085 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:21:14.085 INFO:teuthology.orchestra.run.smithi096.stdout:Creating mon... 2024-09-13T14:21:15.022 INFO:teuthology.orchestra.run.smithi096.stdout:create mon.a on 2024-09-13T14:21:15.223 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-09-13T14:21:15.386 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-13T14:21:15.539 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-51e50ed6-71db-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-51e50ed6-71db-11ef-bceb-c7b262605968.target. 2024-09-13T14:21:15.539 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-51e50ed6-71db-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-51e50ed6-71db-11ef-bceb-c7b262605968.target. 2024-09-13T14:21:15.734 INFO:teuthology.orchestra.run.smithi096.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.a 2024-09-13T14:21:15.734 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stderr Failed to reset failed state of unit ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.a.service: Unit ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.a.service not loaded. 2024-09-13T14:21:15.911 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-51e50ed6-71db-11ef-bceb-c7b262605968.target.wants/ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.a.service → /etc/systemd/system/ceph-51e50ed6-71db-11ef-bceb-c7b262605968@.service. 2024-09-13T14:21:16.447 INFO:teuthology.orchestra.run.smithi096.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:21:16.447 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stdout disabled 2024-09-13T14:21:16.454 INFO:teuthology.orchestra.run.smithi096.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:21:16.454 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stdout inactive 2024-09-13T14:21:16.454 INFO:teuthology.orchestra.run.smithi096.stdout:firewalld.service is not enabled 2024-09-13T14:21:16.454 INFO:teuthology.orchestra.run.smithi096.stdout:Not possible to enable service . firewalld.service is not available 2024-09-13T14:21:16.455 INFO:teuthology.orchestra.run.smithi096.stdout:Waiting for mon to start... 2024-09-13T14:21:16.455 INFO:teuthology.orchestra.run.smithi096.stdout:Waiting for mon... 2024-09-13T14:21:16.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:16 smithi096 systemd[1]: Started Ceph mon.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:21:17.506 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:17 smithi096 ceph-mon[28858]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T14:21:17.506 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:17 smithi096 ceph-mon[28858]: monmap e1: 1 mons at {a=[v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:21:17.506 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:17 smithi096 ceph-mon[28858]: fsmap 2024-09-13T14:21:17.506 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:17 smithi096 ceph-mon[28858]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T14:21:17.506 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:17 smithi096 ceph-mon[28858]: mgrmap e1: no daemons active 2024-09-13T14:21:17.506 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:17 smithi096 ceph-mon[28858]: from='client.? 172.21.15.96:0/1061770903' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T14:21:17.509 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout cluster: 2024-09-13T14:21:17.509 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout id: 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:21:17.510 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-13T14:21:17.510 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 2024-09-13T14:21:17.510 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout services: 2024-09-13T14:21:17.510 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.597815s) 2024-09-13T14:21:17.510 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-13T14:21:17.510 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-13T14:21:17.510 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 2024-09-13T14:21:17.510 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout data: 2024-09-13T14:21:17.511 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-13T14:21:17.511 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-13T14:21:17.511 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-13T14:21:17.511 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout pgs: 2024-09-13T14:21:17.511 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 2024-09-13T14:21:17.511 INFO:teuthology.orchestra.run.smithi096.stdout:mon is available 2024-09-13T14:21:17.511 INFO:teuthology.orchestra.run.smithi096.stdout:Assimilating anything we can from ceph.conf... 2024-09-13T14:21:18.548 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 2024-09-13T14:21:18.548 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout [global] 2024-09-13T14:21:18.548 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout fsid = 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:21:18.548 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-13T14:21:18.549 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.96:3300,v1:172.21.15.96:6789] 2024-09-13T14:21:18.549 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-13T14:21:18.549 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-13T14:21:18.549 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-13T14:21:18.549 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-13T14:21:18.549 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 2024-09-13T14:21:18.549 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-13T14:21:18.549 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-13T14:21:18.549 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 2024-09-13T14:21:18.549 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout [osd] 2024-09-13T14:21:18.549 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-13T14:21:18.549 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-13T14:21:18.549 INFO:teuthology.orchestra.run.smithi096.stdout:Generating new minimal ceph.conf... 2024-09-13T14:21:18.653 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:18 smithi096 ceph-mon[28858]: from='client.? 172.21.15.96:0/1624009148' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T14:21:18.654 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:18 smithi096 ceph-mon[28858]: from='client.? 172.21.15.96:0/1624009148' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-13T14:21:19.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:19 smithi096 ceph-mon[28858]: from='client.? 172.21.15.96:0/1336761780' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:19.600 INFO:teuthology.orchestra.run.smithi096.stdout:Restarting the monitor... 2024-09-13T14:21:19.879 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:19 smithi096 systemd[1]: Stopping Ceph mon.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:21:20.254 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:19 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a[28854]: 2024-09-13T14:21:19.878+0000 7f84c7d1e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:21:20.255 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:19 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a[28854]: 2024-09-13T14:21:19.878+0000 7f84c7d1e640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-13T14:21:20.255 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:20 smithi096 podman[29103]: 2024-09-13 14:21:20.062592142 +0000 UTC m=+0.298753495 container died 0540445e6ebbc7e87ad3f20c06c98dcc8db792f152bb639dad3ff25701d0d1a8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-13T14:21:20.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:20 smithi096 podman[29103]: 2024-09-13 14:21:20.319687519 +0000 UTC m=+0.555848869 container cleanup 0540445e6ebbc7e87ad3f20c06c98dcc8db792f152bb639dad3ff25701d0d1a8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:21:20.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:20 smithi096 bash[29103]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a 2024-09-13T14:21:20.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:20 smithi096 podman[29116]: 2024-09-13 14:21:20.495523405 +0000 UTC m=+0.434020680 container remove 0540445e6ebbc7e87ad3f20c06c98dcc8db792f152bb639dad3ff25701d0d1a8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-13T14:21:21.051 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:20 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T14:21:21.051 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:20 smithi096 systemd[1]: Stopped Ceph mon.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:21:21.051 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:20 smithi096 systemd[1]: Starting Ceph mon.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:21:21.051 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:20 smithi096 podman[29209]: 2024-09-13 14:21:20.8373031 +0000 UTC m=+0.015426720 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph:quincy 2024-09-13T14:21:21.051 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:20 smithi096 podman[29209]: 2024-09-13 14:21:20.937799015 +0000 UTC m=+0.115922643 container create 96e95ed67f926384baf6702dc5fbb3e4686072ff9aa3c2f5ebce42533fb138d1 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-13T14:21:21.243 INFO:teuthology.orchestra.run.smithi096.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-09-13T14:21:21.469 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 podman[29209]: 2024-09-13 14:21:21.156730488 +0000 UTC m=+0.334854113 container init 96e95ed67f926384baf6702dc5fbb3e4686072ff9aa3c2f5ebce42533fb138d1 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, GIT_BRANCH=HEAD) 2024-09-13T14:21:21.469 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 podman[29209]: 2024-09-13 14:21:21.159773562 +0000 UTC m=+0.337897182 container start 96e95ed67f926384baf6702dc5fbb3e4686072ff9aa3c2f5ebce42533fb138d1 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a, org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-13T14:21:21.469 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:21:21.469 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 2 2024-09-13T14:21:21.469 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: pidfile_write: ignore empty --pid-file 2024-09-13T14:21:21.469 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: load: jerasure load: lrc 2024-09-13T14:21:21.469 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: RocksDB version: 6.15.5 2024-09-13T14:21:21.469 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T14:21:21.469 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Compile date Sep 12 2024 2024-09-13T14:21:21.470 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: DB SUMMARY 2024-09-13T14:21:21.470 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: DB Session ID: D42AABV3G5MJ8BB13ET9 2024-09-13T14:21:21.470 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:21:21.470 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:21:21.470 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-13T14:21:21.470 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-13T14:21:21.470 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83688 ; 2024-09-13T14:21:21.470 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:21:21.470 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:21:21.470 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:21:21.470 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:21:21.470 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.env: 0x5615930f8440 2024-09-13T14:21:21.470 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.fs: Posix File System 2024-09-13T14:21:21.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.info_log: 0x561594a70ec0 2024-09-13T14:21:21.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:21:21.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.statistics: (nil) 2024-09-13T14:21:21.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.use_fsync: 0 2024-09-13T14:21:21.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:21:21.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:21:21.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:21:21.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:21:21.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:21:21.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:21:21.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:21:21.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:21:21.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:21:21.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:21:21.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:21:21.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.db_log_dir: 2024-09-13T14:21:21.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-13T14:21:21.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:21:21.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:21:21.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:21:21.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:21:21.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:21:21.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:21:21.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:21:21.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:21:21.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.write_buffer_manager: 0x561594b2e7e0 2024-09-13T14:21:21.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:21:21.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T14:21:21.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:21:21.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:21:21.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:21:21.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:21:21.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:21:21.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:21:21.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:21:21.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.unordered_write: 0 2024-09-13T14:21:21.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:21:21.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:21:21.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:21:21.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:21:21.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.row_cache: None 2024-09-13T14:21:21.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.wal_filter: None 2024-09-13T14:21:21.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:21:21.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:21:21.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.preserve_deletes: 0 2024-09-13T14:21:21.475 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:21:21.475 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:21:21.475 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:21:21.475 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:21:21.475 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:21:21.475 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:21:21.475 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:21:21.475 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:21:21.475 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:21:21.475 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:21:21.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:21:21.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:21:21.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:21:21.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:21:21.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:21:21.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:21:21.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:21:21.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:21:21.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:21:21.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:21:21.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:21:21.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:21:21.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:21:21.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:21:21.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_open_files: -1 2024-09-13T14:21:21.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:21:21.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:21:21.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:21:21.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:21:21.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:21:21.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Compression algorithms supported: 2024-09-13T14:21:21.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: kZSTD supported: 0 2024-09-13T14:21:21.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:21:21.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:21:21.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:21:21.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:21:21.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:21:21.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:21:21.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:21:21.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:21:21.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-09-13T14:21:21.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T14:21:21.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:21:21.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.merge_operator: 2024-09-13T14:21:21.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compaction_filter: None 2024-09-13T14:21:21.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:21:21.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:21:21.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:21:21.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:21:21.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561594a17a38) 2024-09-13T14:21:21.478 INFO:journalctl@ceph.mon.a.smithi096.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:21:21.478 INFO:journalctl@ceph.mon.a.smithi096.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:21:21.479 INFO:journalctl@ceph.mon.a.smithi096.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:21:21.479 INFO:journalctl@ceph.mon.a.smithi096.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:21:21.479 INFO:journalctl@ceph.mon.a.smithi096.stdout: index_type: 0 2024-09-13T14:21:21.479 INFO:journalctl@ceph.mon.a.smithi096.stdout: data_block_index_type: 0 2024-09-13T14:21:21.479 INFO:journalctl@ceph.mon.a.smithi096.stdout: index_shortening: 1 2024-09-13T14:21:21.479 INFO:journalctl@ceph.mon.a.smithi096.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:21:21.479 INFO:journalctl@ceph.mon.a.smithi096.stdout: hash_index_allow_collision: 1 2024-09-13T14:21:21.479 INFO:journalctl@ceph.mon.a.smithi096.stdout: checksum: 1 2024-09-13T14:21:21.479 INFO:journalctl@ceph.mon.a.smithi096.stdout: no_block_cache: 0 2024-09-13T14:21:21.479 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_cache: 0x561594a4d1f0 2024-09-13T14:21:21.479 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:21:21.479 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_cache_options: 2024-09-13T14:21:21.479 INFO:journalctl@ceph.mon.a.smithi096.stdout: capacity : 536870912 2024-09-13T14:21:21.479 INFO:journalctl@ceph.mon.a.smithi096.stdout: num_shard_bits : 4 2024-09-13T14:21:21.480 INFO:journalctl@ceph.mon.a.smithi096.stdout: strict_capacity_limit : 0 2024-09-13T14:21:21.480 INFO:journalctl@ceph.mon.a.smithi096.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:21:21.480 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_cache_compressed: (nil) 2024-09-13T14:21:21.480 INFO:journalctl@ceph.mon.a.smithi096.stdout: persistent_cache: (nil) 2024-09-13T14:21:21.480 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_size: 4096 2024-09-13T14:21:21.480 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_size_deviation: 10 2024-09-13T14:21:21.480 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_restart_interval: 16 2024-09-13T14:21:21.480 INFO:journalctl@ceph.mon.a.smithi096.stdout: index_block_restart_interval: 1 2024-09-13T14:21:21.480 INFO:journalctl@ceph.mon.a.smithi096.stdout: metadata_block_size: 4096 2024-09-13T14:21:21.480 INFO:journalctl@ceph.mon.a.smithi096.stdout: partition_filters: 0 2024-09-13T14:21:21.480 INFO:journalctl@ceph.mon.a.smithi096.stdout: use_delta_encoding: 1 2024-09-13T14:21:21.480 INFO:journalctl@ceph.mon.a.smithi096.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T14:21:21.481 INFO:journalctl@ceph.mon.a.smithi096.stdout: whole_key_filtering: 1 2024-09-13T14:21:21.481 INFO:journalctl@ceph.mon.a.smithi096.stdout: verify_compression: 0 2024-09-13T14:21:21.481 INFO:journalctl@ceph.mon.a.smithi096.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:21:21.481 INFO:journalctl@ceph.mon.a.smithi096.stdout: format_version: 4 2024-09-13T14:21:21.481 INFO:journalctl@ceph.mon.a.smithi096.stdout: enable_index_compression: 1 2024-09-13T14:21:21.481 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_align: 0 2024-09-13T14:21:21.481 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:21:21.481 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:21:21.481 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compression: NoCompression 2024-09-13T14:21:21.481 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:21:21.481 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:21:21.481 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:21:21.481 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.num_levels: 7 2024-09-13T14:21:21.481 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:21:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:21:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:21:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:21:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:21:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:21:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:21:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:21:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:21:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:21:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:21:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:21:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:21:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:21:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:21:21.483 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:21:21.483 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:21:21.483 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:21:21.483 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:21:21.483 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:21:21.483 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:21:21.483 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:21:21.483 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:21:21.483 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:21:21.483 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:21:21.483 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:21:21.483 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:21:21.483 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:21:21.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:21:21.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:21:21.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:21:21.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:21:21.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:21:21.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:21:21.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T14:21:21.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:21:21.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:21:21.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T14:21:21.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:21:21.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:21:21.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:21:21.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:21:21.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:21:21.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:21:21.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:21:21.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:21:21.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:21:21.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:21:21.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:21:21.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:21:21.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:21:21.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:21:21.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:21:21.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:21:21.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:21:21.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:21:21.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:21:21.486 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:21:21.486 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:21:21.486 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:21:21.486 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:21:21.486 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.ttl: 2592000 2024-09-13T14:21:21.486 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:21:21.486 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:21:21.486 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:21:21.486 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:21:21.486 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:21:21.486 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:21:21.486 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:21:21.486 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:21:21.486 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-09-13T14:21:21.486 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-09-13T14:21:21.487 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237281184152, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-13T14:21:21.487 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-09-13T14:21:21.487 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T14:21:21.487 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237281184862, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80261, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78503, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10130, "raw_average_key_size": 47, "raw_value_size": 72716, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726237281, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "41eb8f6f-7217-4b59-8d92-ef940ef74c68", "db_session_id": "D42AABV3G5MJ8BB13ET9"}} 2024-09-13T14:21:21.487 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-09-13T14:21:21.487 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237281185224, "job": 1, "event": "recovery_finished"} 2024-09-13T14:21:21.487 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:21:21.487 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x561594a80700 2024-09-13T14:21:21.487 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: DB pointer 0x561594a50000 2024-09-13T14:21:21.487 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T14:21:21.487 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T14:21:21.487 INFO:journalctl@ceph.mon.a.smithi096.stdout: ** DB Stats ** 2024-09-13T14:21:21.487 INFO:journalctl@ceph.mon.a.smithi096.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:21:21.487 INFO:journalctl@ceph.mon.a.smithi096.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:21:21.488 INFO:journalctl@ceph.mon.a.smithi096.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:21:21.488 INFO:journalctl@ceph.mon.a.smithi096.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:21:21.488 INFO:journalctl@ceph.mon.a.smithi096.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:21:21.488 INFO:journalctl@ceph.mon.a.smithi096.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T14:21:21.488 INFO:journalctl@ceph.mon.a.smithi096.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:21:21.488 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:21:21.488 INFO:journalctl@ceph.mon.a.smithi096.stdout: ** Compaction Stats [default] ** 2024-09-13T14:21:21.488 INFO:journalctl@ceph.mon.a.smithi096.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:21:21.488 INFO:journalctl@ceph.mon.a.smithi096.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:21:21.488 INFO:journalctl@ceph.mon.a.smithi096.stdout: L0 2/0 80.17 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.5 0.00 0.00 1 0.000 0 0 2024-09-13T14:21:21.488 INFO:journalctl@ceph.mon.a.smithi096.stdout: Sum 2/0 80.17 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.5 0.00 0.00 1 0.000 0 0 2024-09-13T14:21:21.488 INFO:journalctl@ceph.mon.a.smithi096.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.5 0.00 0.00 1 0.000 0 0 2024-09-13T14:21:21.488 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:21:21.488 INFO:journalctl@ceph.mon.a.smithi096.stdout: ** Compaction Stats [default] ** 2024-09-13T14:21:21.489 INFO:journalctl@ceph.mon.a.smithi096.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:21:21.489 INFO:journalctl@ceph.mon.a.smithi096.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:21:21.489 INFO:journalctl@ceph.mon.a.smithi096.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 159.5 0.00 0.00 1 0.000 0 0 2024-09-13T14:21:21.489 INFO:journalctl@ceph.mon.a.smithi096.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:21:21.489 INFO:journalctl@ceph.mon.a.smithi096.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:21:21.489 INFO:journalctl@ceph.mon.a.smithi096.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:21:21.489 INFO:journalctl@ceph.mon.a.smithi096.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:21:21.489 INFO:journalctl@ceph.mon.a.smithi096.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:21:21.489 INFO:journalctl@ceph.mon.a.smithi096.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:21:21.489 INFO:journalctl@ceph.mon.a.smithi096.stdout: Cumulative compaction: 0.00 GB write, 25.73 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:21:21.489 INFO:journalctl@ceph.mon.a.smithi096.stdout: Interval compaction: 0.00 GB write, 25.73 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:21:21.490 INFO:journalctl@ceph.mon.a.smithi096.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:21:21.490 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:21:21.490 INFO:journalctl@ceph.mon.a.smithi096.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:21:21.490 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:21:21.490 INFO:journalctl@ceph.mon.a.smithi096.stdout: ** Compaction Stats [default] ** 2024-09-13T14:21:21.490 INFO:journalctl@ceph.mon.a.smithi096.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:21:21.490 INFO:journalctl@ceph.mon.a.smithi096.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:21:21.490 INFO:journalctl@ceph.mon.a.smithi096.stdout: L0 2/0 80.17 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.5 0.00 0.00 1 0.000 0 0 2024-09-13T14:21:21.490 INFO:journalctl@ceph.mon.a.smithi096.stdout: Sum 2/0 80.17 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.5 0.00 0.00 1 0.000 0 0 2024-09-13T14:21:21.490 INFO:journalctl@ceph.mon.a.smithi096.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:21:21.490 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:21:21.490 INFO:journalctl@ceph.mon.a.smithi096.stdout: ** Compaction Stats [default] ** 2024-09-13T14:21:21.490 INFO:journalctl@ceph.mon.a.smithi096.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:21:21.491 INFO:journalctl@ceph.mon.a.smithi096.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:21:21.491 INFO:journalctl@ceph.mon.a.smithi096.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 159.5 0.00 0.00 1 0.000 0 0 2024-09-13T14:21:21.491 INFO:journalctl@ceph.mon.a.smithi096.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:21:21.491 INFO:journalctl@ceph.mon.a.smithi096.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:21:21.491 INFO:journalctl@ceph.mon.a.smithi096.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:21:21.491 INFO:journalctl@ceph.mon.a.smithi096.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:21:21.491 INFO:journalctl@ceph.mon.a.smithi096.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:21:21.491 INFO:journalctl@ceph.mon.a.smithi096.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:21:21.491 INFO:journalctl@ceph.mon.a.smithi096.stdout: Cumulative compaction: 0.00 GB write, 25.46 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:21:21.491 INFO:journalctl@ceph.mon.a.smithi096.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:21:21.491 INFO:journalctl@ceph.mon.a.smithi096.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:21:21.491 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:21:21.491 INFO:journalctl@ceph.mon.a.smithi096.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:21:21.492 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: starting mon.a rank 0 at public addrs [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] at bind addrs [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:21:21.492 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: mon.a@-1(???) e1 preinit fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:21:21.492 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: mon.a@-1(???).mds e1 new map 2024-09-13T14:21:21.492 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: mon.a@-1(???).mds e1 print_map 2024-09-13T14:21:21.492 INFO:journalctl@ceph.mon.a.smithi096.stdout: e1 2024-09-13T14:21:21.492 INFO:journalctl@ceph.mon.a.smithi096.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:21:21.492 INFO:journalctl@ceph.mon.a.smithi096.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:21:21.492 INFO:journalctl@ceph.mon.a.smithi096.stdout: legacy client fscid: -1 2024-09-13T14:21:21.492 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:21:21.492 INFO:journalctl@ceph.mon.a.smithi096.stdout: No filesystems configured 2024-09-13T14:21:21.492 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T14:21:21.492 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:21:21.492 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:21:21.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:21:21.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-13T14:21:21.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 bash[29209]: 96e95ed67f926384baf6702dc5fbb3e4686072ff9aa3c2f5ebce42533fb138d1 2024-09-13T14:21:21.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 systemd[1]: Started Ceph mon.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:21:21.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T14:21:21.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: monmap e1: 1 mons at {a=[v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:21:21.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: fsmap 2024-09-13T14:21:21.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T14:21:21.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:21 smithi096 ceph-mon[29229]: mgrmap e1: no daemons active 2024-09-13T14:21:23.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:22 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/813833061' entity='client.admin' 2024-09-13T14:21:27.868 INFO:teuthology.orchestra.run.smithi096.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-13T14:21:27.869 INFO:teuthology.orchestra.run.smithi096.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:21:27.869 INFO:teuthology.orchestra.run.smithi096.stdout:Creating mgr... 2024-09-13T14:21:27.869 INFO:teuthology.orchestra.run.smithi096.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-13T14:21:27.869 INFO:teuthology.orchestra.run.smithi096.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-13T14:21:28.096 INFO:teuthology.orchestra.run.smithi096.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.y 2024-09-13T14:21:28.096 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stderr Failed to reset failed state of unit ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.y.service: Unit ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.y.service not loaded. 2024-09-13T14:21:28.416 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-51e50ed6-71db-11ef-bceb-c7b262605968.target.wants/ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-51e50ed6-71db-11ef-bceb-c7b262605968@.service. 2024-09-13T14:21:28.824 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:28 smithi096 podman[29443]: 2024-09-13 14:21:28.54713313 +0000 UTC m=+0.016007972 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph:quincy 2024-09-13T14:21:30.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:29 smithi096 podman[29443]: 2024-09-13 14:21:29.745715127 +0000 UTC m=+1.214589968 container create 7acb3784c92cd2665fee91086513160b5ab7ac81cce043e46641f77cab1b2412 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa) 2024-09-13T14:21:31.817 INFO:teuthology.orchestra.run.smithi096.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:21:31.818 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stdout disabled 2024-09-13T14:21:31.825 INFO:teuthology.orchestra.run.smithi096.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:21:31.825 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stdout inactive 2024-09-13T14:21:31.825 INFO:teuthology.orchestra.run.smithi096.stdout:firewalld.service is not enabled 2024-09-13T14:21:31.825 INFO:teuthology.orchestra.run.smithi096.stdout:Not possible to enable service . firewalld.service is not available 2024-09-13T14:21:31.834 INFO:teuthology.orchestra.run.smithi096.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:21:31.834 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stdout disabled 2024-09-13T14:21:31.842 INFO:teuthology.orchestra.run.smithi096.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:21:31.842 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stdout inactive 2024-09-13T14:21:31.842 INFO:teuthology.orchestra.run.smithi096.stdout:firewalld.service is not enabled 2024-09-13T14:21:31.842 INFO:teuthology.orchestra.run.smithi096.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-13T14:21:31.842 INFO:teuthology.orchestra.run.smithi096.stdout:Waiting for mgr to start... 2024-09-13T14:21:31.842 INFO:teuthology.orchestra.run.smithi096.stdout:Waiting for mgr... 2024-09-13T14:21:31.962 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:31 smithi096 podman[29443]: 2024-09-13 14:21:31.666975241 +0000 UTC m=+3.135850083 container init 7acb3784c92cd2665fee91086513160b5ab7ac81cce043e46641f77cab1b2412 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-13T14:21:31.962 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:31 smithi096 podman[29443]: 2024-09-13 14:21:31.670615872 +0000 UTC m=+3.139490726 container start 7acb3784c92cd2665fee91086513160b5ab7ac81cce043e46641f77cab1b2412 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y, org.label-schema.build-date=20240731, RELEASE=quincy-bbdadfa, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:21:31.962 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:31 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:31.789+0000 7f77aef5fec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:21:31.963 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:31 smithi096 bash[29443]: 7acb3784c92cd2665fee91086513160b5ab7ac81cce043e46641f77cab1b2412 2024-09-13T14:21:31.963 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:31 smithi096 systemd[1]: Started Ceph mgr.y for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:21:31.963 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:31 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:31.847+0000 7f77aef5fec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:21:31.963 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:31 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:31.915+0000 7f77aef5fec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:21:32.231 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:32 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:32.043+0000 7f77aef5fec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:21:32.232 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:32 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:32.185+0000 7f77aef5fec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:21:32.543 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:32 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2452204549' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T14:21:32.543 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:32 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:32.479+0000 7f77aef5fec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:21:32.822 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:32 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:32.542+0000 7f77aef5fec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:21:32.822 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:32 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:32.710+0000 7f77aef5fec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:21:32.955 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 2024-09-13T14:21:32.961 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:21:32.961 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "fsid": "51e50ed6-71db-11ef-bceb-c7b262605968", 2024-09-13T14:21:32.961 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T14:21:32.961 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T14:21:32.961 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T14:21:32.961 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T14:21:32.961 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:32.962 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T14:21:32.962 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T14:21:32.962 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 0 2024-09-13T14:21:32.962 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:21:32.962 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T14:21:32.962 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T14:21:32.962 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:21:32.962 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "quorum_age": 11, 2024-09-13T14:21:32.962 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T14:21:32.962 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:21:32.962 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-13T14:21:32.962 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T14:21:32.962 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:32.962 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T14:21:32.963 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:21:32.963 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T14:21:32.963 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T14:21:32.963 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T14:21:32.963 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T14:21:32.963 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T14:21:32.963 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T14:21:32.963 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:32.963 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T14:21:32.963 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T14:21:32.963 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T14:21:32.963 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T14:21:32.964 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T14:21:32.964 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T14:21:32.964 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T14:21:32.964 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T14:21:32.964 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T14:21:32.964 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:32.964 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T14:21:32.964 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:21:32.964 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T14:21:32.964 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T14:21:32.964 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:32.964 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T14:21:32.964 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-13T14:21:32.965 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T14:21:32.965 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T14:21:32.965 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T14:21:32.965 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T14:21:32.965 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T14:21:32.965 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:21:32.965 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:21:32.965 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:32.965 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T14:21:32.965 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:21:32.965 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T14:21:16.368194+0000", 2024-09-13T14:21:32.965 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:21:32.965 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:32.965 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T14:21:32.965 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:21:32.966 INFO:teuthology.orchestra.run.smithi096.stdout:mgr not available, waiting (1/15)... 2024-09-13T14:21:33.822 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:33 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:33.559+0000 7f77aef5fec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:21:33.822 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:33 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:33.622+0000 7f77aef5fec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:21:34.609 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:34 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:34.323+0000 7f77aef5fec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:21:34.610 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:34 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:34.433+0000 7f77aef5fec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:21:34.610 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:34 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:34.551+0000 7f77aef5fec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:21:34.991 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:34 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:34.608+0000 7f77aef5fec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:21:34.991 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:34 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:34.733+0000 7f77aef5fec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:21:34.991 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:34 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:34.798+0000 7f77aef5fec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:21:34.991 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:34 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:34.854+0000 7f77aef5fec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:21:34.991 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:34 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:34.989+0000 7f77aef5fec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:21:35.260 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:35 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:35.053+0000 7f77aef5fec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:21:35.260 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:35 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:35.118+0000 7f77aef5fec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:21:35.260 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:35 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:35.198+0000 7f77aef5fec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:21:35.260 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:35 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:35.259+0000 7f77aef5fec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:21:35.545 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:35 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:35.316+0000 7f77aef5fec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:21:35.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:35 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1128603521' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T14:21:36.063 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 2024-09-13T14:21:36.063 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:21:36.063 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "fsid": "51e50ed6-71db-11ef-bceb-c7b262605968", 2024-09-13T14:21:36.063 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T14:21:36.063 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T14:21:36.063 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T14:21:36.063 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T14:21:36.064 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:36.064 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T14:21:36.064 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T14:21:36.064 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 0 2024-09-13T14:21:36.064 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:21:36.064 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T14:21:36.064 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T14:21:36.064 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:21:36.064 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "quorum_age": 14, 2024-09-13T14:21:36.064 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T14:21:36.064 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:21:36.064 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-13T14:21:36.064 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T14:21:36.064 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:36.065 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T14:21:36.065 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:21:36.065 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T14:21:36.065 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T14:21:36.065 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T14:21:36.065 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T14:21:36.065 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T14:21:36.065 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T14:21:36.068 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:36.068 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T14:21:36.068 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T14:21:36.068 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T14:21:36.069 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T14:21:36.069 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T14:21:36.069 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T14:21:36.069 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T14:21:36.069 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T14:21:36.069 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T14:21:36.069 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:36.069 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T14:21:36.069 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:21:36.069 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T14:21:36.069 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T14:21:36.069 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:36.069 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T14:21:36.070 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-13T14:21:36.070 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T14:21:36.070 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T14:21:36.070 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T14:21:36.070 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T14:21:36.070 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T14:21:36.070 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:21:36.070 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:21:36.070 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:36.070 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T14:21:36.070 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:21:36.070 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T14:21:16.368194+0000", 2024-09-13T14:21:36.070 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:21:36.071 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:36.071 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T14:21:36.071 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:21:36.071 INFO:teuthology.orchestra.run.smithi096.stdout:mgr not available, waiting (2/15)... 2024-09-13T14:21:36.546 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:36.078+0000 7f77aef5fec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:21:36.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:36 smithi096 ceph-mon[29229]: Activating manager daemon y 2024-09-13T14:21:36.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:36 smithi096 ceph-mon[29229]: mgrmap e2: y(active, starting, since 0.0026988s) 2024-09-13T14:21:36.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:36 smithi096 ceph-mon[29229]: from='mgr.14100 172.21.15.96:0/2008403078' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:21:36.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:36 smithi096 ceph-mon[29229]: from='mgr.14100 172.21.15.96:0/2008403078' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:21:36.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:36 smithi096 ceph-mon[29229]: from='mgr.14100 172.21.15.96:0/2008403078' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:21:36.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:36 smithi096 ceph-mon[29229]: from='mgr.14100 172.21.15.96:0/2008403078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:21:36.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:36 smithi096 ceph-mon[29229]: from='mgr.14100 172.21.15.96:0/2008403078' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:21:36.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:36 smithi096 ceph-mon[29229]: Manager daemon y is now available 2024-09-13T14:21:36.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:36 smithi096 ceph-mon[29229]: from='mgr.14100 172.21.15.96:0/2008403078' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:21:36.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:36 smithi096 ceph-mon[29229]: from='mgr.14100 172.21.15.96:0/2008403078' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:21:36.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:36 smithi096 ceph-mon[29229]: from='mgr.14100 172.21.15.96:0/2008403078' entity='mgr.y' 2024-09-13T14:21:36.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:36 smithi096 ceph-mon[29229]: from='mgr.14100 172.21.15.96:0/2008403078' entity='mgr.y' 2024-09-13T14:21:36.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:36 smithi096 ceph-mon[29229]: from='mgr.14100 172.21.15.96:0/2008403078' entity='mgr.y' 2024-09-13T14:21:38.378 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:38 smithi096 ceph-mon[29229]: mgrmap e3: y(active, since 1.00704s) 2024-09-13T14:21:39.381 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:39 smithi096 ceph-mon[29229]: mgrmap e4: y(active, since 2s) 2024-09-13T14:21:39.381 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:39 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3491110002' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T14:21:39.419 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 2024-09-13T14:21:39.423 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:21:39.423 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "fsid": "51e50ed6-71db-11ef-bceb-c7b262605968", 2024-09-13T14:21:39.423 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T14:21:39.423 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T14:21:39.423 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T14:21:39.423 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T14:21:39.423 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:39.423 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T14:21:39.423 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T14:21:39.423 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 0 2024-09-13T14:21:39.423 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:21:39.424 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T14:21:39.424 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T14:21:39.424 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:21:39.424 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "quorum_age": 17, 2024-09-13T14:21:39.424 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T14:21:39.424 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:21:39.424 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-13T14:21:39.424 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T14:21:39.424 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:39.424 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T14:21:39.424 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:21:39.424 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T14:21:39.424 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T14:21:39.425 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T14:21:39.425 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T14:21:39.425 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T14:21:39.425 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T14:21:39.425 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:39.425 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T14:21:39.425 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T14:21:39.425 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T14:21:39.425 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T14:21:39.425 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T14:21:39.425 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T14:21:39.425 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T14:21:39.425 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T14:21:39.425 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T14:21:39.426 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:39.426 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T14:21:39.426 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:21:39.426 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T14:21:39.426 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T14:21:39.428 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:39.428 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T14:21:39.428 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T14:21:39.428 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T14:21:39.428 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T14:21:39.428 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T14:21:39.428 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T14:21:39.428 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T14:21:39.428 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:21:39.428 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:21:39.428 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:39.429 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T14:21:39.429 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:21:39.429 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T14:21:16.368194+0000", 2024-09-13T14:21:39.429 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:21:39.429 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:21:39.429 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T14:21:39.429 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:21:39.429 INFO:teuthology.orchestra.run.smithi096.stdout:mgr is available 2024-09-13T14:21:40.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:40 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2688985199' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T14:21:40.601 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 2024-09-13T14:21:40.602 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout [global] 2024-09-13T14:21:40.602 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout fsid = 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:21:40.602 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-13T14:21:40.602 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-13T14:21:40.602 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-13T14:21:40.602 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-13T14:21:40.602 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-13T14:21:40.602 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 2024-09-13T14:21:40.602 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-13T14:21:40.602 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-13T14:21:40.602 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 2024-09-13T14:21:40.603 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout [osd] 2024-09-13T14:21:40.603 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-13T14:21:40.603 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-13T14:21:40.603 INFO:teuthology.orchestra.run.smithi096.stdout:Enabling cephadm module... 2024-09-13T14:21:41.224 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:41 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3505691431' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-13T14:21:41.570 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:41 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ignoring --setuser ceph since I am not root 2024-09-13T14:21:41.571 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:41 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ignoring --setgroup ceph since I am not root 2024-09-13T14:21:41.571 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:41 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:41.344+0000 7f430f954ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:21:41.571 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:41 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:41.398+0000 7f430f954ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:21:41.571 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:41 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:41.463+0000 7f430f954ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:21:41.822 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:41 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:41.591+0000 7f430f954ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:21:41.822 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:41 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:41.726+0000 7f430f954ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:21:42.112 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:41 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:41.990+0000 7f430f954ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:21:42.125 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:42 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:42.050+0000 7f430f954ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:21:42.491 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:42 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:42.216+0000 7f430f954ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:21:42.491 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:42 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3505691431' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-13T14:21:42.491 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:42 smithi096 ceph-mon[29229]: mgrmap e5: y(active, since 5s) 2024-09-13T14:21:43.097 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:21:43.097 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-09-13T14:21:43.097 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T14:21:43.097 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-13T14:21:43.097 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-13T14:21:43.097 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:21:43.098 INFO:teuthology.orchestra.run.smithi096.stdout:Waiting for the mgr to restart... 2024-09-13T14:21:43.098 INFO:teuthology.orchestra.run.smithi096.stdout:Waiting for mgr epoch 5... 2024-09-13T14:21:43.344 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:43 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2020724306' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T14:21:43.345 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:43 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:43.079+0000 7f430f954ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:21:43.345 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:43 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:43.152+0000 7f430f954ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:21:44.162 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:43 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:43.876+0000 7f430f954ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:21:44.162 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:43 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:43.984+0000 7f430f954ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:21:44.163 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:44 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:44.101+0000 7f430f954ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:21:44.163 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:44 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:44.160+0000 7f430f954ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:21:44.543 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:44 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:44.286+0000 7f430f954ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:21:44.543 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:44 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:44.352+0000 7f430f954ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:21:44.544 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:44 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:44.410+0000 7f430f954ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:21:44.544 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:44 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:44.541+0000 7f430f954ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:21:44.804 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:44 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:44.604+0000 7f430f954ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:21:44.804 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:44 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:44.668+0000 7f430f954ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:21:44.804 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:44 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:44.746+0000 7f430f954ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:21:44.804 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:44 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:44.802+0000 7f430f954ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:21:45.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:44 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:44.858+0000 7f430f954ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:21:46.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:45.589+0000 7f430f954ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:21:46.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: Active manager daemon y restarted 2024-09-13T14:21:46.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: Activating manager daemon y 2024-09-13T14:21:46.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: osdmap e2: 0 total, 0 up, 0 in 2024-09-13T14:21:46.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: mgrmap e6: y(active, starting, since 0.00308583s) 2024-09-13T14:21:46.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:21:46.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:21:46.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:21:46.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:21:46.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:21:46.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: Manager daemon y is now available 2024-09-13T14:21:46.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' 2024-09-13T14:21:46.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' 2024-09-13T14:21:46.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:46.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:46.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:21:46.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:45 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:21:46.882 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:46 smithi096 ceph-mon[29229]: Found migration_current of "None". Setting to last migration. 2024-09-13T14:21:46.882 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:46 smithi096 ceph-mon[29229]: mgrmap e7: y(active, since 1.00474s) 2024-09-13T14:21:47.160 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:21:47.160 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-09-13T14:21:47.160 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-13T14:21:47.160 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:21:47.160 INFO:teuthology.orchestra.run.smithi096.stdout:mgr epoch 5 is available 2024-09-13T14:21:47.160 INFO:teuthology.orchestra.run.smithi096.stdout:Setting orchestrator backend to cephadm... 2024-09-13T14:21:47.779 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:47 smithi096 ceph-mon[29229]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T14:21:47.779 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:47 smithi096 ceph-mon[29229]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T14:21:48.331 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:48.152+0000 7f42c1d03640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:21:48] ENGINE Error in HTTPServer.serve 2024-09-13T14:21:48.331 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: Traceback (most recent call last): 2024-09-13T14:21:48.331 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:21:48.331 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: self._connections.run(self.expiration_interval) 2024-09-13T14:21:48.331 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:21:48.331 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: self._run(expiration_interval) 2024-09-13T14:21:48.332 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:21:48.332 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:21:48.332 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:21:48.332 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:21:48.332 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:21:48.332 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: s = self.context.wrap_socket( 2024-09-13T14:21:48.332 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:21:48.332 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: return self.sslsocket_class._create( 2024-09-13T14:21:48.332 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:21:48.332 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: self.do_handshake() 2024-09-13T14:21:48.332 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:21:48.332 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: self._sslobj.do_handshake() 2024-09-13T14:21:48.332 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:21:48.332 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:21:48.907 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-mon[29229]: mgrmap e8: y(active, since 2s) 2024-09-13T14:21:48.907 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-mon[29229]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:48.907 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' 2024-09-13T14:21:48.907 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:48.907 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-mon[29229]: [13/Sep/2024:14:21:48] ENGINE Bus STARTING 2024-09-13T14:21:48.907 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-mon[29229]: [13/Sep/2024:14:21:48] ENGINE Serving on https://172.21.15.96:7150 2024-09-13T14:21:48.907 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-mon[29229]: [13/Sep/2024:14:21:48] ENGINE Bus STARTED 2024-09-13T14:21:48.907 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-mon[29229]: [13/Sep/2024:14:21:48] ENGINE Error in HTTPServer.serve 2024-09-13T14:21:48.907 INFO:journalctl@ceph.mon.a.smithi096.stdout: Traceback (most recent call last): 2024-09-13T14:21:48.907 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:21:48.907 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:21:48.908 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:21:48.908 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._run(expiration_interval) 2024-09-13T14:21:48.908 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:21:48.908 INFO:journalctl@ceph.mon.a.smithi096.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:21:48.908 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:21:48.908 INFO:journalctl@ceph.mon.a.smithi096.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:21:48.908 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:21:48.908 INFO:journalctl@ceph.mon.a.smithi096.stdout: s = self.context.wrap_socket( 2024-09-13T14:21:48.908 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:21:48.908 INFO:journalctl@ceph.mon.a.smithi096.stdout: return self.sslsocket_class._create( 2024-09-13T14:21:48.908 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:21:48.908 INFO:journalctl@ceph.mon.a.smithi096.stdout: self.do_handshake() 2024-09-13T14:21:48.909 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:21:48.909 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._sslobj.do_handshake() 2024-09-13T14:21:48.909 INFO:journalctl@ceph.mon.a.smithi096.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:21:48.909 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' 2024-09-13T14:21:48.909 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:48 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:49.422 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-13T14:21:49.422 INFO:teuthology.orchestra.run.smithi096.stdout:Generating ssh key... 2024-09-13T14:21:49.805 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:49 smithi096 ceph-mon[29229]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:50.400 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: Generating public/private rsa key pair. 2024-09-13T14:21:50.400 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: Your identification has been saved in /tmp/tmpchq4ikrq/key 2024-09-13T14:21:50.401 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: Your public key has been saved in /tmp/tmpchq4ikrq/key.pub 2024-09-13T14:21:50.401 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: The key fingerprint is: 2024-09-13T14:21:50.401 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: SHA256:ODhU815ThS3GevFmrhSZNuiwddiGS8ZQolDDD6Ab+rg ceph-51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:21:50.401 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: The key's randomart image is: 2024-09-13T14:21:50.401 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: +---[RSA 3072]----+ 2024-09-13T14:21:50.401 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: | o+= .....+. | 2024-09-13T14:21:50.401 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: | . oo=.. .* . | 2024-09-13T14:21:50.401 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: | o . .o.ooB * | 2024-09-13T14:21:50.401 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: | . + . oo.X.@ + | 2024-09-13T14:21:50.401 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: |. . o o SB * * | 2024-09-13T14:21:50.401 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: | o . .. o . . | 2024-09-13T14:21:50.402 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: |. . . . | 2024-09-13T14:21:50.402 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: | . . | 2024-09-13T14:21:50.402 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: |E | 2024-09-13T14:21:50.402 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:21:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: +----[SHA256]-----+ 2024-09-13T14:21:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:51 smithi096 ceph-mon[29229]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:51 smithi096 ceph-mon[29229]: Generating ssh key... 2024-09-13T14:21:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:51 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' 2024-09-13T14:21:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:51 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' 2024-09-13T14:21:51.852 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0hnjkiwSpTWqCcfamr6ExkLA2cTqZnXKMKyO1duIhMBleLuyl73ZiW0KJXF7jaJ/T77YIYFOOeV2swiScVLgJETPNsG8YF4mFf16hYUxmKoaW74xShm4GGmrMtpD0jqotFi4/VJ+DrPq4oQGu+1p6UuCK1bZseZjOXry7iUAKJIAZj0owIzJzbUfW7KkJSSR9D8DQGRgjlmihbhS5/a/McUrnFOOdAW6oNnY0A3PNNu2L8YFXfVTjs6G2zK1gRjU019o5VtuOvdZ9j5E4+w7UbimeDmryRjfaPLhhSbVXbwNNQY2x1a0iPWz1DzPNAPgbA7yUcH6dkd4esnEMOCzxmnBgvCDQOlAtTe0X0KVWqfxhFsjgB4pAVogN54Pb+GTLd6+MBdT43RU0hAvGYt7b63QvADqoI1lNBiv6w84vZqc3Y7OhZ+6JG1Q+O8kmNW11GoGqkFzKuJZfexgThUO3nui/iSqWBybYyi+P3GxVnrhcVwT5SDVmUydrjZLPAdE= ceph-51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:21:51.853 INFO:teuthology.orchestra.run.smithi096.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-13T14:21:51.853 INFO:teuthology.orchestra.run.smithi096.stdout:Adding key to root@localhost authorized_keys... 2024-09-13T14:21:51.862 INFO:teuthology.orchestra.run.smithi096.stdout:Adding host smithi096... 2024-09-13T14:21:52.538 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:52 smithi096 ceph-mon[29229]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:53.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:53 smithi096 ceph-mon[29229]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi096", "addr": "172.21.15.96", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:54.559 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:54 smithi096 ceph-mon[29229]: Deploying cephadm binary to smithi096 2024-09-13T14:21:55.741 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout Added host 'smithi096' with addr '172.21.15.96' 2024-09-13T14:21:55.741 INFO:teuthology.orchestra.run.smithi096.stdout:Deploying unmanaged mon service... 2024-09-13T14:21:56.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:56 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' 2024-09-13T14:21:56.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:56 smithi096 ceph-mon[29229]: Added host smithi096 2024-09-13T14:21:56.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:56 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:57.108 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-13T14:21:57.108 INFO:teuthology.orchestra.run.smithi096.stdout:Deploying unmanaged mgr service... 2024-09-13T14:21:57.361 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:57 smithi096 ceph-mon[29229]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:57.361 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:57 smithi096 ceph-mon[29229]: Saving service mon spec with placement count:5 2024-09-13T14:21:57.361 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:57 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' 2024-09-13T14:21:59.303 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-13T14:21:59.687 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:59 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' 2024-09-13T14:21:59.687 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:59 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' 2024-09-13T14:21:59.687 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:59 smithi096 ceph-mon[29229]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:59.687 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:59 smithi096 ceph-mon[29229]: Saving service mgr spec with placement count:2 2024-09-13T14:21:59.687 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:21:59 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' 2024-09-13T14:22:01.210 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:00 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1004447952' entity='client.admin' 2024-09-13T14:22:01.210 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:00 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' 2024-09-13T14:22:02.462 INFO:teuthology.orchestra.run.smithi096.stdout:Enabling the dashboard module... 2024-09-13T14:22:02.513 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:02 smithi096 ceph-mon[29229]: from='mgr.14118 172.21.15.96:0/1281428765' entity='mgr.y' 2024-09-13T14:22:02.513 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:02 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/77463578' entity='client.admin' 2024-09-13T14:22:04.103 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:03 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2874371647' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-13T14:22:04.104 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:03 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ignoring --setuser ceph since I am not root 2024-09-13T14:22:04.104 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:03 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ignoring --setgroup ceph since I am not root 2024-09-13T14:22:04.104 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:03 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:03.981+0000 7ffa62780ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:22:04.104 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:04.036+0000 7ffa62780ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:22:04.104 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:04.102+0000 7ffa62780ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:22:04.364 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:04.226+0000 7ffa62780ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:22:04.364 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:04.363+0000 7ffa62780ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:22:05.062 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:04 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2874371647' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-13T14:22:05.063 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:04 smithi096 ceph-mon[29229]: mgrmap e9: y(active, since 18s) 2024-09-13T14:22:05.063 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:04.624+0000 7ffa62780ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:22:05.063 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:04.687+0000 7ffa62780ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:22:05.063 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:04.854+0000 7ffa62780ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:22:05.605 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:22:05.605 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-13T14:22:05.605 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T14:22:05.605 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-13T14:22:05.605 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-13T14:22:05.605 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:22:05.605 INFO:teuthology.orchestra.run.smithi096.stdout:Waiting for the mgr to restart... 2024-09-13T14:22:05.605 INFO:teuthology.orchestra.run.smithi096.stdout:Waiting for mgr epoch 9... 2024-09-13T14:22:06.004 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:05 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1355315363' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T14:22:06.005 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:05 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:05.716+0000 7ffa62780ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:22:06.005 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:05 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:05.781+0000 7ffa62780ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:22:06.771 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:06 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:06.483+0000 7ffa62780ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:22:06.771 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:06 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:06.592+0000 7ffa62780ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:22:06.771 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:06 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:06.712+0000 7ffa62780ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:22:07.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:06 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:06.769+0000 7ffa62780ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:22:07.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:06 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:06.894+0000 7ffa62780ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:22:07.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:06 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:06.958+0000 7ffa62780ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:22:07.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:07.017+0000 7ffa62780ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:22:07.411 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:07.150+0000 7ffa62780ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:22:07.412 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:07.212+0000 7ffa62780ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:22:07.412 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:07.275+0000 7ffa62780ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:22:07.412 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:07.353+0000 7ffa62780ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:22:07.412 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:07.410+0000 7ffa62780ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:22:07.822 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:07.468+0000 7ffa62780ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:22:08.572 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:08 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:08.205+0000 7ffa62780ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:22:08.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:08 smithi096 ceph-mon[29229]: Active manager daemon y restarted 2024-09-13T14:22:08.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:08 smithi096 ceph-mon[29229]: Activating manager daemon y 2024-09-13T14:22:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:08 smithi096 ceph-mon[29229]: osdmap e3: 0 total, 0 up, 0 in 2024-09-13T14:22:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:08 smithi096 ceph-mon[29229]: mgrmap e10: y(active, starting, since 0.00319039s) 2024-09-13T14:22:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:22:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:22:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:22:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:22:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:08 smithi096 ceph-mon[29229]: Manager daemon y is now available 2024-09-13T14:22:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:22:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:22:09.811 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:09.511+0000 7ffa132de640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:22:09] ENGINE Error in HTTPServer.serve 2024-09-13T14:22:09.811 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: Traceback (most recent call last): 2024-09-13T14:22:09.811 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:22:09.811 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: self._connections.run(self.expiration_interval) 2024-09-13T14:22:09.811 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:22:09.811 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: self._run(expiration_interval) 2024-09-13T14:22:09.812 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:22:09.812 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:22:09.812 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:22:09.812 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:22:09.812 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:22:09.812 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: s = self.context.wrap_socket( 2024-09-13T14:22:09.812 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:22:09.812 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: return self.sslsocket_class._create( 2024-09-13T14:22:09.812 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:22:09.812 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: self.do_handshake() 2024-09-13T14:22:09.812 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:22:09.812 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: self._sslobj.do_handshake() 2024-09-13T14:22:09.812 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:22:09.813 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:22:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:22:10.097 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:22:10.097 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-13T14:22:10.097 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-13T14:22:10.097 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:22:10.098 INFO:teuthology.orchestra.run.smithi096.stdout:mgr epoch 9 is available 2024-09-13T14:22:10.098 INFO:teuthology.orchestra.run.smithi096.stdout:Generating a dashboard self-signed certificate... 2024-09-13T14:22:10.422 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:10 smithi096 ceph-mon[29229]: mgrmap e11: y(active, since 1.00539s) 2024-09-13T14:22:10.423 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:10 smithi096 ceph-mon[29229]: [13/Sep/2024:14:22:09] ENGINE Bus STARTING 2024-09-13T14:22:10.423 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:10 smithi096 ceph-mon[29229]: [13/Sep/2024:14:22:09] ENGINE Serving on https://172.21.15.96:7150 2024-09-13T14:22:10.423 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:10 smithi096 ceph-mon[29229]: [13/Sep/2024:14:22:09] ENGINE Bus STARTED 2024-09-13T14:22:10.423 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:10 smithi096 ceph-mon[29229]: [13/Sep/2024:14:22:09] ENGINE Error in HTTPServer.serve 2024-09-13T14:22:10.423 INFO:journalctl@ceph.mon.a.smithi096.stdout: Traceback (most recent call last): 2024-09-13T14:22:10.423 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:22:10.423 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:22:10.423 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:22:10.423 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._run(expiration_interval) 2024-09-13T14:22:10.423 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:22:10.423 INFO:journalctl@ceph.mon.a.smithi096.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:22:10.423 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:22:10.424 INFO:journalctl@ceph.mon.a.smithi096.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:22:10.424 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:22:10.424 INFO:journalctl@ceph.mon.a.smithi096.stdout: s = self.context.wrap_socket( 2024-09-13T14:22:10.424 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:22:10.424 INFO:journalctl@ceph.mon.a.smithi096.stdout: return self.sslsocket_class._create( 2024-09-13T14:22:10.424 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:22:10.424 INFO:journalctl@ceph.mon.a.smithi096.stdout: self.do_handshake() 2024-09-13T14:22:10.424 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:22:10.424 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._sslobj.do_handshake() 2024-09-13T14:22:10.424 INFO:journalctl@ceph.mon.a.smithi096.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:22:10.424 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:10 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:10.424 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:10 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:11.320 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-13T14:22:11.320 INFO:teuthology.orchestra.run.smithi096.stdout:Creating initial admin user... 2024-09-13T14:22:11.757 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:11 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:11.757 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:11 smithi096 ceph-mon[29229]: mgrmap e12: y(active, since 2s) 2024-09-13T14:22:11.757 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:11 smithi096 ceph-mon[29229]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:11.757 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:11 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:11.757 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:11 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:13.265 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$xOfV7bTAItixIS2WKnLCbeQNxSgDlQ3e1xHPMQ/Y1lFW5KIPPVOUS", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726237332, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-13T14:22:13.265 INFO:teuthology.orchestra.run.smithi096.stdout:Fetching dashboard port number... 2024-09-13T14:22:13.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:13 smithi096 ceph-mon[29229]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:13.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:13 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:14.396 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stdout 8443 2024-09-13T14:22:14.406 INFO:teuthology.orchestra.run.smithi096.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:22:14.406 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stdout disabled 2024-09-13T14:22:14.414 INFO:teuthology.orchestra.run.smithi096.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:22:14.414 INFO:teuthology.orchestra.run.smithi096.stdout:systemctl: stdout inactive 2024-09-13T14:22:14.414 INFO:teuthology.orchestra.run.smithi096.stdout:firewalld.service is not enabled 2024-09-13T14:22:14.414 INFO:teuthology.orchestra.run.smithi096.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-13T14:22:14.415 INFO:teuthology.orchestra.run.smithi096.stdout:Ceph Dashboard is now available at: 2024-09-13T14:22:14.416 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:22:14.416 INFO:teuthology.orchestra.run.smithi096.stdout: URL: https://smithi096.front.sepia.ceph.com:8443/ 2024-09-13T14:22:14.416 INFO:teuthology.orchestra.run.smithi096.stdout: User: admin 2024-09-13T14:22:14.416 INFO:teuthology.orchestra.run.smithi096.stdout: Password: pn8eh34m34 2024-09-13T14:22:14.416 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:22:14.416 INFO:teuthology.orchestra.run.smithi096.stdout:Saving cluster configuration to /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config directory 2024-09-13T14:22:14.416 INFO:teuthology.orchestra.run.smithi096.stdout:Enabling autotune for osd_memory_target 2024-09-13T14:22:14.528 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:14 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3349642444' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-13T14:22:15.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:15 smithi096 ceph-mon[29229]: mgrmap e13: y(active, since 6s) 2024-09-13T14:22:16.783 INFO:teuthology.orchestra.run.smithi096.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-13T14:22:16.783 INFO:teuthology.orchestra.run.smithi096.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-13T14:22:16.783 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:22:16.783 INFO:teuthology.orchestra.run.smithi096.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:16.784 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:22:16.784 INFO:teuthology.orchestra.run.smithi096.stdout:Or, if you are only running a single cluster on this host: 2024-09-13T14:22:16.784 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:22:16.784 INFO:teuthology.orchestra.run.smithi096.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-13T14:22:16.784 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:22:16.784 INFO:teuthology.orchestra.run.smithi096.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-13T14:22:16.784 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:22:16.784 INFO:teuthology.orchestra.run.smithi096.stdout: ceph telemetry on 2024-09-13T14:22:16.784 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:22:16.784 INFO:teuthology.orchestra.run.smithi096.stdout:For more information see: 2024-09-13T14:22:16.784 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:22:16.784 INFO:teuthology.orchestra.run.smithi096.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-13T14:22:16.785 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:22:16.785 INFO:teuthology.orchestra.run.smithi096.stdout:Bootstrap complete. 2024-09-13T14:22:16.821 INFO:tasks.cephadm:Fetching config... 2024-09-13T14:22:16.821 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:22:16.821 DEBUG:teuthology.orchestra.run.smithi096:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-13T14:22:16.838 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-13T14:22:16.838 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:22:16.838 DEBUG:teuthology.orchestra.run.smithi096:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-13T14:22:16.893 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-13T14:22:16.896 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:22:16.896 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/keyring of=/dev/stdout 2024-09-13T14:22:16.958 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-13T14:22:16.958 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:22:16.958 DEBUG:teuthology.orchestra.run.smithi096:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-13T14:22:17.015 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-13T14:22:17.015 DEBUG:teuthology.orchestra.run.smithi096:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0hnjkiwSpTWqCcfamr6ExkLA2cTqZnXKMKyO1duIhMBleLuyl73ZiW0KJXF7jaJ/T77YIYFOOeV2swiScVLgJETPNsG8YF4mFf16hYUxmKoaW74xShm4GGmrMtpD0jqotFi4/VJ+DrPq4oQGu+1p6UuCK1bZseZjOXry7iUAKJIAZj0owIzJzbUfW7KkJSSR9D8DQGRgjlmihbhS5/a/McUrnFOOdAW6oNnY0A3PNNu2L8YFXfVTjs6G2zK1gRjU019o5VtuOvdZ9j5E4+w7UbimeDmryRjfaPLhhSbVXbwNNQY2x1a0iPWz1DzPNAPgbA7yUcH6dkd4esnEMOCzxmnBgvCDQOlAtTe0X0KVWqfxhFsjgB4pAVogN54Pb+GTLd6+MBdT43RU0hAvGYt7b63QvADqoI1lNBiv6w84vZqc3Y7OhZ+6JG1Q+O8kmNW11GoGqkFzKuJZfexgThUO3nui/iSqWBybYyi+P3GxVnrhcVwT5SDVmUydrjZLPAdE= ceph-51e50ed6-71db-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T14:22:17.090 INFO:teuthology.orchestra.run.smithi096.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0hnjkiwSpTWqCcfamr6ExkLA2cTqZnXKMKyO1duIhMBleLuyl73ZiW0KJXF7jaJ/T77YIYFOOeV2swiScVLgJETPNsG8YF4mFf16hYUxmKoaW74xShm4GGmrMtpD0jqotFi4/VJ+DrPq4oQGu+1p6UuCK1bZseZjOXry7iUAKJIAZj0owIzJzbUfW7KkJSSR9D8DQGRgjlmihbhS5/a/McUrnFOOdAW6oNnY0A3PNNu2L8YFXfVTjs6G2zK1gRjU019o5VtuOvdZ9j5E4+w7UbimeDmryRjfaPLhhSbVXbwNNQY2x1a0iPWz1DzPNAPgbA7yUcH6dkd4esnEMOCzxmnBgvCDQOlAtTe0X0KVWqfxhFsjgB4pAVogN54Pb+GTLd6+MBdT43RU0hAvGYt7b63QvADqoI1lNBiv6w84vZqc3Y7OhZ+6JG1Q+O8kmNW11GoGqkFzKuJZfexgThUO3nui/iSqWBybYyi+P3GxVnrhcVwT5SDVmUydrjZLPAdE= ceph-51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:22:17.101 DEBUG:teuthology.orchestra.run.smithi204:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0hnjkiwSpTWqCcfamr6ExkLA2cTqZnXKMKyO1duIhMBleLuyl73ZiW0KJXF7jaJ/T77YIYFOOeV2swiScVLgJETPNsG8YF4mFf16hYUxmKoaW74xShm4GGmrMtpD0jqotFi4/VJ+DrPq4oQGu+1p6UuCK1bZseZjOXry7iUAKJIAZj0owIzJzbUfW7KkJSSR9D8DQGRgjlmihbhS5/a/McUrnFOOdAW6oNnY0A3PNNu2L8YFXfVTjs6G2zK1gRjU019o5VtuOvdZ9j5E4+w7UbimeDmryRjfaPLhhSbVXbwNNQY2x1a0iPWz1DzPNAPgbA7yUcH6dkd4esnEMOCzxmnBgvCDQOlAtTe0X0KVWqfxhFsjgB4pAVogN54Pb+GTLd6+MBdT43RU0hAvGYt7b63QvADqoI1lNBiv6w84vZqc3Y7OhZ+6JG1Q+O8kmNW11GoGqkFzKuJZfexgThUO3nui/iSqWBybYyi+P3GxVnrhcVwT5SDVmUydrjZLPAdE= ceph-51e50ed6-71db-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T14:22:17.240 INFO:teuthology.orchestra.run.smithi204.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0hnjkiwSpTWqCcfamr6ExkLA2cTqZnXKMKyO1duIhMBleLuyl73ZiW0KJXF7jaJ/T77YIYFOOeV2swiScVLgJETPNsG8YF4mFf16hYUxmKoaW74xShm4GGmrMtpD0jqotFi4/VJ+DrPq4oQGu+1p6UuCK1bZseZjOXry7iUAKJIAZj0owIzJzbUfW7KkJSSR9D8DQGRgjlmihbhS5/a/McUrnFOOdAW6oNnY0A3PNNu2L8YFXfVTjs6G2zK1gRjU019o5VtuOvdZ9j5E4+w7UbimeDmryRjfaPLhhSbVXbwNNQY2x1a0iPWz1DzPNAPgbA7yUcH6dkd4esnEMOCzxmnBgvCDQOlAtTe0X0KVWqfxhFsjgB4pAVogN54Pb+GTLd6+MBdT43RU0hAvGYt7b63QvADqoI1lNBiv6w84vZqc3Y7OhZ+6JG1Q+O8kmNW11GoGqkFzKuJZfexgThUO3nui/iSqWBybYyi+P3GxVnrhcVwT5SDVmUydrjZLPAdE= ceph-51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:22:17.254 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-13T14:22:17.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:17 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/426101041' entity='client.admin' 2024-09-13T14:22:17.498 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:19.995 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:19 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:19.995 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:19 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:19.996 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:19 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi096", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:22:19.996 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:19 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:19.996 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:19 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:19.996 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:19 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:21.299 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:21 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1378400028' entity='client.admin' 2024-09-13T14:22:21.299 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:21 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:21.299 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:21 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:21.324 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-13T14:22:21.324 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-13T14:22:21.560 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:23.864 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi204 2024-09-13T14:22:23.864 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-09-13T14:22:23.864 DEBUG:teuthology.orchestra.run.smithi204:> dd of=/etc/ceph/ceph.conf 2024-09-13T14:22:23.885 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-09-13T14:22:23.885 DEBUG:teuthology.orchestra.run.smithi204:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:23.943 INFO:tasks.cephadm:Adding host smithi204 to orchestrator... 2024-09-13T14:22:23.943 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch host add smithi204 2024-09-13T14:22:24.180 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:24.300 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:24 smithi096 ceph-mon[29229]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:24.300 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:24 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:24.300 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:24 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:24.300 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:24 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:24.300 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:24 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:24.301 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:24 smithi096 ceph-mon[29229]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:22:24.301 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:24 smithi096 ceph-mon[29229]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:24.301 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:24 smithi096 ceph-mon[29229]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:24.301 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:24 smithi096 ceph-mon[29229]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:22:24.301 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:24 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:24.301 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:24 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:24.301 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:24 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:26.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:25 smithi096 ceph-mon[29229]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi204", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:27.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:26 smithi096 ceph-mon[29229]: Deploying cephadm binary to smithi204 2024-09-13T14:22:28.347 INFO:teuthology.orchestra.run.smithi096.stdout:Added host 'smithi204' with addr '172.21.15.204' 2024-09-13T14:22:28.925 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-13T14:22:29.162 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:29.570 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:29 smithi096 ceph-mon[29229]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:29.570 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:29 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:29.570 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:29 smithi096 ceph-mon[29229]: Added host smithi204 2024-09-13T14:22:29.570 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:29 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:29.570 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:29 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:29.570 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:29 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:30.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:30.864 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:22:30.864 INFO:teuthology.orchestra.run.smithi096.stdout:[{"addr": "172.21.15.96", "hostname": "smithi096", "labels": [], "status": ""}, {"addr": "172.21.15.204", "hostname": "smithi204", "labels": [], "status": ""}] 2024-09-13T14:22:31.464 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-13T14:22:31.464 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-13T14:22:31.701 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:31.949 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:31 smithi096 ceph-mon[29229]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:31.949 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:31 smithi096 ceph-mon[29229]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:22:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:33 smithi096 ceph-mon[29229]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:33 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:33 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:33 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:33.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:33 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:33.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:33 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi204", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:22:33.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:33 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:33.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:33 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:33.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:33 smithi096 ceph-mon[29229]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:22:33.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:33 smithi096 ceph-mon[29229]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:33.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:33 smithi096 ceph-mon[29229]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:33.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:33 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:33.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:33 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:33.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:33 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:34.438 INFO:teuthology.orchestra.run.smithi096.stderr:adjusted tunables profile to default 2024-09-13T14:22:34.744 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:34 smithi096 ceph-mon[29229]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:22:34.744 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:34 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1783448714' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T14:22:35.050 INFO:tasks.cephadm:Adding mon.a on smithi096 2024-09-13T14:22:35.050 INFO:tasks.cephadm:Adding mon.c on smithi096 2024-09-13T14:22:35.050 INFO:tasks.cephadm:Adding mon.b on smithi204 2024-09-13T14:22:35.051 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch apply mon '3;smithi096:172.21.15.96=a;smithi096:[v2:172.21.15.96:3301,v1:172.21.15.96:6790]=c;smithi204:172.21.15.204=b' 2024-09-13T14:22:35.292 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:35.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:35 smithi096 ceph-mon[29229]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:35.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:35 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1783448714' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T14:22:35.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:35 smithi096 ceph-mon[29229]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:22:36.999 INFO:teuthology.orchestra.run.smithi204.stdout:Scheduled mon update... 2024-09-13T14:22:37.564 DEBUG:teuthology.orchestra.run.smithi096:mon.c> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.c.service 2024-09-13T14:22:37.567 DEBUG:teuthology.orchestra.run.smithi204:mon.b> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.b.service 2024-09-13T14:22:37.569 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T14:22:37.569 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T14:22:37.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:37 smithi096 ceph-mon[29229]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:37.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:37 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:37.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:37 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:37.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:37 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:37.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:37 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:37.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:37 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:37.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:37 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:37.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:37 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:38.695 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:38.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:38 smithi096 ceph-mon[29229]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi096:172.21.15.96=a;smithi096:[v2:172.21.15.96:3301,v1:172.21.15.96:6790]=c;smithi204:172.21.15.204=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:38.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:38 smithi096 ceph-mon[29229]: Saving service mon spec with placement smithi096:172.21.15.96=a;smithi096:[v2:172.21.15.96:3301,v1:172.21.15.96:6790]=c;smithi204:172.21.15.204=b;count:3 2024-09-13T14:22:38.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:38 smithi096 ceph-mon[29229]: Deploying daemon mon.b on smithi204 2024-09-13T14:22:39.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:39 smithi096 ceph-mon[29229]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:40.932 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:40 smithi204 systemd[1]: Starting Ceph mon.b for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:22:41.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:41 smithi096 ceph-mon[29229]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:41.880 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:22:41.880 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":1,"fsid":"51e50ed6-71db-11ef-bceb-c7b262605968","modified":"2024-09-13T14:21:13.478657Z","created":"2024-09-13T14:21:13.478657Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:3300","nonce":0},{"type":"v1","addr":"172.21.15.96:6789","nonce":0}]},"addr":"172.21.15.96:6789/0","public_addr":"172.21.15.96:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T14:22:41.880 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 1 2024-09-13T14:22:41.891 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:41 smithi204 podman[32514]: 2024-09-13 14:22:41.801963252 +0000 UTC m=+0.110779634 container create a37b0845979d4d4a3406a0b68c169aed006fdda0ae2ea5c448ebe7f228884387 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:22:41.892 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:41 smithi204 podman[32514]: 2024-09-13 14:22:41.708820427 +0000 UTC m=+0.017636817 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T14:22:42.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 podman[32514]: 2024-09-13 14:22:42.155481434 +0000 UTC m=+0.464297816 container init a37b0845979d4d4a3406a0b68c169aed006fdda0ae2ea5c448ebe7f228884387 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-13T14:22:42.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 podman[32514]: 2024-09-13 14:22:42.158623515 +0000 UTC m=+0.467439898 container start a37b0845979d4d4a3406a0b68c169aed006fdda0ae2ea5c448ebe7f228884387 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-13T14:22:42.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:22:42.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 2 2024-09-13T14:22:42.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: pidfile_write: ignore empty --pid-file 2024-09-13T14:22:42.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: load: jerasure load: lrc 2024-09-13T14:22:42.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: RocksDB version: 6.15.5 2024-09-13T14:22:42.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T14:22:42.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Compile date Sep 12 2024 2024-09-13T14:22:42.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: DB SUMMARY 2024-09-13T14:22:42.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: DB Session ID: MEG1IK9THT23CO5CDWWJ 2024-09-13T14:22:42.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:22:42.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:22:42.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T14:22:42.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-09-13T14:22:42.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-09-13T14:22:42.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:22:42.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:22:42.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:22:42.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:22:42.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.env: 0x5580270bf440 2024-09-13T14:22:42.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.fs: Posix File System 2024-09-13T14:22:42.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.info_log: 0x558028500ec0 2024-09-13T14:22:42.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:22:42.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.statistics: (nil) 2024-09-13T14:22:42.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.use_fsync: 0 2024-09-13T14:22:42.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:22:42.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:22:42.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:22:42.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:22:42.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:22:42.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:22:42.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:22:42.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:22:42.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:22:42.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:22:42.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:22:42.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.db_log_dir: 2024-09-13T14:22:42.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-09-13T14:22:42.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:22:42.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:22:42.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:22:42.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:22:42.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:22:42.473 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:22:42.473 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:22:42.473 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:22:42.473 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.write_buffer_manager: 0x5580285be7e0 2024-09-13T14:22:42.473 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:22:42.473 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T14:22:42.473 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:22:42.473 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:22:42.473 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:22:42.473 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:22:42.473 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:22:42.473 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:22:42.473 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:22:42.474 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.unordered_write: 0 2024-09-13T14:22:42.474 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:22:42.474 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:22:42.474 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:22:42.474 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:22:42.474 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.row_cache: None 2024-09-13T14:22:42.474 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.wal_filter: None 2024-09-13T14:22:42.474 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:22:42.475 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:22:42.475 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.preserve_deletes: 0 2024-09-13T14:22:42.475 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:22:42.476 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:22:42.476 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:22:42.476 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:22:42.476 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:22:42.476 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:22:42.476 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:22:42.476 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:22:42.476 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:22:42.476 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:22:42.476 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:22:42.476 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:22:42.476 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:22:42.476 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:22:42.477 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:22:42.477 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:22:42.477 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:22:42.477 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:22:42.477 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:22:42.477 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:22:42.477 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:22:42.477 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:22:42.477 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:22:42.477 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:22:42.477 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_open_files: -1 2024-09-13T14:22:42.477 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:22:42.477 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:22:42.477 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:22:42.477 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:22:42.478 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:22:42.478 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Compression algorithms supported: 2024-09-13T14:22:42.478 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: kZSTD supported: 0 2024-09-13T14:22:42.478 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:22:42.478 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:22:42.478 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:22:42.478 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:22:42.478 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:22:42.478 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:22:42.478 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:22:42.478 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:22:42.478 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-09-13T14:22:42.478 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T14:22:42.478 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:22:42.479 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.merge_operator: 2024-09-13T14:22:42.479 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compaction_filter: None 2024-09-13T14:22:42.479 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:22:42.479 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:22:42.479 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:22:42.479 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:22:42.479 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5580284a7a38) 2024-09-13T14:22:42.479 INFO:journalctl@ceph.mon.b.smithi204.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:22:42.479 INFO:journalctl@ceph.mon.b.smithi204.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:22:42.479 INFO:journalctl@ceph.mon.b.smithi204.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:22:42.479 INFO:journalctl@ceph.mon.b.smithi204.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:22:42.479 INFO:journalctl@ceph.mon.b.smithi204.stdout: index_type: 0 2024-09-13T14:22:42.479 INFO:journalctl@ceph.mon.b.smithi204.stdout: data_block_index_type: 0 2024-09-13T14:22:42.479 INFO:journalctl@ceph.mon.b.smithi204.stdout: index_shortening: 1 2024-09-13T14:22:42.479 INFO:journalctl@ceph.mon.b.smithi204.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:22:42.480 INFO:journalctl@ceph.mon.b.smithi204.stdout: hash_index_allow_collision: 1 2024-09-13T14:22:42.480 INFO:journalctl@ceph.mon.b.smithi204.stdout: checksum: 1 2024-09-13T14:22:42.480 INFO:journalctl@ceph.mon.b.smithi204.stdout: no_block_cache: 0 2024-09-13T14:22:42.480 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_cache: 0x5580284dd1f0 2024-09-13T14:22:42.480 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:22:42.480 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_cache_options: 2024-09-13T14:22:42.480 INFO:journalctl@ceph.mon.b.smithi204.stdout: capacity : 536870912 2024-09-13T14:22:42.480 INFO:journalctl@ceph.mon.b.smithi204.stdout: num_shard_bits : 4 2024-09-13T14:22:42.480 INFO:journalctl@ceph.mon.b.smithi204.stdout: strict_capacity_limit : 0 2024-09-13T14:22:42.480 INFO:journalctl@ceph.mon.b.smithi204.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:22:42.480 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_cache_compressed: (nil) 2024-09-13T14:22:42.480 INFO:journalctl@ceph.mon.b.smithi204.stdout: persistent_cache: (nil) 2024-09-13T14:22:42.480 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_size: 4096 2024-09-13T14:22:42.481 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_size_deviation: 10 2024-09-13T14:22:42.481 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_restart_interval: 16 2024-09-13T14:22:42.481 INFO:journalctl@ceph.mon.b.smithi204.stdout: index_block_restart_interval: 1 2024-09-13T14:22:42.481 INFO:journalctl@ceph.mon.b.smithi204.stdout: metadata_block_size: 4096 2024-09-13T14:22:42.481 INFO:journalctl@ceph.mon.b.smithi204.stdout: partition_filters: 0 2024-09-13T14:22:42.481 INFO:journalctl@ceph.mon.b.smithi204.stdout: use_delta_encoding: 1 2024-09-13T14:22:42.481 INFO:journalctl@ceph.mon.b.smithi204.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T14:22:42.481 INFO:journalctl@ceph.mon.b.smithi204.stdout: whole_key_filtering: 1 2024-09-13T14:22:42.481 INFO:journalctl@ceph.mon.b.smithi204.stdout: verify_compression: 0 2024-09-13T14:22:42.481 INFO:journalctl@ceph.mon.b.smithi204.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:22:42.481 INFO:journalctl@ceph.mon.b.smithi204.stdout: format_version: 4 2024-09-13T14:22:42.481 INFO:journalctl@ceph.mon.b.smithi204.stdout: enable_index_compression: 1 2024-09-13T14:22:42.481 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_align: 0 2024-09-13T14:22:42.481 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:22:42.481 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:22:42.482 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compression: NoCompression 2024-09-13T14:22:42.482 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:22:42.482 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:22:42.482 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:22:42.482 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.num_levels: 7 2024-09-13T14:22:42.482 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:22:42.482 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:22:42.482 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:22:42.482 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:22:42.482 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:22:42.482 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:22:42.482 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:22:42.482 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:22:42.483 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:22:42.483 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:22:42.483 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:22:42.483 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:22:42.483 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:22:42.483 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:22:42.483 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:22:42.483 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:22:42.483 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:22:42.483 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:22:42.483 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:22:42.483 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:22:42.483 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:22:42.483 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:22:42.484 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:22:42.484 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:22:42.484 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:22:42.484 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:22:42.484 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:22:42.484 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:22:42.484 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:22:42.484 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:22:42.484 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:22:42.484 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:22:42.484 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:22:42.484 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:22:42.484 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T14:22:42.484 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:22:42.485 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:22:42.485 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T14:22:42.485 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:22:42.485 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:22:42.485 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:22:42.485 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:22:42.485 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:22:42.485 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:22:42.485 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:22:42.485 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:22:42.485 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:22:42.485 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:22:42.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:22:42.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:22:42.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:22:42.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:22:42.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:22:42.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:22:42.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:22:42.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:22:42.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:22:42.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:22:42.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:22:42.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:22:42.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:22:42.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.ttl: 2592000 2024-09-13T14:22:42.487 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:22:42.487 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:22:42.487 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:22:42.487 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:22:42.487 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:22:42.487 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:22:42.487 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:22:42.487 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:22:42.487 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-13T14:22:42.487 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-13T14:22:42.488 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237362183155, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T14:22:42.488 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-13T14:22:42.488 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T14:22:42.488 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237362183479, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1571, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726237362, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "751cd824-e39f-4ea8-8f14-a7dccdbb0b52", "db_session_id": "MEG1IK9THT23CO5CDWWJ"}} 2024-09-13T14:22:42.488 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-13T14:22:42.488 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237362183854, "job": 1, "event": "recovery_finished"} 2024-09-13T14:22:42.488 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:22:42.488 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x558028510700 2024-09-13T14:22:42.488 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: DB pointer 0x5580284e0000 2024-09-13T14:22:42.488 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T14:22:42.488 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T14:22:42.488 INFO:journalctl@ceph.mon.b.smithi204.stdout: ** DB Stats ** 2024-09-13T14:22:42.488 INFO:journalctl@ceph.mon.b.smithi204.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:22:42.488 INFO:journalctl@ceph.mon.b.smithi204.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:22:42.489 INFO:journalctl@ceph.mon.b.smithi204.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:22:42.489 INFO:journalctl@ceph.mon.b.smithi204.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:22:42.489 INFO:journalctl@ceph.mon.b.smithi204.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:22:42.489 INFO:journalctl@ceph.mon.b.smithi204.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T14:22:42.489 INFO:journalctl@ceph.mon.b.smithi204.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:22:42.489 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:22:42.489 INFO:journalctl@ceph.mon.b.smithi204.stdout: ** Compaction Stats [default] ** 2024-09-13T14:22:42.489 INFO:journalctl@ceph.mon.b.smithi204.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:22:42.489 INFO:journalctl@ceph.mon.b.smithi204.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:22:42.489 INFO:journalctl@ceph.mon.b.smithi204.stdout: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-09-13T14:22:42.489 INFO:journalctl@ceph.mon.b.smithi204.stdout: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-09-13T14:22:42.489 INFO:journalctl@ceph.mon.b.smithi204.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-09-13T14:22:42.489 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:22:42.490 INFO:journalctl@ceph.mon.b.smithi204.stdout: ** Compaction Stats [default] ** 2024-09-13T14:22:42.490 INFO:journalctl@ceph.mon.b.smithi204.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:22:42.490 INFO:journalctl@ceph.mon.b.smithi204.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:22:42.490 INFO:journalctl@ceph.mon.b.smithi204.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-09-13T14:22:42.490 INFO:journalctl@ceph.mon.b.smithi204.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:22:42.490 INFO:journalctl@ceph.mon.b.smithi204.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:22:42.490 INFO:journalctl@ceph.mon.b.smithi204.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:22:42.490 INFO:journalctl@ceph.mon.b.smithi204.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:22:42.490 INFO:journalctl@ceph.mon.b.smithi204.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:22:42.490 INFO:journalctl@ceph.mon.b.smithi204.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:22:42.490 INFO:journalctl@ceph.mon.b.smithi204.stdout: Cumulative compaction: 0.00 GB write, 0.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:22:42.490 INFO:journalctl@ceph.mon.b.smithi204.stdout: Interval compaction: 0.00 GB write, 0.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:22:42.490 INFO:journalctl@ceph.mon.b.smithi204.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:22:42.491 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:22:42.491 INFO:journalctl@ceph.mon.b.smithi204.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:22:42.491 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:22:42.491 INFO:journalctl@ceph.mon.b.smithi204.stdout: ** Compaction Stats [default] ** 2024-09-13T14:22:42.491 INFO:journalctl@ceph.mon.b.smithi204.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:22:42.491 INFO:journalctl@ceph.mon.b.smithi204.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:22:42.491 INFO:journalctl@ceph.mon.b.smithi204.stdout: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-09-13T14:22:42.491 INFO:journalctl@ceph.mon.b.smithi204.stdout: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-09-13T14:22:42.491 INFO:journalctl@ceph.mon.b.smithi204.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:22:42.491 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:22:42.491 INFO:journalctl@ceph.mon.b.smithi204.stdout: ** Compaction Stats [default] ** 2024-09-13T14:22:42.491 INFO:journalctl@ceph.mon.b.smithi204.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:22:42.491 INFO:journalctl@ceph.mon.b.smithi204.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:22:42.491 INFO:journalctl@ceph.mon.b.smithi204.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-09-13T14:22:42.492 INFO:journalctl@ceph.mon.b.smithi204.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:22:42.492 INFO:journalctl@ceph.mon.b.smithi204.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:22:42.492 INFO:journalctl@ceph.mon.b.smithi204.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:22:42.492 INFO:journalctl@ceph.mon.b.smithi204.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:22:42.492 INFO:journalctl@ceph.mon.b.smithi204.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:22:42.492 INFO:journalctl@ceph.mon.b.smithi204.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:22:42.492 INFO:journalctl@ceph.mon.b.smithi204.stdout: Cumulative compaction: 0.00 GB write, 0.50 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:22:42.492 INFO:journalctl@ceph.mon.b.smithi204.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:22:42.492 INFO:journalctl@ceph.mon.b.smithi204.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:22:42.492 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:22:42.492 INFO:journalctl@ceph.mon.b.smithi204.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:22:42.492 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-09-13T14:22:42.492 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: using public_addr v2:172.21.15.204:0/0 -> [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] 2024-09-13T14:22:42.493 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: starting mon.b rank -1 at public addrs [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] at bind addrs [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:22:42.493 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: mon.b@-1(???) e0 preinit fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:22:42.493 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: mon.b@-1(synchronizing).mds e1 new map 2024-09-13T14:22:42.493 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: mon.b@-1(synchronizing).mds e1 print_map 2024-09-13T14:22:42.493 INFO:journalctl@ceph.mon.b.smithi204.stdout: e1 2024-09-13T14:22:42.493 INFO:journalctl@ceph.mon.b.smithi204.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:22:42.493 INFO:journalctl@ceph.mon.b.smithi204.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:22:42.493 INFO:journalctl@ceph.mon.b.smithi204.stdout: legacy client fscid: -1 2024-09-13T14:22:42.493 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:22:42.493 INFO:journalctl@ceph.mon.b.smithi204.stdout: No filesystems configured 2024-09-13T14:22:42.493 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T14:22:42.493 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T14:22:42.493 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T14:22:42.493 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T14:22:42.494 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T14:22:42.494 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T14:22:42.494 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T14:22:42.494 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:22:42.494 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:22:42.494 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:22:42.494 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:42.494 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:42.494 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:42.496 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:42.496 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:42.496 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:42.496 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:42.496 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:42.496 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi096:172.21.15.96=a;smithi096:[v2:172.21.15.96:3301,v1:172.21.15.96:6790]=c;smithi204:172.21.15.204=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:42.496 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: Saving service mon spec with placement smithi096:172.21.15.96=a;smithi096:[v2:172.21.15.96:3301,v1:172.21.15.96:6790]=c;smithi204:172.21.15.204=b;count:3 2024-09-13T14:22:42.496 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: Deploying daemon mon.b on smithi204 2024-09-13T14:22:42.496 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:42.496 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:42.496 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 ceph-mon[32546]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-13T14:22:42.497 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 bash[32514]: a37b0845979d4d4a3406a0b68c169aed006fdda0ae2ea5c448ebe7f228884387 2024-09-13T14:22:42.497 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:42 smithi204 systemd[1]: Started Ceph mon.b for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:22:43.726 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T14:22:43.726 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T14:22:43.987 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:22:47.522 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:47.522 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:47.522 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: mon.a calling monitor election 2024-09-13T14:22:47.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:47.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:47.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:47.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: mon.b calling monitor election 2024-09-13T14:22:47.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:47.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:47.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:47.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:47.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:47.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:22:47.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: monmap e2: 2 mons at {a=[v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0],b=[v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:22:47.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: fsmap 2024-09-13T14:22:47.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:22:47.524 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: mgrmap e13: y(active, since 39s) 2024-09-13T14:22:47.524 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: overall HEALTH_OK 2024-09-13T14:22:47.524 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:47.524 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:47.524 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:47.524 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:47.524 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:47 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:47.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: mon.a calling monitor election 2024-09-13T14:22:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: mon.b calling monitor election 2024-09-13T14:22:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:22:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: monmap e2: 2 mons at {a=[v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0],b=[v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:22:47.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: fsmap 2024-09-13T14:22:47.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:22:47.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: mgrmap e13: y(active, since 39s) 2024-09-13T14:22:47.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: overall HEALTH_OK 2024-09-13T14:22:47.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:47.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:47.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:47.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:47.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:47 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:48.549 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:48 smithi096 ceph-mon[29229]: Deploying daemon mon.c on smithi096 2024-09-13T14:22:48.549 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:48 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:48.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:48 smithi204 ceph-mon[32546]: Deploying daemon mon.c on smithi096 2024-09-13T14:22:48.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:48 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:48.757 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:22:48.757 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":2,"fsid":"51e50ed6-71db-11ef-bceb-c7b262605968","modified":"2024-09-13T14:22:42.206993Z","created":"2024-09-13T14:21:13.478657Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:3300","nonce":0},{"type":"v1","addr":"172.21.15.96:6789","nonce":0}]},"addr":"172.21.15.96:6789/0","public_addr":"172.21.15.96:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:3300","nonce":0},{"type":"v1","addr":"172.21.15.204:6789","nonce":0}]},"addr":"172.21.15.204:6789/0","public_addr":"172.21.15.204:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-13T14:22:48.757 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 2 2024-09-13T14:22:49.330 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:49 smithi204 ceph-mon[32546]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:49.330 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:49 smithi204 ceph-mon[32546]: from='client.? 172.21.15.204:0/1938816353' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:22:49.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:49 smithi096 ceph-mon[29229]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:49.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:49 smithi096 ceph-mon[29229]: from='client.? 172.21.15.204:0/1938816353' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:22:50.279 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 podman[39618]: 2024-09-13 14:22:50.223090345 +0000 UTC m=+0.125675462 container create 44a18794779ce339a72b7a128c5a8fe5eb09b65f67484a9ab4a3cbb4c6d52f64 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-13T14:22:50.358 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T14:22:50.361 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T14:22:50.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 podman[39618]: 2024-09-13 14:22:50.417412558 +0000 UTC m=+0.319997682 container init 44a18794779ce339a72b7a128c5a8fe5eb09b65f67484a9ab4a3cbb4c6d52f64 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:22:50.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 podman[39618]: 2024-09-13 14:22:50.420794586 +0000 UTC m=+0.323379701 container start 44a18794779ce339a72b7a128c5a8fe5eb09b65f67484a9ab4a3cbb4c6d52f64 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-13T14:22:50.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:22:50.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 2 2024-09-13T14:22:50.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: pidfile_write: ignore empty --pid-file 2024-09-13T14:22:50.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: load: jerasure load: lrc 2024-09-13T14:22:50.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: RocksDB version: 6.15.5 2024-09-13T14:22:50.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T14:22:50.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Compile date Sep 12 2024 2024-09-13T14:22:50.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: DB SUMMARY 2024-09-13T14:22:50.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: DB Session ID: H0Q5Q7TS1QWAXZ4KK80E 2024-09-13T14:22:50.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:22:50.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:22:50.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T14:22:50.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-13T14:22:50.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-09-13T14:22:50.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:22:50.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:22:50.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:22:50.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:22:50.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.env: 0x5646d99ac440 2024-09-13T14:22:50.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.fs: Posix File System 2024-09-13T14:22:50.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.info_log: 0x5646da566ec0 2024-09-13T14:22:50.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:22:50.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.statistics: (nil) 2024-09-13T14:22:50.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.use_fsync: 0 2024-09-13T14:22:50.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:22:50.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:22:50.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:22:50.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:22:50.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:22:50.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:22:50.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:22:50.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:22:50.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:22:50.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:22:50.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:22:50.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.db_log_dir: 2024-09-13T14:22:50.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-09-13T14:22:50.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:22:50.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:22:50.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:22:50.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:22:50.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:22:50.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:22:50.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:22:50.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:22:50.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.write_buffer_manager: 0x5646da6247b0 2024-09-13T14:22:50.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:22:50.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T14:22:50.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:22:50.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:22:50.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:22:50.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:22:50.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:22:50.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:22:50.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:22:50.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.unordered_write: 0 2024-09-13T14:22:50.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:22:50.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:22:50.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:22:50.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:22:50.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.row_cache: None 2024-09-13T14:22:50.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.wal_filter: None 2024-09-13T14:22:50.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:22:50.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:22:50.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.preserve_deletes: 0 2024-09-13T14:22:50.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:22:50.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:22:50.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:22:50.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:22:50.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:22:50.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:22:50.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:22:50.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:22:50.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:22:50.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:22:50.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:22:50.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:22:50.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:22:50.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:22:50.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:22:50.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:22:50.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:22:50.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:22:50.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:22:50.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:22:50.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:22:50.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:22:50.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:22:50.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:22:50.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_open_files: -1 2024-09-13T14:22:50.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:22:50.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:22:50.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:22:50.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:22:50.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:22:50.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Compression algorithms supported: 2024-09-13T14:22:50.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: kZSTD supported: 0 2024-09-13T14:22:50.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:22:50.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:22:50.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:22:50.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:22:50.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:22:50.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:22:50.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:22:50.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:22:50.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-09-13T14:22:50.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T14:22:50.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:22:50.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.merge_operator: 2024-09-13T14:22:50.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compaction_filter: None 2024-09-13T14:22:50.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:22:50.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:22:50.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:22:50.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:22:50.583 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5646da50da38) 2024-09-13T14:22:50.583 INFO:journalctl@ceph.mon.c.smithi096.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:22:50.583 INFO:journalctl@ceph.mon.c.smithi096.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:22:50.583 INFO:journalctl@ceph.mon.c.smithi096.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:22:50.583 INFO:journalctl@ceph.mon.c.smithi096.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:22:50.583 INFO:journalctl@ceph.mon.c.smithi096.stdout: index_type: 0 2024-09-13T14:22:50.583 INFO:journalctl@ceph.mon.c.smithi096.stdout: data_block_index_type: 0 2024-09-13T14:22:50.583 INFO:journalctl@ceph.mon.c.smithi096.stdout: index_shortening: 1 2024-09-13T14:22:50.583 INFO:journalctl@ceph.mon.c.smithi096.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:22:50.583 INFO:journalctl@ceph.mon.c.smithi096.stdout: hash_index_allow_collision: 1 2024-09-13T14:22:50.583 INFO:journalctl@ceph.mon.c.smithi096.stdout: checksum: 1 2024-09-13T14:22:50.583 INFO:journalctl@ceph.mon.c.smithi096.stdout: no_block_cache: 0 2024-09-13T14:22:50.583 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_cache: 0x5646da5431f0 2024-09-13T14:22:50.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:22:50.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_cache_options: 2024-09-13T14:22:50.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: capacity : 536870912 2024-09-13T14:22:50.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: num_shard_bits : 4 2024-09-13T14:22:50.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: strict_capacity_limit : 0 2024-09-13T14:22:50.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:22:50.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_cache_compressed: (nil) 2024-09-13T14:22:50.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: persistent_cache: (nil) 2024-09-13T14:22:50.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_size: 4096 2024-09-13T14:22:50.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_size_deviation: 10 2024-09-13T14:22:50.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_restart_interval: 16 2024-09-13T14:22:50.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: index_block_restart_interval: 1 2024-09-13T14:22:50.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: metadata_block_size: 4096 2024-09-13T14:22:50.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: partition_filters: 0 2024-09-13T14:22:50.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: use_delta_encoding: 1 2024-09-13T14:22:50.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T14:22:50.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: whole_key_filtering: 1 2024-09-13T14:22:50.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: verify_compression: 0 2024-09-13T14:22:50.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:22:50.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: format_version: 4 2024-09-13T14:22:50.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: enable_index_compression: 1 2024-09-13T14:22:50.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_align: 0 2024-09-13T14:22:50.585 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:22:50.585 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:22:50.585 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compression: NoCompression 2024-09-13T14:22:50.585 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:22:50.586 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:22:50.586 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:22:50.586 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.num_levels: 7 2024-09-13T14:22:50.586 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:22:50.586 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:22:50.586 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:22:50.586 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:22:50.586 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:22:50.586 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:22:50.586 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:22:50.586 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:22:50.586 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:22:50.586 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:22:50.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:22:50.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:22:50.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:22:50.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:22:50.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:22:50.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:22:50.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:22:50.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:22:50.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:22:50.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:22:50.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:22:50.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:22:50.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:22:50.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:22:50.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:22:50.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:22:50.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:22:50.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:22:50.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:22:50.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:22:50.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:22:50.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:22:50.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:22:50.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:22:50.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T14:22:50.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:22:50.589 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:22:50.589 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T14:22:50.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:22:50.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:22:50.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:22:50.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:22:50.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:22:50.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:22:50.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:22:50.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:22:50.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:22:50.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:22:50.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:22:50.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:22:50.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:22:50.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:22:50.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:22:50.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:22:50.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:22:50.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:22:50.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:22:50.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:22:50.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:22:50.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:22:50.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:22:50.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.ttl: 2592000 2024-09-13T14:22:50.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:22:50.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:22:50.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:22:50.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:22:50.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:22:50.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:22:50.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:22:50.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:22:50.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-13T14:22:50.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-13T14:22:50.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237370444598, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T14:22:50.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-13T14:22:50.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T14:22:50.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237370444897, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1696, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726237370, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "23a2321b-de2b-4a7a-8e5d-cecc562acfad", "db_session_id": "H0Q5Q7TS1QWAXZ4KK80E"}} 2024-09-13T14:22:50.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-13T14:22:50.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237370445210, "job": 1, "event": "recovery_finished"} 2024-09-13T14:22:50.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:22:50.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5646da576700 2024-09-13T14:22:50.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: DB pointer 0x5646da546000 2024-09-13T14:22:50.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T14:22:50.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T14:22:50.593 INFO:journalctl@ceph.mon.c.smithi096.stdout: ** DB Stats ** 2024-09-13T14:22:50.593 INFO:journalctl@ceph.mon.c.smithi096.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:22:50.593 INFO:journalctl@ceph.mon.c.smithi096.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:22:50.593 INFO:journalctl@ceph.mon.c.smithi096.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:22:50.593 INFO:journalctl@ceph.mon.c.smithi096.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:22:50.593 INFO:journalctl@ceph.mon.c.smithi096.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:22:50.593 INFO:journalctl@ceph.mon.c.smithi096.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T14:22:50.593 INFO:journalctl@ceph.mon.c.smithi096.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:22:50.593 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:22:50.594 INFO:journalctl@ceph.mon.c.smithi096.stdout: ** Compaction Stats [default] ** 2024-09-13T14:22:50.594 INFO:journalctl@ceph.mon.c.smithi096.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:22:50.594 INFO:journalctl@ceph.mon.c.smithi096.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:22:50.594 INFO:journalctl@ceph.mon.c.smithi096.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-13T14:22:50.594 INFO:journalctl@ceph.mon.c.smithi096.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-13T14:22:50.594 INFO:journalctl@ceph.mon.c.smithi096.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-13T14:22:50.594 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:22:50.594 INFO:journalctl@ceph.mon.c.smithi096.stdout: ** Compaction Stats [default] ** 2024-09-13T14:22:50.594 INFO:journalctl@ceph.mon.c.smithi096.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:22:50.594 INFO:journalctl@ceph.mon.c.smithi096.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:22:50.594 INFO:journalctl@ceph.mon.c.smithi096.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-13T14:22:50.594 INFO:journalctl@ceph.mon.c.smithi096.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:22:50.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:22:50.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:22:50.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:22:50.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:22:50.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:22:50.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:22:50.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: Interval compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:22:50.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:22:50.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:22:50.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:22:50.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:22:50.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: ** Compaction Stats [default] ** 2024-09-13T14:22:50.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:22:50.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:22:50.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-13T14:22:50.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-13T14:22:50.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:22:50.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:22:50.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: ** Compaction Stats [default] ** 2024-09-13T14:22:50.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:22:50.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:22:50.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-13T14:22:50.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:22:50.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:22:50.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:22:50.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:22:50.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:22:50.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:22:50.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:22:50.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:22:50.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:22:50.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:22:50.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:22:50.597 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-13T14:22:50.597 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: using public_addrv [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0] 2024-09-13T14:22:50.597 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: starting mon.c rank -1 at public addrs [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0] at bind addrs [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:22:50.597 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.c@-1(???) e0 preinit fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:22:50.597 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.c@-1(synchronizing).mds e1 new map 2024-09-13T14:22:50.597 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.c@-1(synchronizing).mds e1 print_map 2024-09-13T14:22:50.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: e1 2024-09-13T14:22:50.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:22:50.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:22:50.598 INFO:journalctl@ceph.mon.c.smithi096.stdout: legacy client fscid: -1 2024-09-13T14:22:50.598 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:22:50.598 INFO:journalctl@ceph.mon.c.smithi096.stdout: No filesystems configured 2024-09-13T14:22:50.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T14:22:50.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T14:22:50.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T14:22:50.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T14:22:50.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T14:22:50.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T14:22:50.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T14:22:50.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:22:50.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:22:50.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:22:50.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:50.599 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:50.599 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:50.599 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:50.599 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:50.599 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:50.599 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:50.599 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:50.599 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi096:172.21.15.96=a;smithi096:[v2:172.21.15.96:3301,v1:172.21.15.96:6790]=c;smithi204:172.21.15.204=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:50.599 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: Saving service mon spec with placement smithi096:172.21.15.96=a;smithi096:[v2:172.21.15.96:3301,v1:172.21.15.96:6790]=c;smithi204:172.21.15.204=b;count:3 2024-09-13T14:22:50.599 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: Deploying daemon mon.b on smithi204 2024-09-13T14:22:50.599 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:50.599 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:50.599 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:50.600 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:50.600 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.a calling monitor election 2024-09-13T14:22:50.600 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:50.600 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:50.600 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:50.600 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.b calling monitor election 2024-09-13T14:22:50.600 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:50.600 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:50.600 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:50.600 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:50.600 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:50.600 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:22:50.600 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: monmap e2: 2 mons at {a=[v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0],b=[v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:22:50.600 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: fsmap 2024-09-13T14:22:50.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:22:50.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mgrmap e13: y(active, since 39s) 2024-09-13T14:22:50.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: overall HEALTH_OK 2024-09-13T14:22:50.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:50.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:50.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:50.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:50.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:50.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: Deploying daemon mon.c on smithi096 2024-09-13T14:22:50.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:50.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:50.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: from='client.? 172.21.15.204:0/1938816353' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:22:50.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 ceph-mon[39645]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-13T14:22:50.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 bash[39618]: 44a18794779ce339a72b7a128c5a8fe5eb09b65f67484a9ab4a3cbb4c6d52f64 2024-09-13T14:22:50.602 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:50 smithi096 systemd[1]: Started Ceph mon.c for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:22:50.616 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:22:55.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:55.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:55.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: mon.a calling monitor election 2024-09-13T14:22:55.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: mon.b calling monitor election 2024-09-13T14:22:55.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:55.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:55.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: mon.c calling monitor election 2024-09-13T14:22:55.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:55.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:22:55.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: monmap e3: 3 mons at {a=[v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0],b=[v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0],c=[v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:22:55.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: fsmap 2024-09-13T14:22:55.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:22:55.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: mgrmap e13: y(active, since 47s) 2024-09-13T14:22:55.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: overall HEALTH_OK 2024-09-13T14:22:55.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:55.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:55.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:55.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:55.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:55.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:55.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: mon.a calling monitor election 2024-09-13T14:22:55.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: mon.b calling monitor election 2024-09-13T14:22:55.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:55.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:55.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: mon.c calling monitor election 2024-09-13T14:22:55.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:55.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:22:55.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: monmap e3: 3 mons at {a=[v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0],b=[v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0],c=[v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:22:55.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: fsmap 2024-09-13T14:22:55.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:22:55.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: mgrmap e13: y(active, since 47s) 2024-09-13T14:22:55.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: overall HEALTH_OK 2024-09-13T14:22:55.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:55.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:55.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:55.827 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:55.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: mon.a calling monitor election 2024-09-13T14:22:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: mon.b calling monitor election 2024-09-13T14:22:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: mon.c calling monitor election 2024-09-13T14:22:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:55.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:22:55.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: monmap e3: 3 mons at {a=[v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0],b=[v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0],c=[v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:22:55.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: fsmap 2024-09-13T14:22:55.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:22:55.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: mgrmap e13: y(active, since 47s) 2024-09-13T14:22:55.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: overall HEALTH_OK 2024-09-13T14:22:55.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:55.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:55.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:55.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:55.989 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:22:55.992 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":3,"fsid":"51e50ed6-71db-11ef-bceb-c7b262605968","modified":"2024-09-13T14:22:50.469402Z","created":"2024-09-13T14:21:13.478657Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:3300","nonce":0},{"type":"v1","addr":"172.21.15.96:6789","nonce":0}]},"addr":"172.21.15.96:6789/0","public_addr":"172.21.15.96:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:3300","nonce":0},{"type":"v1","addr":"172.21.15.204:6789","nonce":0}]},"addr":"172.21.15.204:6789/0","public_addr":"172.21.15.204:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:3301","nonce":0},{"type":"v1","addr":"172.21.15.96:6790","nonce":0}]},"addr":"172.21.15.96:6790/0","public_addr":"172.21.15.96:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-13T14:22:55.992 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 3 2024-09-13T14:22:56.592 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:56 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:56.592 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:56 smithi204 ceph-mon[32546]: from='client.? 172.21.15.204:0/1935033086' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:22:56.592 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:56 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:56.617 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-13T14:22:56.617 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-13T14:22:56.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:56 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:56.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:56 smithi096 ceph-mon[29229]: from='client.? 172.21.15.204:0/1935033086' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:22:56.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:56 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:56.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:56 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:56.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:56 smithi096 ceph-mon[39645]: from='client.? 172.21.15.204:0/1935033086' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:22:56.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:56 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:56.869 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:57.945 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[29229]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:57.945 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.945 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:57.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:57.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:57.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:22:57.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[39645]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.948 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:57.948 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:22:57.948 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:57 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:57.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:57 smithi204 ceph-mon[32546]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:57.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:57 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:57 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:57 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:57 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:57 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:57 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:57 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:57 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:57 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:57 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:57 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:57 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:57 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:22:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:57 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:58.960 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:58 smithi096 ceph-mon[29229]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:22:58.960 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:58 smithi096 ceph-mon[29229]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:22:58.960 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:58 smithi096 ceph-mon[29229]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:58.960 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:58 smithi096 ceph-mon[29229]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:58.960 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:58 smithi096 ceph-mon[29229]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T14:22:58.960 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:58 smithi096 ceph-mon[29229]: Reconfiguring daemon mon.a on smithi096 2024-09-13T14:22:58.960 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:58 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:58.961 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:58 smithi096 ceph-mon[39645]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:22:58.961 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:58 smithi096 ceph-mon[39645]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:22:58.961 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:58 smithi096 ceph-mon[39645]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:58.961 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:58 smithi096 ceph-mon[39645]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:58.961 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:58 smithi096 ceph-mon[39645]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T14:22:58.961 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:58 smithi096 ceph-mon[39645]: Reconfiguring daemon mon.a on smithi096 2024-09-13T14:22:58.961 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:58 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:59.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:58 smithi204 ceph-mon[32546]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:22:59.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:58 smithi204 ceph-mon[32546]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:22:59.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:58 smithi204 ceph-mon[32546]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:59.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:58 smithi204 ceph-mon[32546]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:59.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:58 smithi204 ceph-mon[32546]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T14:22:59.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:58 smithi204 ceph-mon[32546]: Reconfiguring daemon mon.a on smithi096 2024-09-13T14:22:59.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:58 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:59.589 INFO:teuthology.orchestra.run.smithi096.stdout:# minimal ceph.conf for 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:22:59.589 INFO:teuthology.orchestra.run.smithi096.stdout:[global] 2024-09-13T14:22:59.589 INFO:teuthology.orchestra.run.smithi096.stdout: fsid = 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:22:59.589 INFO:teuthology.orchestra.run.smithi096.stdout: mon_host = [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0] 2024-09-13T14:22:59.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:59 smithi096 ceph-mon[29229]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:59.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:59 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:59.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:59 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:59.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:59 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:59.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:59 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:22:59.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:59 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:59.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:22:59 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1437237591' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:59.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:59 smithi096 ceph-mon[39645]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:22:59.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:59 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:59.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:59 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:22:59.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:59 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:59.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:59 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:22:59.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:59 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:59.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:22:59 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1437237591' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:00.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:59 smithi204 ceph-mon[32546]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:00.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:59 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:00.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:59 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:00.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:59 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:23:00.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:59 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:23:00.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:59 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:00.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:22:59 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1437237591' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:01.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:00 smithi096 ceph-mon[29229]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:23:01.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:00 smithi096 ceph-mon[29229]: Reconfiguring daemon mon.c on smithi096 2024-09-13T14:23:01.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:00 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:01.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:00 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:01.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:00 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:23:01.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:00 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:23:01.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:00 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:01.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:00 smithi096 ceph-mon[39645]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:23:01.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:00 smithi096 ceph-mon[39645]: Reconfiguring daemon mon.c on smithi096 2024-09-13T14:23:01.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:00 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:01.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:00 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:01.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:00 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:23:01.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:00 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:23:01.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:00 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:01.083 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:00 smithi204 ceph-mon[32546]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:23:01.083 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:00 smithi204 ceph-mon[32546]: Reconfiguring daemon mon.c on smithi096 2024-09-13T14:23:01.083 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:00 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:01.083 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:00 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:01.083 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:00 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:23:01.083 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:00 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:23:01.083 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:00 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:01.205 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-13T14:23:01.206 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:23:01.206 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T14:23:01.233 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:23:01.233 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:23:01.297 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-09-13T14:23:01.298 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T14:23:01.327 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-09-13T14:23:01.327 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:23:01.392 INFO:tasks.cephadm:Adding mgr.y on smithi096 2024-09-13T14:23:01.393 INFO:tasks.cephadm:Adding mgr.x on smithi204 2024-09-13T14:23:01.393 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch apply mgr '2;smithi096=y;smithi204=x' 2024-09-13T14:23:02.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:01 smithi096 ceph-mon[29229]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:02.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:01 smithi096 ceph-mon[29229]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:23:02.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:01 smithi096 ceph-mon[29229]: Reconfiguring daemon mon.b on smithi204 2024-09-13T14:23:02.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:01 smithi096 ceph-mon[39645]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:02.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:01 smithi096 ceph-mon[39645]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:23:02.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:01 smithi096 ceph-mon[39645]: Reconfiguring daemon mon.b on smithi204 2024-09-13T14:23:02.090 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:01 smithi204 ceph-mon[32546]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:02.090 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:01 smithi204 ceph-mon[32546]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:23:02.091 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:01 smithi204 ceph-mon[32546]: Reconfiguring daemon mon.b on smithi204 2024-09-13T14:23:02.116 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:23:03.366 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:03 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:03.366 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:03 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:03.366 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:03 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:03.366 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:03 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:03.367 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:03 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:03.367 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:03 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:03.367 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:03 smithi204 ceph-mon[32546]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:03.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:03 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:03.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:03 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:03 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:03 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:03 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:03 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:03 smithi096 ceph-mon[29229]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:03.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:03 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:03.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:03 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:03.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:03 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:03.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:03 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:03.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:03 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:03.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:03 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:03.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:03 smithi096 ceph-mon[39645]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:03.717 INFO:teuthology.orchestra.run.smithi204.stdout:Scheduled mgr update... 2024-09-13T14:23:04.216 DEBUG:teuthology.orchestra.run.smithi204:mgr.x> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.x.service 2024-09-13T14:23:04.218 DEBUG:tasks.cephadm:set 0 configs 2024-09-13T14:23:04.218 INFO:tasks.cephadm:Deploying OSDs... 2024-09-13T14:23:04.219 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:23:04.219 DEBUG:teuthology.orchestra.run.smithi096:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T14:23:04.237 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T14:23:04.237 DEBUG:teuthology.orchestra.run.smithi096:> stat /dev/vg_nvme/lv_1 2024-09-13T14:23:04.295 INFO:teuthology.orchestra.run.smithi096.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T14:23:04.295 INFO:teuthology.orchestra.run.smithi096.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:23:04.295 INFO:teuthology.orchestra.run.smithi096.stdout:Device: 5h/5d Inode: 971 Links: 1 2024-09-13T14:23:04.295 INFO:teuthology.orchestra.run.smithi096.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:23:04.295 INFO:teuthology.orchestra.run.smithi096.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:23:04.295 INFO:teuthology.orchestra.run.smithi096.stdout:Access: 2024-09-13 14:22:49.931492208 +0000 2024-09-13T14:23:04.295 INFO:teuthology.orchestra.run.smithi096.stdout:Modify: 2024-09-13 14:18:00.423945355 +0000 2024-09-13T14:23:04.295 INFO:teuthology.orchestra.run.smithi096.stdout:Change: 2024-09-13 14:18:00.423945355 +0000 2024-09-13T14:23:04.295 INFO:teuthology.orchestra.run.smithi096.stdout: Birth: 2024-09-13 14:18:00.423945355 +0000 2024-09-13T14:23:04.296 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T14:23:04.362 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records in 2024-09-13T14:23:04.362 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records out 2024-09-13T14:23:04.362 INFO:teuthology.orchestra.run.smithi096.stderr:512 bytes copied, 0.000227707 s, 2.2 MB/s 2024-09-13T14:23:04.363 DEBUG:teuthology.orchestra.run.smithi096:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T14:23:04.421 DEBUG:teuthology.orchestra.run.smithi096:> stat /dev/vg_nvme/lv_2 2024-09-13T14:23:04.478 INFO:teuthology.orchestra.run.smithi096.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T14:23:04.478 INFO:teuthology.orchestra.run.smithi096.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:23:04.478 INFO:teuthology.orchestra.run.smithi096.stdout:Device: 5h/5d Inode: 967 Links: 1 2024-09-13T14:23:04.478 INFO:teuthology.orchestra.run.smithi096.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:23:04.478 INFO:teuthology.orchestra.run.smithi096.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:23:04.478 INFO:teuthology.orchestra.run.smithi096.stdout:Access: 2024-09-13 14:22:49.932492186 +0000 2024-09-13T14:23:04.478 INFO:teuthology.orchestra.run.smithi096.stdout:Modify: 2024-09-13 14:18:00.422945378 +0000 2024-09-13T14:23:04.478 INFO:teuthology.orchestra.run.smithi096.stdout:Change: 2024-09-13 14:18:00.422945378 +0000 2024-09-13T14:23:04.478 INFO:teuthology.orchestra.run.smithi096.stdout: Birth: 2024-09-13 14:18:00.422945378 +0000 2024-09-13T14:23:04.479 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T14:23:04.544 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records in 2024-09-13T14:23:04.544 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records out 2024-09-13T14:23:04.544 INFO:teuthology.orchestra.run.smithi096.stderr:512 bytes copied, 0.000216634 s, 2.4 MB/s 2024-09-13T14:23:04.545 DEBUG:teuthology.orchestra.run.smithi096:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T14:23:04.603 DEBUG:teuthology.orchestra.run.smithi096:> stat /dev/vg_nvme/lv_3 2024-09-13T14:23:04.660 INFO:teuthology.orchestra.run.smithi096.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T14:23:04.660 INFO:teuthology.orchestra.run.smithi096.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:23:04.660 INFO:teuthology.orchestra.run.smithi096.stdout:Device: 5h/5d Inode: 980 Links: 1 2024-09-13T14:23:04.660 INFO:teuthology.orchestra.run.smithi096.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:23:04.660 INFO:teuthology.orchestra.run.smithi096.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:23:04.660 INFO:teuthology.orchestra.run.smithi096.stdout:Access: 2024-09-13 14:22:49.932492186 +0000 2024-09-13T14:23:04.660 INFO:teuthology.orchestra.run.smithi096.stdout:Modify: 2024-09-13 14:18:00.425945310 +0000 2024-09-13T14:23:04.660 INFO:teuthology.orchestra.run.smithi096.stdout:Change: 2024-09-13 14:18:00.425945310 +0000 2024-09-13T14:23:04.660 INFO:teuthology.orchestra.run.smithi096.stdout: Birth: 2024-09-13 14:18:00.425945310 +0000 2024-09-13T14:23:04.661 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T14:23:04.725 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records in 2024-09-13T14:23:04.725 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records out 2024-09-13T14:23:04.725 INFO:teuthology.orchestra.run.smithi096.stderr:512 bytes copied, 0.000254748 s, 2.0 MB/s 2024-09-13T14:23:04.726 DEBUG:teuthology.orchestra.run.smithi096:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T14:23:04.783 DEBUG:teuthology.orchestra.run.smithi096:> stat /dev/vg_nvme/lv_4 2024-09-13T14:23:04.839 INFO:teuthology.orchestra.run.smithi096.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T14:23:04.839 INFO:teuthology.orchestra.run.smithi096.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:23:04.839 INFO:teuthology.orchestra.run.smithi096.stdout:Device: 5h/5d Inode: 976 Links: 1 2024-09-13T14:23:04.839 INFO:teuthology.orchestra.run.smithi096.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:23:04.840 INFO:teuthology.orchestra.run.smithi096.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:23:04.840 INFO:teuthology.orchestra.run.smithi096.stdout:Access: 2024-09-13 14:22:49.932492186 +0000 2024-09-13T14:23:04.840 INFO:teuthology.orchestra.run.smithi096.stdout:Modify: 2024-09-13 14:18:00.424945333 +0000 2024-09-13T14:23:04.840 INFO:teuthology.orchestra.run.smithi096.stdout:Change: 2024-09-13 14:18:00.424945333 +0000 2024-09-13T14:23:04.840 INFO:teuthology.orchestra.run.smithi096.stdout: Birth: 2024-09-13 14:18:00.424945333 +0000 2024-09-13T14:23:04.840 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T14:23:04.903 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records in 2024-09-13T14:23:04.903 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records out 2024-09-13T14:23:04.903 INFO:teuthology.orchestra.run.smithi096.stderr:512 bytes copied, 0.000193673 s, 2.6 MB/s 2024-09-13T14:23:04.904 DEBUG:teuthology.orchestra.run.smithi096:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T14:23:04.961 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-09-13T14:23:04.961 DEBUG:teuthology.orchestra.run.smithi204:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T14:23:04.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:04 smithi204 ceph-mon[32546]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi096=y;smithi204=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:04.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:04 smithi204 ceph-mon[32546]: Saving service mgr spec with placement smithi096=y;smithi204=x;count:2 2024-09-13T14:23:04.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:04 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:04.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:04 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:04.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:04 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:04.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:04 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:04.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:04 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:04.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:04 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:23:04.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:04 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T14:23:04.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:04 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:23:04.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:04 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:04.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:04 smithi204 ceph-mon[32546]: Deploying daemon mgr.x on smithi204 2024-09-13T14:23:04.978 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T14:23:04.979 DEBUG:teuthology.orchestra.run.smithi204:> stat /dev/vg_nvme/lv_1 2024-09-13T14:23:05.035 INFO:teuthology.orchestra.run.smithi204.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T14:23:05.035 INFO:teuthology.orchestra.run.smithi204.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:23:05.035 INFO:teuthology.orchestra.run.smithi204.stdout:Device: 5h/5d Inode: 967 Links: 1 2024-09-13T14:23:05.035 INFO:teuthology.orchestra.run.smithi204.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:23:05.035 INFO:teuthology.orchestra.run.smithi204.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:23:05.035 INFO:teuthology.orchestra.run.smithi204.stdout:Access: 2024-09-13 14:22:40.756470750 +0000 2024-09-13T14:23:05.036 INFO:teuthology.orchestra.run.smithi204.stdout:Modify: 2024-09-13 14:18:01.964015082 +0000 2024-09-13T14:23:05.036 INFO:teuthology.orchestra.run.smithi204.stdout:Change: 2024-09-13 14:18:01.964015082 +0000 2024-09-13T14:23:05.036 INFO:teuthology.orchestra.run.smithi204.stdout: Birth: 2024-09-13 14:18:01.964015082 +0000 2024-09-13T14:23:05.036 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T14:23:05.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[29229]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi096=y;smithi204=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:05.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[29229]: Saving service mgr spec with placement smithi096=y;smithi204=x;count:2 2024-09-13T14:23:05.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:05.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:05.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:23:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T14:23:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:23:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[29229]: Deploying daemon mgr.x on smithi204 2024-09-13T14:23:05.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[39645]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi096=y;smithi204=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:05.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[39645]: Saving service mgr spec with placement smithi096=y;smithi204=x;count:2 2024-09-13T14:23:05.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:05.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:05.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:05.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:05.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:05.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:23:05.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T14:23:05.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:23:05.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:05.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:04 smithi096 ceph-mon[39645]: Deploying daemon mgr.x on smithi204 2024-09-13T14:23:05.101 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records in 2024-09-13T14:23:05.101 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records out 2024-09-13T14:23:05.101 INFO:teuthology.orchestra.run.smithi204.stderr:512 bytes copied, 0.000208447 s, 2.5 MB/s 2024-09-13T14:23:05.103 DEBUG:teuthology.orchestra.run.smithi204:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T14:23:05.159 DEBUG:teuthology.orchestra.run.smithi204:> stat /dev/vg_nvme/lv_2 2024-09-13T14:23:05.214 INFO:teuthology.orchestra.run.smithi204.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T14:23:05.214 INFO:teuthology.orchestra.run.smithi204.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:23:05.214 INFO:teuthology.orchestra.run.smithi204.stdout:Device: 5h/5d Inode: 976 Links: 1 2024-09-13T14:23:05.214 INFO:teuthology.orchestra.run.smithi204.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:23:05.215 INFO:teuthology.orchestra.run.smithi204.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:23:05.215 INFO:teuthology.orchestra.run.smithi204.stdout:Access: 2024-09-13 14:22:40.756470750 +0000 2024-09-13T14:23:05.215 INFO:teuthology.orchestra.run.smithi204.stdout:Modify: 2024-09-13 14:18:01.965015069 +0000 2024-09-13T14:23:05.215 INFO:teuthology.orchestra.run.smithi204.stdout:Change: 2024-09-13 14:18:01.965015069 +0000 2024-09-13T14:23:05.215 INFO:teuthology.orchestra.run.smithi204.stdout: Birth: 2024-09-13 14:18:01.965015069 +0000 2024-09-13T14:23:05.215 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T14:23:05.278 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records in 2024-09-13T14:23:05.278 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records out 2024-09-13T14:23:05.278 INFO:teuthology.orchestra.run.smithi204.stderr:512 bytes copied, 0.000312928 s, 1.6 MB/s 2024-09-13T14:23:05.280 DEBUG:teuthology.orchestra.run.smithi204:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T14:23:05.336 DEBUG:teuthology.orchestra.run.smithi204:> stat /dev/vg_nvme/lv_3 2024-09-13T14:23:05.392 INFO:teuthology.orchestra.run.smithi204.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T14:23:05.392 INFO:teuthology.orchestra.run.smithi204.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:23:05.392 INFO:teuthology.orchestra.run.smithi204.stdout:Device: 5h/5d Inode: 973 Links: 1 2024-09-13T14:23:05.392 INFO:teuthology.orchestra.run.smithi204.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:23:05.392 INFO:teuthology.orchestra.run.smithi204.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:23:05.393 INFO:teuthology.orchestra.run.smithi204.stdout:Access: 2024-09-13 14:22:40.757470738 +0000 2024-09-13T14:23:05.393 INFO:teuthology.orchestra.run.smithi204.stdout:Modify: 2024-09-13 14:18:01.965015069 +0000 2024-09-13T14:23:05.393 INFO:teuthology.orchestra.run.smithi204.stdout:Change: 2024-09-13 14:18:01.965015069 +0000 2024-09-13T14:23:05.393 INFO:teuthology.orchestra.run.smithi204.stdout: Birth: 2024-09-13 14:18:01.965015069 +0000 2024-09-13T14:23:05.393 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T14:23:05.551 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records in 2024-09-13T14:23:05.552 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records out 2024-09-13T14:23:05.552 INFO:teuthology.orchestra.run.smithi204.stderr:512 bytes copied, 0.000212532 s, 2.4 MB/s 2024-09-13T14:23:05.553 DEBUG:teuthology.orchestra.run.smithi204:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T14:23:05.569 DEBUG:teuthology.orchestra.run.smithi204:> stat /dev/vg_nvme/lv_4 2024-09-13T14:23:05.630 INFO:teuthology.orchestra.run.smithi204.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T14:23:05.630 INFO:teuthology.orchestra.run.smithi204.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:23:05.630 INFO:teuthology.orchestra.run.smithi204.stdout:Device: 5h/5d Inode: 981 Links: 1 2024-09-13T14:23:05.630 INFO:teuthology.orchestra.run.smithi204.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:23:05.631 INFO:teuthology.orchestra.run.smithi204.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:23:05.631 INFO:teuthology.orchestra.run.smithi204.stdout:Access: 2024-09-13 14:23:05.506156983 +0000 2024-09-13T14:23:05.631 INFO:teuthology.orchestra.run.smithi204.stdout:Modify: 2024-09-13 14:18:01.965015069 +0000 2024-09-13T14:23:05.631 INFO:teuthology.orchestra.run.smithi204.stdout:Change: 2024-09-13 14:18:01.965015069 +0000 2024-09-13T14:23:05.631 INFO:teuthology.orchestra.run.smithi204.stdout: Birth: 2024-09-13 14:18:01.965015069 +0000 2024-09-13T14:23:05.631 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T14:23:05.736 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records in 2024-09-13T14:23:05.736 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records out 2024-09-13T14:23:05.736 INFO:teuthology.orchestra.run.smithi204.stderr:512 bytes copied, 0.000231614 s, 2.2 MB/s 2024-09-13T14:23:05.737 DEBUG:teuthology.orchestra.run.smithi204:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T14:23:05.753 INFO:tasks.cephadm:Deploying osd.0 on smithi096 with /dev/vg_nvme/lv_4... 2024-09-13T14:23:05.753 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T14:23:05.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:05 smithi204 ceph-mon[32546]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:05.969 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:05 smithi204 systemd[1]: Starting Ceph mgr.x for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:23:05.995 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:06.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:05 smithi096 ceph-mon[29229]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:06.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:05 smithi096 ceph-mon[39645]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:06.287 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:05 smithi204 podman[34387]: 2024-09-13 14:23:05.871045227 +0000 UTC m=+0.015714274 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T14:23:06.287 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:05 smithi204 podman[34387]: 2024-09-13 14:23:05.975563492 +0000 UTC m=+0.120232572 container create f7bdd6b72e4d42f5a0c14edf836cea3d22491f063aa51744f2dc765eec1dfda5 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-13T14:23:06.287 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:06 smithi204 podman[34387]: 2024-09-13 14:23:06.177831965 +0000 UTC m=+0.322501004 container init f7bdd6b72e4d42f5a0c14edf836cea3d22491f063aa51744f2dc765eec1dfda5 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731) 2024-09-13T14:23:06.287 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:06 smithi204 podman[34387]: 2024-09-13 14:23:06.180430758 +0000 UTC m=+0.325099799 container start f7bdd6b72e4d42f5a0c14edf836cea3d22491f063aa51744f2dc765eec1dfda5 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=) 2024-09-13T14:23:06.287 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:06 smithi204 ceph-mgr[34406]: -- 172.21.15.204:0/1896499048 <== mon.1 v2:172.21.15.204:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5639f62a6340 con 0x5639f62b3000 2024-09-13T14:23:06.572 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:06 smithi204 bash[34387]: f7bdd6b72e4d42f5a0c14edf836cea3d22491f063aa51744f2dc765eec1dfda5 2024-09-13T14:23:06.572 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:06 smithi204 systemd[1]: Started Ceph mgr.x for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:23:06.572 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:06.296+0000 7f3ab1e5cec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:23:06.572 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:06.370+0000 7f3ab1e5cec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:23:06.572 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:06.438+0000 7f3ab1e5cec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:23:06.572 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:06.570+0000 7f3ab1e5cec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:23:06.968 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:06.724+0000 7f3ab1e5cec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:23:07.362 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:07 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:07.001+0000 7f3ab1e5cec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:23:07.362 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:07 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:07.065+0000 7f3ab1e5cec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:23:07.362 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:07 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:07.234+0000 7f3ab1e5cec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:23:07.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:07 smithi096 ceph-mon[29229]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:07.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:07 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:07.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:07 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:07.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:07 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:07.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:07 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:07.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:07 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:07.427 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:07 smithi096 ceph-mon[39645]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:07.427 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:07 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:07.428 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:07 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:07.428 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:07 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:07.428 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:07 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:07.428 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:07 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:07.699 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:07 smithi204 ceph-mon[32546]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:07.700 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:07 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:07.700 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:07 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:07.700 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:07 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:07.700 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:07 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:07.700 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:07 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:08.438 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:08.140+0000 7f3ab1e5cec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:23:08.438 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:08.208+0000 7f3ab1e5cec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:23:08.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:08.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:08.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:08.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:08.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:08.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[29229]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T14:23:08.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:23:08.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:23:08.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:08.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[29229]: Reconfiguring daemon mgr.y on smithi096 2024-09-13T14:23:08.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:08.472 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:08.472 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:08.472 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:08.472 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:08.473 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:08.473 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[39645]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T14:23:08.473 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:23:08.473 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:23:08.473 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:08.473 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[39645]: Reconfiguring daemon mgr.y on smithi096 2024-09-13T14:23:08.473 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:08 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:08.475 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:23:08.496 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi096:vg_nvme/lv_4 2024-09-13T14:23:08.696 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:08 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:08.696 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:08 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:08.696 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:08 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:08.696 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:08 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:08.696 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:08 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:08.696 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:08 smithi204 ceph-mon[32546]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T14:23:08.697 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:08 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:23:08.697 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:08 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:23:08.697 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:08 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:08.697 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:08 smithi204 ceph-mon[32546]: Reconfiguring daemon mgr.y on smithi096 2024-09-13T14:23:08.697 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:08 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:09.218 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:08.955+0000 7f3ab1e5cec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:23:09.218 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:09.068+0000 7f3ab1e5cec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:23:09.218 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:09.189+0000 7f3ab1e5cec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:23:09.568 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:09 smithi096 ceph-mon[29229]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:09.569 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:09 smithi096 ceph-mon[39645]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:09.632 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:09 smithi204 ceph-mon[32546]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:09.633 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:09.249+0000 7f3ab1e5cec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:23:09.633 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:09.374+0000 7f3ab1e5cec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:23:09.633 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:09.440+0000 7f3ab1e5cec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:23:09.633 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:09.498+0000 7f3ab1e5cec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:23:09.633 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:09.631+0000 7f3ab1e5cec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:23:09.657 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:09.909 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:09.697+0000 7f3ab1e5cec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:23:09.909 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:09.766+0000 7f3ab1e5cec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:23:09.909 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:09.847+0000 7f3ab1e5cec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:23:09.909 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:09.904+0000 7f3ab1e5cec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:23:10.218 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:09.961+0000 7f3ab1e5cec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:23:10.691 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:10 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:10.692 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:10 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:10.692 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:10 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:10.692 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:10 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:10.692 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:10 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:10.692 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:10 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:10.692 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:10 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:10.692 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:10 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:10.692 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:10 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:10.692 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:10 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:10.693 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:10 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:10.693 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:10 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:10.968 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:23:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:23:10.704+0000 7f3ab1e5cec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:23:10.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:10 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:10.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:10 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:10.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:10 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:10.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:10 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:10.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:10 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:10.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:10 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:11.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:11 smithi204 ceph-mon[32546]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:11 smithi204 ceph-mon[32546]: Standby manager daemon x started 2024-09-13T14:23:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:11 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/494128559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:23:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:11 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/494128559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:23:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:11 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/494128559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:23:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:11 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/494128559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:23:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:11 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:23:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:11 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:23:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:11 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:11.996 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[29229]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:11.996 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[29229]: Standby manager daemon x started 2024-09-13T14:23:11.997 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/494128559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:23:11.997 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/494128559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:23:11.997 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/494128559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:23:11.997 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/494128559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:23:11.997 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:23:11.997 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:23:11.998 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:11.998 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[39645]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:11.998 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[39645]: Standby manager daemon x started 2024-09-13T14:23:11.998 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/494128559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:23:11.998 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/494128559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:23:11.999 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/494128559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:23:11.999 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/494128559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:23:11.999 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:23:11.999 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:23:11.999 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:11 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:12.793 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:12 smithi096 ceph-mon[39645]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:12.794 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:12 smithi096 ceph-mon[39645]: mgrmap e14: y(active, since 63s), standbys: x 2024-09-13T14:23:12.794 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:12 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:23:12.794 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:12 smithi096 ceph-mon[29229]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:12.794 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:12 smithi096 ceph-mon[29229]: mgrmap e14: y(active, since 63s), standbys: x 2024-09-13T14:23:12.794 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:12 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:23:12.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:12 smithi204 ceph-mon[32546]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:12.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:12 smithi204 ceph-mon[32546]: mgrmap e14: y(active, since 63s), standbys: x 2024-09-13T14:23:12.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:12 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:23:13.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:13 smithi204 ceph-mon[32546]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:13.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:13 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2302027782' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed"}]: dispatch 2024-09-13T14:23:13.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:13 smithi204 ceph-mon[32546]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed"}]: dispatch 2024-09-13T14:23:13.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:13 smithi204 ceph-mon[32546]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed"}]': finished 2024-09-13T14:23:13.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:13 smithi204 ceph-mon[32546]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T14:23:13.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:13 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:14.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:13 smithi096 ceph-mon[29229]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:14.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:13 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2302027782' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed"}]: dispatch 2024-09-13T14:23:14.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:13 smithi096 ceph-mon[29229]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed"}]: dispatch 2024-09-13T14:23:14.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:13 smithi096 ceph-mon[29229]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed"}]': finished 2024-09-13T14:23:14.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:13 smithi096 ceph-mon[29229]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T14:23:14.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:13 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:14.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:13 smithi096 ceph-mon[39645]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:14.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:13 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2302027782' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed"}]: dispatch 2024-09-13T14:23:14.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:13 smithi096 ceph-mon[39645]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed"}]: dispatch 2024-09-13T14:23:14.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:13 smithi096 ceph-mon[39645]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed"}]': finished 2024-09-13T14:23:14.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:13 smithi096 ceph-mon[39645]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T14:23:14.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:13 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:14.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:14 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/258259361' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:23:15.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:14 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/258259361' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:23:15.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:14 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/258259361' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:23:15.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:15 smithi204 ceph-mon[32546]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:16.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:15 smithi096 ceph-mon[39645]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:16.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:15 smithi096 ceph-mon[29229]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:17.864 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:17 smithi096 ceph-mon[29229]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:17.865 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:17 smithi096 ceph-mon[39645]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:17.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:17 smithi204 ceph-mon[32546]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:19.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:19 smithi204 ceph-mon[32546]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:20.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:19 smithi096 ceph-mon[29229]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:20.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:19 smithi096 ceph-mon[39645]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:20.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T14:23:20.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:20.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:20 smithi096 ceph-mon[39645]: Deploying daemon osd.0 on smithi096 2024-09-13T14:23:20.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T14:23:20.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:20.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:20 smithi096 ceph-mon[29229]: Deploying daemon osd.0 on smithi096 2024-09-13T14:23:20.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T14:23:20.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:20.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:20 smithi204 ceph-mon[32546]: Deploying daemon osd.0 on smithi096 2024-09-13T14:23:21.778 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:21 smithi096 ceph-mon[29229]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:21.779 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:21 smithi096 ceph-mon[39645]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:21.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:21 smithi204 ceph-mon[32546]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:23.949 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:23 smithi096 ceph-mon[29229]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:23.949 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:23 smithi096 ceph-mon[39645]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:23.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:23 smithi204 ceph-mon[32546]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:25.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:25 smithi096 ceph-mon[39645]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:25.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:25.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:25.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:25.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:25 smithi096 ceph-mon[29229]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:25.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:25.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:25.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:25.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:25 smithi204 ceph-mon[32546]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:25.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:25.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:25.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:27.801 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:27 smithi096 ceph-mon[39645]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:27.801 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:27 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:27.802 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:27 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:27.802 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:27 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:27.802 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:27 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:27.802 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:27 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:27.802 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:27 smithi096 ceph-mon[29229]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:27.802 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:27 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:27.802 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:27 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:27.802 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:27 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:27.802 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:27 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:27.802 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:27 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:27.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:27 smithi204 ceph-mon[32546]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:27.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:27 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:27.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:27 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:27.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:27 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:27.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:27 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:27.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:27 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:28.101 INFO:teuthology.orchestra.run.smithi096.stdout:Created osd(s) 0 on host 'smithi096' 2024-09-13T14:23:28.789 DEBUG:teuthology.orchestra.run.smithi096:osd.0> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.0.service 2024-09-13T14:23:28.792 INFO:tasks.cephadm:Deploying osd.1 on smithi096 with /dev/vg_nvme/lv_3... 2024-09-13T14:23:28.792 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T14:23:28.956 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:28 smithi096 ceph-mon[29229]: from='osd.0 [v2:172.21.15.96:6802/766102065,v1:172.21.15.96:6803/766102065]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:23:28.956 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:28 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:28.956 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:28 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:28.957 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:28 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:28.957 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:28 smithi096 ceph-mon[39645]: from='osd.0 [v2:172.21.15.96:6802/766102065,v1:172.21.15.96:6803/766102065]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:23:28.957 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:28 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:28.957 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:28 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:28.957 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:28 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:28.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:28 smithi204 ceph-mon[32546]: from='osd.0 [v2:172.21.15.96:6802/766102065,v1:172.21.15.96:6803/766102065]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:23:28.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:28 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:28.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:28 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:28.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:28 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:29.474 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:29.913 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:29 smithi096 ceph-mon[29229]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:30.062 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:29 smithi096 ceph-mon[29229]: from='osd.0 [v2:172.21.15.96:6802/766102065,v1:172.21.15.96:6803/766102065]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:23:30.063 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:29 smithi096 ceph-mon[29229]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T14:23:30.063 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:29 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:30.063 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:29 smithi096 ceph-mon[29229]: from='osd.0 [v2:172.21.15.96:6802/766102065,v1:172.21.15.96:6803/766102065]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-13T14:23:30.064 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:29 smithi096 ceph-mon[39645]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:30.064 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:29 smithi096 ceph-mon[39645]: from='osd.0 [v2:172.21.15.96:6802/766102065,v1:172.21.15.96:6803/766102065]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:23:30.064 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:29 smithi096 ceph-mon[39645]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T14:23:30.064 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:29 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:30.064 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:29 smithi096 ceph-mon[39645]: from='osd.0 [v2:172.21.15.96:6802/766102065,v1:172.21.15.96:6803/766102065]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-13T14:23:30.064 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:23:29 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0[43517]: 2024-09-13T14:23:29.703+0000 7f9caf7dd640 -1 osd.0 0 waiting for initial osdmap 2024-09-13T14:23:30.064 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:23:29 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0[43517]: 2024-09-13T14:23:29.707+0000 7f9ca7ded640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:23:30.065 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:29 smithi204 ceph-mon[32546]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:30.065 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:29 smithi204 ceph-mon[32546]: from='osd.0 [v2:172.21.15.96:6802/766102065,v1:172.21.15.96:6803/766102065]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:23:30.065 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:29 smithi204 ceph-mon[32546]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T14:23:30.065 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:29 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:30.065 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:29 smithi204 ceph-mon[32546]: from='osd.0 [v2:172.21.15.96:6802/766102065,v1:172.21.15.96:6803/766102065]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-13T14:23:30.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:30 smithi204 ceph-mon[32546]: from='osd.0 [v2:172.21.15.96:6802/766102065,v1:172.21.15.96:6803/766102065]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-13T14:23:30.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:30 smithi204 ceph-mon[32546]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T14:23:30.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:30 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:30.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:30 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:31.010 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:30 smithi096 ceph-mon[29229]: from='osd.0 [v2:172.21.15.96:6802/766102065,v1:172.21.15.96:6803/766102065]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-13T14:23:31.011 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:30 smithi096 ceph-mon[29229]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T14:23:31.011 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:31.011 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:31.011 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:30 smithi096 ceph-mon[39645]: from='osd.0 [v2:172.21.15.96:6802/766102065,v1:172.21.15.96:6803/766102065]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-13T14:23:31.011 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:30 smithi096 ceph-mon[39645]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T14:23:31.011 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:30 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:31.011 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:30 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:31.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:31 smithi096 ceph-mon[29229]: purged_snaps scrub starts 2024-09-13T14:23:31.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:31 smithi096 ceph-mon[29229]: purged_snaps scrub ok 2024-09-13T14:23:31.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:31 smithi096 ceph-mon[29229]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:31.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:31 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:31.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:31 smithi096 ceph-mon[29229]: osd.0 [v2:172.21.15.96:6802/766102065,v1:172.21.15.96:6803/766102065] boot 2024-09-13T14:23:31.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:31 smithi096 ceph-mon[29229]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T14:23:31.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:31 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:31.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:31 smithi096 ceph-mon[39645]: purged_snaps scrub starts 2024-09-13T14:23:31.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:31 smithi096 ceph-mon[39645]: purged_snaps scrub ok 2024-09-13T14:23:31.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:31 smithi096 ceph-mon[39645]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:31.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:31 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:31.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:31 smithi096 ceph-mon[39645]: osd.0 [v2:172.21.15.96:6802/766102065,v1:172.21.15.96:6803/766102065] boot 2024-09-13T14:23:31.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:31 smithi096 ceph-mon[39645]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T14:23:31.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:31 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:32.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:31 smithi204 ceph-mon[32546]: purged_snaps scrub starts 2024-09-13T14:23:32.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:31 smithi204 ceph-mon[32546]: purged_snaps scrub ok 2024-09-13T14:23:32.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:31 smithi204 ceph-mon[32546]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:23:32.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:31 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:32.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:31 smithi204 ceph-mon[32546]: osd.0 [v2:172.21.15.96:6802/766102065,v1:172.21.15.96:6803/766102065] boot 2024-09-13T14:23:32.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:31 smithi204 ceph-mon[32546]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T14:23:32.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:31 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:23:32.922 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[29229]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T14:23:32.922 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:32.922 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:32.922 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:23:32.922 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:32.922 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:32.923 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:32.923 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:32.923 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[39645]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T14:23:32.923 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:32.923 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:32.923 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:23:32.923 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:32.924 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:32.924 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:32.924 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:32 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:33.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:32 smithi204 ceph-mon[32546]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T14:23:33.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:32 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:33.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:32 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:33.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:32 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:23:33.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:32 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:33.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:32 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:33.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:32 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:33.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:32 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:33.925 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:33 smithi096 ceph-mon[29229]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:33.926 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:33 smithi096 ceph-mon[29229]: Detected new or changed devices on smithi096 2024-09-13T14:23:33.926 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:33 smithi096 ceph-mon[29229]: Adjusting osd_memory_target on smithi096 to 16064M 2024-09-13T14:23:33.926 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:33 smithi096 ceph-mon[39645]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:33.926 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:33 smithi096 ceph-mon[39645]: Detected new or changed devices on smithi096 2024-09-13T14:23:33.926 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:33 smithi096 ceph-mon[39645]: Adjusting osd_memory_target on smithi096 to 16064M 2024-09-13T14:23:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:23:33.949 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi096:vg_nvme/lv_3 2024-09-13T14:23:34.187 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:34.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:33 smithi204 ceph-mon[32546]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:34.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:33 smithi204 ceph-mon[32546]: Detected new or changed devices on smithi096 2024-09-13T14:23:34.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:33 smithi204 ceph-mon[32546]: Adjusting osd_memory_target on smithi096 to 16064M 2024-09-13T14:23:35.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:35 smithi096 ceph-mon[29229]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:35.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:35 smithi096 ceph-mon[39645]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:36.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:35 smithi204 ceph-mon[32546]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:37.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:36 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:23:37.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:36 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:23:37.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:36 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:37.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:36 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:23:37.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:36 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:23:37.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:36 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:37.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:36 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:23:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:36 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:23:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:36 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:37.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:37 smithi096 ceph-mon[29229]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:37.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:37 smithi096 ceph-mon[29229]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:37.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:37 smithi096 ceph-mon[39645]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:37.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:37 smithi096 ceph-mon[39645]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:38.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:37 smithi204 ceph-mon[32546]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:38.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:37 smithi204 ceph-mon[32546]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:39.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:38 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1648147461' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7"}]: dispatch 2024-09-13T14:23:39.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:38 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1648147461' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7"}]': finished 2024-09-13T14:23:39.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:38 smithi096 ceph-mon[39645]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T14:23:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:38 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:38 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1326196443' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:23:39.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:38 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1648147461' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7"}]: dispatch 2024-09-13T14:23:39.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:38 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1648147461' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7"}]': finished 2024-09-13T14:23:39.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:38 smithi096 ceph-mon[29229]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T14:23:39.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:38 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:39.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:38 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1326196443' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:23:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:38 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1648147461' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7"}]: dispatch 2024-09-13T14:23:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:38 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1648147461' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7"}]': finished 2024-09-13T14:23:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:38 smithi204 ceph-mon[32546]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T14:23:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:38 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:38 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1326196443' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:23:40.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:39 smithi096 ceph-mon[29229]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:40.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:39 smithi096 ceph-mon[39645]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:40.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:39 smithi204 ceph-mon[32546]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:41.749 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:41 smithi096 ceph-mon[29229]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:41.750 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:41 smithi096 ceph-mon[39645]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:42.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:41 smithi204 ceph-mon[32546]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:44.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:43 smithi096 ceph-mon[29229]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:44.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:43 smithi096 ceph-mon[39645]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:44.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:43 smithi204 ceph-mon[32546]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:44.814 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:44 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T14:23:44.814 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:44 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:44.814 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:44 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T14:23:44.814 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:44 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:45.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:44 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T14:23:45.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:44 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:45.962 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:45 smithi096 ceph-mon[29229]: Deploying daemon osd.1 on smithi096 2024-09-13T14:23:45.962 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:45 smithi096 ceph-mon[29229]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:45.962 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:45 smithi096 ceph-mon[39645]: Deploying daemon osd.1 on smithi096 2024-09-13T14:23:45.962 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:45 smithi096 ceph-mon[39645]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:46.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:45 smithi204 ceph-mon[32546]: Deploying daemon osd.1 on smithi096 2024-09-13T14:23:46.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:45 smithi204 ceph-mon[32546]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:48.021 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:47 smithi204 ceph-mon[32546]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:48.033 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:47 smithi096 ceph-mon[29229]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:48.033 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:47 smithi096 ceph-mon[39645]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:49.993 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:49 smithi096 ceph-mon[29229]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:49.993 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:49 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:49.993 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:49 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:49.994 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:49 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:49.994 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:49 smithi096 ceph-mon[39645]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:49.994 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:49 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:49.994 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:49 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:49.994 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:49 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:50.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:49 smithi204 ceph-mon[32546]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:50.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:49 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:50.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:49 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:50.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:49 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:52.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:51 smithi096 ceph-mon[29229]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:52.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:51 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:52.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:51 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:52.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:51 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:52.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:51 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:52.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:51 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:51 smithi096 ceph-mon[39645]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:51 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:51 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:51 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:51 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:51 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:52.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:51 smithi204 ceph-mon[32546]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:52.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:51 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:52.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:51 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:52.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:51 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:52.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:51 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:52.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:51 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:52.239 INFO:teuthology.orchestra.run.smithi096.stdout:Created osd(s) 1 on host 'smithi096' 2024-09-13T14:23:52.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:52 smithi096 ceph-mon[29229]: from='osd.1 [v2:172.21.15.96:6810/401053665,v1:172.21.15.96:6811/401053665]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:23:52.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:52 smithi096 ceph-mon[29229]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:23:52.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:52 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:52.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:52 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:52.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:52 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:52.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:52 smithi096 ceph-mon[39645]: from='osd.1 [v2:172.21.15.96:6810/401053665,v1:172.21.15.96:6811/401053665]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:23:52.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:52 smithi096 ceph-mon[39645]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:23:52.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:52 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:52.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:52 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:52.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:52 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:52.882 DEBUG:teuthology.orchestra.run.smithi096:osd.1> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.1.service 2024-09-13T14:23:52.885 INFO:tasks.cephadm:Deploying osd.2 on smithi096 with /dev/vg_nvme/lv_2... 2024-09-13T14:23:52.885 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T14:23:53.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:52 smithi204 ceph-mon[32546]: from='osd.1 [v2:172.21.15.96:6810/401053665,v1:172.21.15.96:6811/401053665]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:23:53.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:52 smithi204 ceph-mon[32546]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:23:53.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:52 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:53.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:52 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:53.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:52 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:53.872 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:53 smithi096 ceph-mon[29229]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:53.872 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:53 smithi096 ceph-mon[29229]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:23:53.872 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:53 smithi096 ceph-mon[29229]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T14:23:53.872 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:53 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:53.872 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:53 smithi096 ceph-mon[29229]: from='osd.1 [v2:172.21.15.96:6810/401053665,v1:172.21.15.96:6811/401053665]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-13T14:23:53.872 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:53 smithi096 ceph-mon[29229]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-13T14:23:53.873 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:53 smithi096 ceph-mon[39645]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:53.873 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:53 smithi096 ceph-mon[39645]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:23:53.873 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:53 smithi096 ceph-mon[39645]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T14:23:53.873 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:53 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:53.873 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:53 smithi096 ceph-mon[39645]: from='osd.1 [v2:172.21.15.96:6810/401053665,v1:172.21.15.96:6811/401053665]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-13T14:23:53.873 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:53 smithi096 ceph-mon[39645]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-13T14:23:53.873 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:23:53 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1[49132]: 2024-09-13T14:23:53.779+0000 7ff738d42640 -1 osd.1 0 waiting for initial osdmap 2024-09-13T14:23:53.873 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:23:53 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1[49132]: 2024-09-13T14:23:53.782+0000 7ff73437a640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:23:53.906 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:54.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:53 smithi204 ceph-mon[32546]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:54.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:53 smithi204 ceph-mon[32546]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:23:54.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:53 smithi204 ceph-mon[32546]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T14:23:54.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:53 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:54.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:53 smithi204 ceph-mon[32546]: from='osd.1 [v2:172.21.15.96:6810/401053665,v1:172.21.15.96:6811/401053665]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-13T14:23:54.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:53 smithi204 ceph-mon[32546]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-13T14:23:55.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:54 smithi096 ceph-mon[29229]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-13T14:23:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:54 smithi096 ceph-mon[29229]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T14:23:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:54 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:54 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:54 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:55.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:54 smithi096 ceph-mon[39645]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-13T14:23:55.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:54 smithi096 ceph-mon[39645]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T14:23:55.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:54 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:55.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:54 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:55.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:54 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:55.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:54 smithi204 ceph-mon[32546]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-13T14:23:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:54 smithi204 ceph-mon[32546]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T14:23:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:54 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:54 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:54 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:56.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:55 smithi096 ceph-mon[29229]: purged_snaps scrub starts 2024-09-13T14:23:56.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:55 smithi096 ceph-mon[29229]: purged_snaps scrub ok 2024-09-13T14:23:56.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:55 smithi096 ceph-mon[29229]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:56.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:55 smithi096 ceph-mon[29229]: osd.1 [v2:172.21.15.96:6810/401053665,v1:172.21.15.96:6811/401053665] boot 2024-09-13T14:23:56.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:55 smithi096 ceph-mon[29229]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T14:23:56.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:55 smithi096 ceph-mon[39645]: purged_snaps scrub starts 2024-09-13T14:23:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:55 smithi096 ceph-mon[39645]: purged_snaps scrub ok 2024-09-13T14:23:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:55 smithi096 ceph-mon[39645]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:56.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:55 smithi096 ceph-mon[39645]: osd.1 [v2:172.21.15.96:6810/401053665,v1:172.21.15.96:6811/401053665] boot 2024-09-13T14:23:56.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:55 smithi096 ceph-mon[39645]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T14:23:56.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:56.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:55 smithi204 ceph-mon[32546]: purged_snaps scrub starts 2024-09-13T14:23:56.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:55 smithi204 ceph-mon[32546]: purged_snaps scrub ok 2024-09-13T14:23:56.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:55 smithi204 ceph-mon[32546]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:23:56.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:55 smithi204 ceph-mon[32546]: osd.1 [v2:172.21.15.96:6810/401053665,v1:172.21.15.96:6811/401053665] boot 2024-09-13T14:23:56.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:55 smithi204 ceph-mon[32546]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T14:23:56.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:23:57.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[39645]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T14:23:57.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:57.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:57.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:23:57.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:23:57.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:57.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:57.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:57.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:57.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[29229]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T14:23:57.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:57.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:57.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:23:57.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:23:57.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:57.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:57.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:57.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:56 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:57.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:56 smithi204 ceph-mon[32546]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T14:23:57.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:56 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:56 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:56 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:23:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:56 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:23:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:56 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:56 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:56 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:56 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:23:57.474 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:23:57.497 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi096:vg_nvme/lv_2 2024-09-13T14:23:57.740 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:58.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:57 smithi096 ceph-mon[29229]: Detected new or changed devices on smithi096 2024-09-13T14:23:58.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:57 smithi096 ceph-mon[29229]: Adjusting osd_memory_target on smithi096 to 8032M 2024-09-13T14:23:58.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:57 smithi096 ceph-mon[29229]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:23:58.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:57 smithi096 ceph-mon[39645]: Detected new or changed devices on smithi096 2024-09-13T14:23:58.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:57 smithi096 ceph-mon[39645]: Adjusting osd_memory_target on smithi096 to 8032M 2024-09-13T14:23:58.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:57 smithi096 ceph-mon[39645]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:23:58.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:57 smithi204 ceph-mon[32546]: Detected new or changed devices on smithi096 2024-09-13T14:23:58.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:57 smithi204 ceph-mon[32546]: Adjusting osd_memory_target on smithi096 to 8032M 2024-09-13T14:23:58.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:57 smithi204 ceph-mon[32546]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:00.043 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:59 smithi096 ceph-mon[29229]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:00.043 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:59 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:24:00.043 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:59 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:24:00.044 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:23:59 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:00.044 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:59 smithi096 ceph-mon[39645]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:00.044 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:59 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:24:00.044 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:59 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:24:00.044 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:23:59 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:00.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:59 smithi204 ceph-mon[32546]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:00.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:59 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:24:00.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:59 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:24:00.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:23:59 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:00.900 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:00 smithi096 ceph-mon[29229]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:00.901 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:00 smithi096 ceph-mon[39645]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:01.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:00 smithi204 ceph-mon[32546]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:02.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:01 smithi096 ceph-mon[29229]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:02.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:01 smithi096 ceph-mon[39645]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:02.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:01 smithi204 ceph-mon[32546]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:03.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:02 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/493999972' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a92a5b3-bdc7-499a-8b0f-2b0936fd9598"}]: dispatch 2024-09-13T14:24:03.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:02 smithi096 ceph-mon[29229]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a92a5b3-bdc7-499a-8b0f-2b0936fd9598"}]: dispatch 2024-09-13T14:24:03.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:02 smithi096 ceph-mon[29229]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6a92a5b3-bdc7-499a-8b0f-2b0936fd9598"}]': finished 2024-09-13T14:24:03.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:02 smithi096 ceph-mon[29229]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T14:24:03.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:02 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:03.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:02 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/4117940293' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:24:03.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:02 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/493999972' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a92a5b3-bdc7-499a-8b0f-2b0936fd9598"}]: dispatch 2024-09-13T14:24:03.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:02 smithi096 ceph-mon[39645]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a92a5b3-bdc7-499a-8b0f-2b0936fd9598"}]: dispatch 2024-09-13T14:24:03.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:02 smithi096 ceph-mon[39645]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6a92a5b3-bdc7-499a-8b0f-2b0936fd9598"}]': finished 2024-09-13T14:24:03.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:02 smithi096 ceph-mon[39645]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T14:24:03.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:02 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:03.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:02 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/4117940293' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:24:03.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:02 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/493999972' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a92a5b3-bdc7-499a-8b0f-2b0936fd9598"}]: dispatch 2024-09-13T14:24:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:02 smithi204 ceph-mon[32546]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a92a5b3-bdc7-499a-8b0f-2b0936fd9598"}]: dispatch 2024-09-13T14:24:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:02 smithi204 ceph-mon[32546]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6a92a5b3-bdc7-499a-8b0f-2b0936fd9598"}]': finished 2024-09-13T14:24:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:02 smithi204 ceph-mon[32546]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T14:24:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:02 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:02 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/4117940293' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:24:04.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:03 smithi204 ceph-mon[32546]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:04.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:03 smithi096 ceph-mon[39645]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:04.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:03 smithi096 ceph-mon[29229]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:05.928 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:05 smithi096 ceph-mon[29229]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:05.928 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:05 smithi096 ceph-mon[39645]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:06.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:05 smithi204 ceph-mon[32546]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:08.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:07 smithi204 ceph-mon[32546]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:08.293 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:07 smithi096 ceph-mon[39645]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:08.293 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:07 smithi096 ceph-mon[29229]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:08.973 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T14:24:08.973 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:08 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:08.973 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:08 smithi096 ceph-mon[29229]: Deploying daemon osd.2 on smithi096 2024-09-13T14:24:08.973 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:08 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T14:24:08.973 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:08 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:08.973 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:08 smithi096 ceph-mon[39645]: Deploying daemon osd.2 on smithi096 2024-09-13T14:24:09.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:08 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T14:24:09.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:08 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:09.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:08 smithi204 ceph-mon[32546]: Deploying daemon osd.2 on smithi096 2024-09-13T14:24:09.869 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:09 smithi096 ceph-mon[29229]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:09.869 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:09 smithi096 ceph-mon[39645]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:10.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:09 smithi204 ceph-mon[32546]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:11.999 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:11 smithi096 ceph-mon[29229]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:12.000 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:11 smithi096 ceph-mon[39645]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:12.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:11 smithi204 ceph-mon[32546]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:14.069 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:13 smithi096 ceph-mon[39645]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:14.069 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:13 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:14.069 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:13 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:14.069 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:13 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:14.070 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:13 smithi096 ceph-mon[29229]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:14.070 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:13 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:14.070 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:13 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:14.070 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:13 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:14.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:13 smithi204 ceph-mon[32546]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:14.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:13 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:14.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:13 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:14.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:13 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:16.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:15 smithi204 ceph-mon[32546]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:16.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:15 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:16.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:15 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:16.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:15 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:16.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:15 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:16.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:15 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:16.312 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:15 smithi096 ceph-mon[29229]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:16.312 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:15 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:16.313 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:15 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:16.313 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:15 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:16.313 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:15 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:16.313 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:15 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:16.313 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:15 smithi096 ceph-mon[39645]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:16.313 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:15 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:16.313 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:15 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:16.314 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:15 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:16.314 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:15 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:16.314 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:15 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:16.346 INFO:teuthology.orchestra.run.smithi096.stdout:Created osd(s) 2 on host 'smithi096' 2024-09-13T14:24:16.924 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:16 smithi096 ceph-mon[39645]: from='osd.2 [v2:172.21.15.96:6818/2386475672,v1:172.21.15.96:6819/2386475672]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:24:16.924 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:16 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:16.924 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:16 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:16.924 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:16 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:16.924 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:16 smithi096 ceph-mon[29229]: from='osd.2 [v2:172.21.15.96:6818/2386475672,v1:172.21.15.96:6819/2386475672]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:24:16.924 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:16 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:16.924 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:16 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:16.924 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:16 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:16.948 DEBUG:teuthology.orchestra.run.smithi096:osd.2> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.2.service 2024-09-13T14:24:16.950 INFO:tasks.cephadm:Deploying osd.3 on smithi096 with /dev/vg_nvme/lv_1... 2024-09-13T14:24:16.950 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T14:24:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:16 smithi204 ceph-mon[32546]: from='osd.2 [v2:172.21.15.96:6818/2386475672,v1:172.21.15.96:6819/2386475672]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:24:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:16 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:16 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:16 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:18.047 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:17 smithi096 ceph-mon[29229]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:18.047 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:17 smithi096 ceph-mon[29229]: from='osd.2 [v2:172.21.15.96:6818/2386475672,v1:172.21.15.96:6819/2386475672]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:24:18.047 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:17 smithi096 ceph-mon[29229]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T14:24:18.047 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:17 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:18.047 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:17 smithi096 ceph-mon[29229]: from='osd.2 [v2:172.21.15.96:6818/2386475672,v1:172.21.15.96:6819/2386475672]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-13T14:24:18.048 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:17 smithi096 ceph-mon[39645]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:18.048 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:17 smithi096 ceph-mon[39645]: from='osd.2 [v2:172.21.15.96:6818/2386475672,v1:172.21.15.96:6819/2386475672]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:24:18.048 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:17 smithi096 ceph-mon[39645]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T14:24:18.048 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:17 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:18.048 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:17 smithi096 ceph-mon[39645]: from='osd.2 [v2:172.21.15.96:6818/2386475672,v1:172.21.15.96:6819/2386475672]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-13T14:24:18.048 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:24:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2[55161]: 2024-09-13T14:24:17.863+0000 7f08936ec640 -1 osd.2 0 waiting for initial osdmap 2024-09-13T14:24:18.048 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:24:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2[55161]: 2024-09-13T14:24:17.867+0000 7f088acfa640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:24:18.119 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:17 smithi204 ceph-mon[32546]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:18.120 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:17 smithi204 ceph-mon[32546]: from='osd.2 [v2:172.21.15.96:6818/2386475672,v1:172.21.15.96:6819/2386475672]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:24:18.120 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:17 smithi204 ceph-mon[32546]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T14:24:18.120 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:17 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:18.120 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:17 smithi204 ceph-mon[32546]: from='osd.2 [v2:172.21.15.96:6818/2386475672,v1:172.21.15.96:6819/2386475672]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-13T14:24:18.559 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:18.945 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:18 smithi096 ceph-mon[39645]: from='osd.2 [v2:172.21.15.96:6818/2386475672,v1:172.21.15.96:6819/2386475672]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-13T14:24:18.945 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:18 smithi096 ceph-mon[39645]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T14:24:18.945 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:18 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:18.945 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:18 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:18.945 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:18 smithi096 ceph-mon[29229]: from='osd.2 [v2:172.21.15.96:6818/2386475672,v1:172.21.15.96:6819/2386475672]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-13T14:24:18.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:18 smithi096 ceph-mon[29229]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T14:24:18.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:18 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:18.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:18 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:19.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:18 smithi204 ceph-mon[32546]: from='osd.2 [v2:172.21.15.96:6818/2386475672,v1:172.21.15.96:6819/2386475672]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-13T14:24:19.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:18 smithi204 ceph-mon[32546]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T14:24:19.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:18 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:19.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:18 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:20.027 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:19 smithi096 ceph-mon[29229]: purged_snaps scrub starts 2024-09-13T14:24:20.027 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:19 smithi096 ceph-mon[29229]: purged_snaps scrub ok 2024-09-13T14:24:20.027 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:19 smithi096 ceph-mon[29229]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:20.027 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:19 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:20.027 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:19 smithi096 ceph-mon[29229]: osd.2 [v2:172.21.15.96:6818/2386475672,v1:172.21.15.96:6819/2386475672] boot 2024-09-13T14:24:20.028 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:19 smithi096 ceph-mon[29229]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T14:24:20.028 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:19 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:20.028 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:19 smithi096 ceph-mon[39645]: purged_snaps scrub starts 2024-09-13T14:24:20.028 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:19 smithi096 ceph-mon[39645]: purged_snaps scrub ok 2024-09-13T14:24:20.028 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:19 smithi096 ceph-mon[39645]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:20.028 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:19 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:20.028 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:19 smithi096 ceph-mon[39645]: osd.2 [v2:172.21.15.96:6818/2386475672,v1:172.21.15.96:6819/2386475672] boot 2024-09-13T14:24:20.028 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:19 smithi096 ceph-mon[39645]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T14:24:20.028 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:19 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:20.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:19 smithi204 ceph-mon[32546]: purged_snaps scrub starts 2024-09-13T14:24:20.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:19 smithi204 ceph-mon[32546]: purged_snaps scrub ok 2024-09-13T14:24:20.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:19 smithi204 ceph-mon[32546]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:24:20.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:19 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:20.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:19 smithi204 ceph-mon[32546]: osd.2 [v2:172.21.15.96:6818/2386475672,v1:172.21.15.96:6819/2386475672] boot 2024-09-13T14:24:20.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:19 smithi204 ceph-mon[32546]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T14:24:20.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:19 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:21.023 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[39645]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T14:24:21.024 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T14:24:21.024 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:21.024 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:21.024 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:21.024 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:21.024 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:21.024 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:21.024 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:21.024 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:21.024 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:21.025 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[29229]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T14:24:21.025 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T14:24:21.025 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:21.025 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:21.025 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:21.025 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:21.025 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:21.025 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:21.025 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:21.025 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:21.025 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:21.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:20 smithi204 ceph-mon[32546]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T14:24:21.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T14:24:21.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:21.748 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:24:21.768 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi096:vg_nvme/lv_1 2024-09-13T14:24:22.037 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:22.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:21 smithi096 ceph-mon[29229]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:22.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:21 smithi096 ceph-mon[29229]: Detected new or changed devices on smithi096 2024-09-13T14:24:22.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:21 smithi096 ceph-mon[29229]: Adjusting osd_memory_target on smithi096 to 5354M 2024-09-13T14:24:22.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:21 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T14:24:22.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:21 smithi096 ceph-mon[29229]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T14:24:22.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:21 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:24:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:21 smithi096 ceph-mon[39645]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:21 smithi096 ceph-mon[39645]: Detected new or changed devices on smithi096 2024-09-13T14:24:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:21 smithi096 ceph-mon[39645]: Adjusting osd_memory_target on smithi096 to 5354M 2024-09-13T14:24:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:21 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T14:24:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:21 smithi096 ceph-mon[39645]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T14:24:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:21 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:24:22.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:21 smithi204 ceph-mon[32546]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:22.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:21 smithi204 ceph-mon[32546]: Detected new or changed devices on smithi096 2024-09-13T14:24:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:21 smithi204 ceph-mon[32546]: Adjusting osd_memory_target on smithi096 to 5354M 2024-09-13T14:24:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:21 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T14:24:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:21 smithi204 ceph-mon[32546]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T14:24:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:21 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:24:22.408 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59126]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:24:22.409 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59126]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:24:22.409 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59126]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:24:22.409 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59126]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:24:22.729 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59148]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:24:22.729 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59148]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:24:22.729 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59148]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:24:22.729 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59148]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:24:22.992 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:22 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:24:22.992 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:22 smithi096 ceph-mon[29229]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T14:24:22.992 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:22 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:22.993 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:22 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:24:22.993 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:22 smithi096 ceph-mon[39645]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T14:24:22.993 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:22 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:22.993 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59151]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:24:22.993 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59151]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:24:22.993 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59151]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:24:22.993 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59151]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:24:23.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:22 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:24:23.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:22 smithi204 ceph-mon[32546]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T14:24:23.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:22 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:23.246 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59155]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:24:23.246 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59155]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:24:23.246 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59155]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:24:23.246 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:24:22 smithi096 sudo[59155]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:24:23.510 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:24:23 smithi096 sudo[59180]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:24:23.510 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:24:23 smithi096 sudo[59180]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:24:23.510 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:24:23 smithi096 sudo[59180]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:24:23.510 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:24:23 smithi096 sudo[59180]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:24:23.765 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:24:23 smithi096 sudo[59188]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:24:23.765 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:24:23 smithi096 sudo[59188]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:24:23.765 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:24:23 smithi096 sudo[59188]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:24:23.765 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:24:23 smithi096 sudo[59188]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:24:24.018 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:23 smithi096 sudo[59212]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:24:24.018 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:23 smithi096 sudo[59212]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:24:24.018 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:23 smithi096 sudo[59212]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:24:24.018 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:23 smithi096 sudo[59212]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:24:24.283 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:24 smithi204 ceph-mon[32546]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:24.283 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:24 smithi204 ceph-mon[32546]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T14:24:24.283 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:24 smithi204 ceph-mon[32546]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T14:24:24.283 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:24 smithi204 ceph-mon[32546]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:24:24.284 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:24 smithi204 sudo[35041]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:24:24.284 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:24 smithi204 sudo[35041]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:24:24.284 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:24 smithi204 sudo[35041]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:24:24.284 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:24 smithi204 sudo[35041]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:24:24.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:24 smithi096 ceph-mon[39645]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:24.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:24 smithi096 ceph-mon[39645]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T14:24:24.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:24 smithi096 ceph-mon[39645]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T14:24:24.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:24 smithi096 ceph-mon[39645]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:24:24.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:24 smithi096 sudo[59216]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:24:24.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:24 smithi096 sudo[59216]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:24:24.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:24 smithi096 sudo[59216]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:24:24.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:24 smithi096 sudo[59216]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:24:24.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:24 smithi096 ceph-mon[29229]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:24.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:24 smithi096 ceph-mon[29229]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T14:24:24.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:24 smithi096 ceph-mon[29229]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T14:24:24.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:24 smithi096 ceph-mon[29229]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:24:24.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:24 smithi204 sudo[35044]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:24:24.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:24 smithi204 sudo[35044]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:24:24.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:24 smithi204 sudo[35044]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:24:24.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:24 smithi204 sudo[35044]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:24:25.046 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:24 smithi096 sudo[59357]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:24:25.046 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:24 smithi096 sudo[59357]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:24:25.046 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:24 smithi096 sudo[59357]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:24:25.046 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:24 smithi096 sudo[59357]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:24:25.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:24:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:24:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:24:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:24:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:24:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:24:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:24:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:24:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:24:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:24:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:24:25.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:25.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:24:25.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:24:25.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:24:25.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 sudo[59361]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:24:25.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 sudo[59361]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:24:25.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 sudo[59361]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:24:25.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 sudo[59361]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:24:25.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:25.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:24:25.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:24:25.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:24:25.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:24:25.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:24:25.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:24:25.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:24:25.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:24:25.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:24:25.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:24:25.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:25.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:25.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:24:25.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:25.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:24:25.326 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:24:25.326 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:24:25.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:24:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:24:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:24:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:24:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:24:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:24:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:24:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:24:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:24:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:24:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:25.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:24:25.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:25.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:24:25.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:24:25.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:24:26.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:26 smithi096 ceph-mon[39645]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:24:26.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:26 smithi096 ceph-mon[39645]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:24:26.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:26 smithi096 ceph-mon[29229]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:24:26.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:26 smithi096 ceph-mon[29229]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:24:26.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:26 smithi204 ceph-mon[32546]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:24:26.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:26 smithi204 ceph-mon[32546]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:24:27.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:27 smithi096 ceph-mon[39645]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:27.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:27 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3601878598' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1230646c-934e-4eb4-b4b9-16be52ffba66"}]: dispatch 2024-09-13T14:24:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:27 smithi096 ceph-mon[39645]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1230646c-934e-4eb4-b4b9-16be52ffba66"}]: dispatch 2024-09-13T14:24:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:27 smithi096 ceph-mon[39645]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1230646c-934e-4eb4-b4b9-16be52ffba66"}]': finished 2024-09-13T14:24:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:27 smithi096 ceph-mon[39645]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T14:24:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:27 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:27 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/4133399053' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:24:27.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:27 smithi096 ceph-mon[29229]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:27.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:27 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3601878598' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1230646c-934e-4eb4-b4b9-16be52ffba66"}]: dispatch 2024-09-13T14:24:27.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:27 smithi096 ceph-mon[29229]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1230646c-934e-4eb4-b4b9-16be52ffba66"}]: dispatch 2024-09-13T14:24:27.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:27 smithi096 ceph-mon[29229]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1230646c-934e-4eb4-b4b9-16be52ffba66"}]': finished 2024-09-13T14:24:27.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:27 smithi096 ceph-mon[29229]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T14:24:27.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:27 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:27.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:27 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/4133399053' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:24:27.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:27 smithi204 ceph-mon[32546]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:27.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:27 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3601878598' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1230646c-934e-4eb4-b4b9-16be52ffba66"}]: dispatch 2024-09-13T14:24:27.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:27 smithi204 ceph-mon[32546]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1230646c-934e-4eb4-b4b9-16be52ffba66"}]: dispatch 2024-09-13T14:24:27.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:27 smithi204 ceph-mon[32546]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1230646c-934e-4eb4-b4b9-16be52ffba66"}]': finished 2024-09-13T14:24:27.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:27 smithi204 ceph-mon[32546]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T14:24:27.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:27 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:27.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:27 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/4133399053' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:24:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:29 smithi096 ceph-mon[29229]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:29 smithi096 ceph-mon[39645]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:29.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:29 smithi204 ceph-mon[32546]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:31.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:31 smithi096 ceph-mon[29229]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:31.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:31 smithi096 ceph-mon[39645]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:31.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:31 smithi204 ceph-mon[32546]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:32.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:32 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T14:24:32.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:32 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:32.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:32 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T14:24:32.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:32 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:32.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:32 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T14:24:32.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:32 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:33.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:33 smithi096 ceph-mon[39645]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:33.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:33 smithi096 ceph-mon[39645]: Deploying daemon osd.3 on smithi096 2024-09-13T14:24:33.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:33 smithi096 ceph-mon[29229]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:33.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:33 smithi096 ceph-mon[29229]: Deploying daemon osd.3 on smithi096 2024-09-13T14:24:33.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:33 smithi204 ceph-mon[32546]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:33.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:33 smithi204 ceph-mon[32546]: Deploying daemon osd.3 on smithi096 2024-09-13T14:24:35.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:35 smithi096 ceph-mon[29229]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:35.477 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:35 smithi096 ceph-mon[39645]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:35.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:35 smithi204 ceph-mon[32546]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:37.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:37 smithi096 ceph-mon[29229]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:37.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:37 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:37.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:37 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:37.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:37 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:37.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:37 smithi096 ceph-mon[39645]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:37.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:37 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:37.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:37 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:37.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:37 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:37.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:37 smithi204 ceph-mon[32546]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:37.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:37 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:37.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:37 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:37.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:37 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:39.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:39 smithi096 ceph-mon[29229]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:39.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:39 smithi096 ceph-mon[39645]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:39.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:39 smithi204 ceph-mon[32546]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:40.060 INFO:teuthology.orchestra.run.smithi096.stdout:Created osd(s) 3 on host 'smithi096' 2024-09-13T14:24:40.633 DEBUG:teuthology.orchestra.run.smithi096:osd.3> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.3.service 2024-09-13T14:24:40.636 INFO:tasks.cephadm:Deploying osd.4 on smithi204 with /dev/vg_nvme/lv_4... 2024-09-13T14:24:40.636 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T14:24:40.900 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:24:41.288 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:41 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:41.288 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:41 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:41.288 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:41 smithi204 ceph-mon[32546]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:41.288 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:41 smithi204 ceph-mon[32546]: from='osd.3 [v2:172.21.15.96:6826/282100645,v1:172.21.15.96:6827/282100645]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:24:41.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:41 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:41.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:41 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:41.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:41 smithi096 ceph-mon[39645]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:41.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:41 smithi096 ceph-mon[39645]: from='osd.3 [v2:172.21.15.96:6826/282100645,v1:172.21.15.96:6827/282100645]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:24:41.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:41 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:41.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:41 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:41.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:41 smithi096 ceph-mon[29229]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:41.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:41 smithi096 ceph-mon[29229]: from='osd.3 [v2:172.21.15.96:6826/282100645,v1:172.21.15.96:6827/282100645]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:24:42.158 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:42 smithi204 ceph-mon[32546]: from='osd.3 [v2:172.21.15.96:6826/282100645,v1:172.21.15.96:6827/282100645]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:24:42.158 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:42 smithi204 ceph-mon[32546]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T14:24:42.158 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:42 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:42.158 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:42 smithi204 ceph-mon[32546]: from='osd.3 [v2:172.21.15.96:6826/282100645,v1:172.21.15.96:6827/282100645]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-13T14:24:42.158 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:42 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:42.158 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:42 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:42.248 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:42 smithi096 ceph-mon[39645]: from='osd.3 [v2:172.21.15.96:6826/282100645,v1:172.21.15.96:6827/282100645]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:24:42.248 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:42 smithi096 ceph-mon[39645]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T14:24:42.248 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:42 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:42.248 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:42 smithi096 ceph-mon[39645]: from='osd.3 [v2:172.21.15.96:6826/282100645,v1:172.21.15.96:6827/282100645]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-13T14:24:42.248 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:42 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:42.248 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:42 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:42.249 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:42 smithi096 ceph-mon[29229]: from='osd.3 [v2:172.21.15.96:6826/282100645,v1:172.21.15.96:6827/282100645]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:24:42.249 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:42 smithi096 ceph-mon[29229]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T14:24:42.249 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:42 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:42.249 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:42 smithi096 ceph-mon[29229]: from='osd.3 [v2:172.21.15.96:6826/282100645,v1:172.21.15.96:6827/282100645]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-13T14:24:42.249 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:42 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:42.249 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:42 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:42.249 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:24:42 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[61087]: 2024-09-13T14:24:42.075+0000 7fdb83c46640 -1 osd.3 0 waiting for initial osdmap 2024-09-13T14:24:42.249 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:24:42 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[61087]: 2024-09-13T14:24:42.079+0000 7fdb7f27e640 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:24:43.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:43 smithi204 ceph-mon[32546]: from='osd.3 [v2:172.21.15.96:6826/282100645,v1:172.21.15.96:6827/282100645]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-13T14:24:43.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:43 smithi204 ceph-mon[32546]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T14:24:43.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:43 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:43.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:43 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:43.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:43 smithi204 ceph-mon[32546]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:43.229 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:24:43.248 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi204:vg_nvme/lv_4 2024-09-13T14:24:43.346 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:43 smithi096 ceph-mon[29229]: from='osd.3 [v2:172.21.15.96:6826/282100645,v1:172.21.15.96:6827/282100645]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-13T14:24:43.346 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:43 smithi096 ceph-mon[29229]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T14:24:43.347 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:43 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:43.347 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:43 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:43.347 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:43 smithi096 ceph-mon[29229]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:43.347 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:43 smithi096 ceph-mon[39645]: from='osd.3 [v2:172.21.15.96:6826/282100645,v1:172.21.15.96:6827/282100645]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-13T14:24:43.347 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:43 smithi096 ceph-mon[39645]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T14:24:43.347 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:43 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:43.347 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:43 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:43.347 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:43 smithi096 ceph-mon[39645]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:24:43.494 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:24:44.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:44 smithi204 ceph-mon[32546]: purged_snaps scrub starts 2024-09-13T14:24:44.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:44 smithi204 ceph-mon[32546]: purged_snaps scrub ok 2024-09-13T14:24:44.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:44 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:44.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:44 smithi204 ceph-mon[32546]: osd.3 [v2:172.21.15.96:6826/282100645,v1:172.21.15.96:6827/282100645] boot 2024-09-13T14:24:44.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:44 smithi204 ceph-mon[32546]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T14:24:44.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:44 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:44.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:44 smithi096 ceph-mon[29229]: purged_snaps scrub starts 2024-09-13T14:24:44.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:44 smithi096 ceph-mon[29229]: purged_snaps scrub ok 2024-09-13T14:24:44.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:44 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:44.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:44 smithi096 ceph-mon[29229]: osd.3 [v2:172.21.15.96:6826/282100645,v1:172.21.15.96:6827/282100645] boot 2024-09-13T14:24:44.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:44 smithi096 ceph-mon[29229]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T14:24:44.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:44 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:44.493 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:44 smithi096 ceph-mon[39645]: purged_snaps scrub starts 2024-09-13T14:24:44.493 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:44 smithi096 ceph-mon[39645]: purged_snaps scrub ok 2024-09-13T14:24:44.494 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:44 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:44.494 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:44 smithi096 ceph-mon[39645]: osd.3 [v2:172.21.15.96:6826/282100645,v1:172.21.15.96:6827/282100645] boot 2024-09-13T14:24:44.494 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:44 smithi096 ceph-mon[39645]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T14:24:44.494 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:44 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T14:24:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: Detected new or changed devices on smithi096 2024-09-13T14:24:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:45.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: Adjusting osd_memory_target on smithi096 to 4016M 2024-09-13T14:24:45.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:45.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:45.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:45.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:45.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:45.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:45.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:45.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T14:24:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: Detected new or changed devices on smithi096 2024-09-13T14:24:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: Adjusting osd_memory_target on smithi096 to 4016M 2024-09-13T14:24:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:45.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:45.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:45.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:45.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:45.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:45.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:45.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T14:24:45.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:45.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: Detected new or changed devices on smithi096 2024-09-13T14:24:45.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:45.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:45.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:45.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:45.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:45.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:45.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: Adjusting osd_memory_target on smithi096 to 4016M 2024-09-13T14:24:45.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:45.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:45.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:45.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:45.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:45.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:45.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:45.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:46.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:46 smithi204 ceph-mon[32546]: from='client.24190 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:46.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:46 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:24:46.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:46 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:24:46.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:46 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:46 smithi096 ceph-mon[29229]: from='client.24190 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:46 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:24:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:46 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:24:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:46 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:46.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:46 smithi096 ceph-mon[39645]: from='client.24190 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:46.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:46 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:24:46.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:46 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:24:46.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:46 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:47.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:47 smithi204 ceph-mon[32546]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:47.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:47 smithi096 ceph-mon[29229]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:47.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:47 smithi096 ceph-mon[39645]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:48.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:48 smithi204 ceph-mon[32546]: from='client.? 172.21.15.204:0/3301635031' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14115269-3d41-4ebf-8613-71ed59ec0f29"}]: dispatch 2024-09-13T14:24:48.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:48 smithi204 ceph-mon[32546]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14115269-3d41-4ebf-8613-71ed59ec0f29"}]: dispatch 2024-09-13T14:24:48.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:48 smithi204 ceph-mon[32546]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "14115269-3d41-4ebf-8613-71ed59ec0f29"}]': finished 2024-09-13T14:24:48.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:48 smithi204 ceph-mon[32546]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T14:24:48.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:48 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:24:48.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:48 smithi204 ceph-mon[32546]: from='client.? 172.21.15.204:0/234443823' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:24:48.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:48 smithi096 ceph-mon[29229]: from='client.? 172.21.15.204:0/3301635031' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14115269-3d41-4ebf-8613-71ed59ec0f29"}]: dispatch 2024-09-13T14:24:48.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:48 smithi096 ceph-mon[29229]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14115269-3d41-4ebf-8613-71ed59ec0f29"}]: dispatch 2024-09-13T14:24:48.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:48 smithi096 ceph-mon[29229]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "14115269-3d41-4ebf-8613-71ed59ec0f29"}]': finished 2024-09-13T14:24:48.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:48 smithi096 ceph-mon[29229]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T14:24:48.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:48 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:24:48.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:48 smithi096 ceph-mon[29229]: from='client.? 172.21.15.204:0/234443823' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:24:48.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:48 smithi096 ceph-mon[39645]: from='client.? 172.21.15.204:0/3301635031' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14115269-3d41-4ebf-8613-71ed59ec0f29"}]: dispatch 2024-09-13T14:24:48.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:48 smithi096 ceph-mon[39645]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14115269-3d41-4ebf-8613-71ed59ec0f29"}]: dispatch 2024-09-13T14:24:48.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:48 smithi096 ceph-mon[39645]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "14115269-3d41-4ebf-8613-71ed59ec0f29"}]': finished 2024-09-13T14:24:48.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:48 smithi096 ceph-mon[39645]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T14:24:48.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:48 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:24:48.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:48 smithi096 ceph-mon[39645]: from='client.? 172.21.15.204:0/234443823' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:24:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:49 smithi204 ceph-mon[32546]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:49.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:49 smithi096 ceph-mon[29229]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:49.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:49 smithi096 ceph-mon[39645]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:51.560 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:51 smithi204 ceph-mon[32546]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:51.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:51 smithi096 ceph-mon[29229]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:51.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:51 smithi096 ceph-mon[39645]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:53.420 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:53 smithi204 ceph-mon[32546]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:53.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:53 smithi096 ceph-mon[29229]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:53.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:53 smithi096 ceph-mon[39645]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:54.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:54 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T14:24:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:54 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:54 smithi204 ceph-mon[32546]: Deploying daemon osd.4 on smithi204 2024-09-13T14:24:54.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:54 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T14:24:54.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:54 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:54.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:54 smithi096 ceph-mon[29229]: Deploying daemon osd.4 on smithi204 2024-09-13T14:24:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:54 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T14:24:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:54 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:54 smithi096 ceph-mon[39645]: Deploying daemon osd.4 on smithi204 2024-09-13T14:24:55.380 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:55 smithi204 ceph-mon[32546]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:55.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:55 smithi096 ceph-mon[29229]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:55.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:55 smithi096 ceph-mon[39645]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:57.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:57 smithi096 ceph-mon[29229]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:57.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:57 smithi096 ceph-mon[39645]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:57.575 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:57 smithi204 ceph-mon[32546]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:59 smithi204 ceph-mon[32546]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:59 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:59 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:24:59 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:59 smithi096 ceph-mon[29229]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:59 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:59 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:24:59 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:59 smithi096 ceph-mon[39645]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:24:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:59 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:59 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:24:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:24:59 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:01.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:01 smithi204 ceph-mon[32546]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:25:01.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:01 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:01.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:01 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:01.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:01 smithi096 ceph-mon[29229]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:25:01.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:01 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:01.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:01 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:01.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:01 smithi096 ceph-mon[39645]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:25:01.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:01 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:01 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:01.850 INFO:teuthology.orchestra.run.smithi204.stdout:Created osd(s) 4 on host 'smithi204' 2024-09-13T14:25:02.431 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:02 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:02.431 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:02 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:02.431 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:02 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:02.431 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:02 smithi204 ceph-mon[32546]: from='osd.4 [v2:172.21.15.204:6800/1188704632,v1:172.21.15.204:6801/1188704632]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:25:02.431 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:02 smithi204 ceph-mon[32546]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:25:02.431 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:02 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:02.431 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:02 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:02.432 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:02 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:02.456 DEBUG:teuthology.orchestra.run.smithi204:osd.4> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.4.service 2024-09-13T14:25:02.458 INFO:tasks.cephadm:Deploying osd.5 on smithi204 with /dev/vg_nvme/lv_3... 2024-09-13T14:25:02.458 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T14:25:02.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:02.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:02.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:02.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[29229]: from='osd.4 [v2:172.21.15.204:6800/1188704632,v1:172.21.15.204:6801/1188704632]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:25:02.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[29229]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:25:02.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:02.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:02.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:02.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:02.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:02.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:02.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[39645]: from='osd.4 [v2:172.21.15.204:6800/1188704632,v1:172.21.15.204:6801/1188704632]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:25:02.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[39645]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:25:02.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:02.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:02.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:02 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:03.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:03 smithi096 ceph-mon[29229]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:25:03.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:03 smithi096 ceph-mon[29229]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:25:03.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:03 smithi096 ceph-mon[29229]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T14:25:03.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:03 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:25:03.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:03 smithi096 ceph-mon[29229]: from='osd.4 [v2:172.21.15.204:6800/1188704632,v1:172.21.15.204:6801/1188704632]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:03.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:03 smithi096 ceph-mon[29229]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:03.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:03 smithi096 ceph-mon[39645]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:25:03.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:03 smithi096 ceph-mon[39645]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:25:03.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:03 smithi096 ceph-mon[39645]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T14:25:03.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:03 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:25:03.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:03 smithi096 ceph-mon[39645]: from='osd.4 [v2:172.21.15.204:6800/1188704632,v1:172.21.15.204:6801/1188704632]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:03.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:03 smithi096 ceph-mon[39645]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:03.828 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:03 smithi204 ceph-mon[32546]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:25:03.828 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:03 smithi204 ceph-mon[32546]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:25:03.828 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:03 smithi204 ceph-mon[32546]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T14:25:03.829 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:03 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:25:03.829 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:03 smithi204 ceph-mon[32546]: from='osd.4 [v2:172.21.15.204:6800/1188704632,v1:172.21.15.204:6801/1188704632]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:03.829 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:03 smithi204 ceph-mon[32546]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:03.829 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 14:25:03 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T14:25:03.529+0000 7f75a6829640 -1 osd.4 0 waiting for initial osdmap 2024-09-13T14:25:03.829 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 14:25:03 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T14:25:03.533+0000 7f75a1660640 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:25:04.037 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:25:04.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:04 smithi204 ceph-mon[32546]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-09-13T14:25:04.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:04 smithi204 ceph-mon[32546]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T14:25:04.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:04 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:25:04.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:04 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:25:04.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:04 smithi204 ceph-mon[32546]: osd.4 [v2:172.21.15.204:6800/1188704632,v1:172.21.15.204:6801/1188704632] boot 2024-09-13T14:25:04.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:04 smithi204 ceph-mon[32546]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T14:25:04.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:04 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:25:04.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:04 smithi096 ceph-mon[29229]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-09-13T14:25:04.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:04 smithi096 ceph-mon[29229]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T14:25:04.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:04 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:25:04.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:04 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:25:04.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:04 smithi096 ceph-mon[29229]: osd.4 [v2:172.21.15.204:6800/1188704632,v1:172.21.15.204:6801/1188704632] boot 2024-09-13T14:25:04.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:04 smithi096 ceph-mon[29229]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T14:25:04.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:04 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:25:04.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:04 smithi096 ceph-mon[39645]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-09-13T14:25:04.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:04 smithi096 ceph-mon[39645]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T14:25:04.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:04 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:25:04.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:04 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:25:04.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:04 smithi096 ceph-mon[39645]: osd.4 [v2:172.21.15.204:6800/1188704632,v1:172.21.15.204:6801/1188704632] boot 2024-09-13T14:25:04.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:04 smithi096 ceph-mon[39645]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T14:25:04.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:04 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:25:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:05 smithi204 ceph-mon[32546]: purged_snaps scrub starts 2024-09-13T14:25:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:05 smithi204 ceph-mon[32546]: purged_snaps scrub ok 2024-09-13T14:25:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:05 smithi204 ceph-mon[32546]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:25:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:05 smithi204 ceph-mon[32546]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T14:25:05.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:05 smithi096 ceph-mon[29229]: purged_snaps scrub starts 2024-09-13T14:25:05.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:05 smithi096 ceph-mon[29229]: purged_snaps scrub ok 2024-09-13T14:25:05.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:05 smithi096 ceph-mon[29229]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:25:05.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:05 smithi096 ceph-mon[29229]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T14:25:05.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:05 smithi096 ceph-mon[39645]: purged_snaps scrub starts 2024-09-13T14:25:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:05 smithi096 ceph-mon[39645]: purged_snaps scrub ok 2024-09-13T14:25:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:05 smithi096 ceph-mon[39645]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:25:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:05 smithi096 ceph-mon[39645]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T14:25:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[29229]: Detected new or changed devices on smithi204 2024-09-13T14:25:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:07.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:07.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:07.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[29229]: Adjusting osd_memory_target on smithi204 to 17088M 2024-09-13T14:25:07.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:07.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:07.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:07.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:07.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[29229]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T14:25:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[39645]: Detected new or changed devices on smithi204 2024-09-13T14:25:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:07.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:07.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:07.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[39645]: Adjusting osd_memory_target on smithi204 to 17088M 2024-09-13T14:25:07.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:07.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:07.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:07.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:07.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:06 smithi096 ceph-mon[39645]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T14:25:07.183 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:06 smithi204 ceph-mon[32546]: Detected new or changed devices on smithi204 2024-09-13T14:25:07.183 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:06 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:07.183 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:06 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:07.183 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:06 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:07.183 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:06 smithi204 ceph-mon[32546]: Adjusting osd_memory_target on smithi204 to 17088M 2024-09-13T14:25:07.184 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:06 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:07.184 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:06 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:07.184 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:06 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:07.184 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:06 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:07.184 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:06 smithi204 ceph-mon[32546]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T14:25:07.189 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:25:07.209 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi204:vg_nvme/lv_3 2024-09-13T14:25:07.449 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:25:07.827 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:07 smithi204 ceph-mon[32546]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:08.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:07 smithi096 ceph-mon[29229]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:08.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:07 smithi096 ceph-mon[39645]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:09.871 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:09 smithi204 ceph-mon[32546]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:09.871 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:09 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:25:09.871 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:09 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:25:09.871 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:09 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:10.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:09 smithi096 ceph-mon[29229]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:10.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:09 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:25:10.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:09 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:25:10.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:09 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:10.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:09 smithi096 ceph-mon[39645]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:10.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:09 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:25:10.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:09 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:25:10.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:09 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:11.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:10 smithi096 ceph-mon[29229]: from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:11.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:10 smithi096 ceph-mon[39645]: from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:11.154 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:10 smithi204 ceph-mon[32546]: from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:11.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:11 smithi204 ceph-mon[32546]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:11 smithi204 ceph-mon[32546]: from='client.? 172.21.15.204:0/1884915015' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "15148a0c-df44-419e-b0f0-2b1600c207f4"}]: dispatch 2024-09-13T14:25:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:11 smithi204 ceph-mon[32546]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "15148a0c-df44-419e-b0f0-2b1600c207f4"}]: dispatch 2024-09-13T14:25:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:11 smithi204 ceph-mon[32546]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "15148a0c-df44-419e-b0f0-2b1600c207f4"}]': finished 2024-09-13T14:25:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:11 smithi204 ceph-mon[32546]: osdmap e34: 6 total, 5 up, 6 in 2024-09-13T14:25:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:11 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:25:12.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:11 smithi096 ceph-mon[29229]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:12.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:11 smithi096 ceph-mon[29229]: from='client.? 172.21.15.204:0/1884915015' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "15148a0c-df44-419e-b0f0-2b1600c207f4"}]: dispatch 2024-09-13T14:25:12.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:11 smithi096 ceph-mon[29229]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "15148a0c-df44-419e-b0f0-2b1600c207f4"}]: dispatch 2024-09-13T14:25:12.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:11 smithi096 ceph-mon[29229]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "15148a0c-df44-419e-b0f0-2b1600c207f4"}]': finished 2024-09-13T14:25:12.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:11 smithi096 ceph-mon[29229]: osdmap e34: 6 total, 5 up, 6 in 2024-09-13T14:25:12.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:11 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:25:12.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:11 smithi096 ceph-mon[39645]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:12.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:11 smithi096 ceph-mon[39645]: from='client.? 172.21.15.204:0/1884915015' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "15148a0c-df44-419e-b0f0-2b1600c207f4"}]: dispatch 2024-09-13T14:25:12.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:11 smithi096 ceph-mon[39645]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "15148a0c-df44-419e-b0f0-2b1600c207f4"}]: dispatch 2024-09-13T14:25:12.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:11 smithi096 ceph-mon[39645]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "15148a0c-df44-419e-b0f0-2b1600c207f4"}]': finished 2024-09-13T14:25:12.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:11 smithi096 ceph-mon[39645]: osdmap e34: 6 total, 5 up, 6 in 2024-09-13T14:25:12.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:11 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:25:13.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:12 smithi096 ceph-mon[29229]: from='client.? 172.21.15.204:0/1745960664' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:25:13.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:12 smithi096 ceph-mon[39645]: from='client.? 172.21.15.204:0/1745960664' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:25:13.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:12 smithi204 ceph-mon[32546]: from='client.? 172.21.15.204:0/1745960664' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:25:14.119 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:13 smithi204 ceph-mon[32546]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T14:25:14.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:13 smithi096 ceph-mon[29229]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T14:25:14.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:13 smithi096 ceph-mon[39645]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T14:25:16.148 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:15 smithi204 ceph-mon[32546]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-13T14:25:16.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:15 smithi096 ceph-mon[29229]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-13T14:25:16.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:15 smithi096 ceph-mon[39645]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-13T14:25:17.960 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:17 smithi204 ceph-mon[32546]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-13T14:25:17.960 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:17 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T14:25:17.960 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:17 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:18.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:17 smithi096 ceph-mon[29229]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-13T14:25:18.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:17 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T14:25:18.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:17 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:18.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:17 smithi096 ceph-mon[39645]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-13T14:25:18.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:17 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T14:25:18.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:17 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:19.121 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:18 smithi204 ceph-mon[32546]: Deploying daemon osd.5 on smithi204 2024-09-13T14:25:19.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:18 smithi096 ceph-mon[29229]: Deploying daemon osd.5 on smithi204 2024-09-13T14:25:19.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:18 smithi096 ceph-mon[39645]: Deploying daemon osd.5 on smithi204 2024-09-13T14:25:19.959 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:19 smithi204 ceph-mon[32546]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:25:20.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:19 smithi096 ceph-mon[29229]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:25:20.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:19 smithi096 ceph-mon[39645]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:25:21.928 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:21 smithi204 ceph-mon[32546]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:25:22.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:21 smithi096 ceph-mon[29229]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:25:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:21 smithi096 ceph-mon[39645]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:25:23.157 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:22 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:23.157 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:22 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:23.157 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:22 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:23.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:22 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:23.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:22 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:23.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:22 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:23.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:22 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:23.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:22 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:23.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:22 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:24.136 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:23 smithi204 ceph-mon[32546]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:24.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:23 smithi096 ceph-mon[29229]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:24.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:23 smithi096 ceph-mon[39645]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:25.880 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:25 smithi204 ceph-mon[32546]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:25.880 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:25.880 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:25.880 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:25.880 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:25.881 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:25 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:25.881 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:25 smithi204 ceph-mon[32546]: from='osd.5 [v2:172.21.15.204:6808/580729752,v1:172.21.15.204:6809/580729752]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:25:25.881 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:25 smithi204 ceph-mon[32546]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:25:25.916 INFO:teuthology.orchestra.run.smithi204.stdout:Created osd(s) 5 on host 'smithi204' 2024-09-13T14:25:26.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[29229]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:26.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:26.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:26.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:26.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:26.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:26.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[29229]: from='osd.5 [v2:172.21.15.204:6808/580729752,v1:172.21.15.204:6809/580729752]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:25:26.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[29229]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:25:26.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[39645]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:26.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:26.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:26.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:26.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:26.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:26.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[39645]: from='osd.5 [v2:172.21.15.204:6808/580729752,v1:172.21.15.204:6809/580729752]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:25:26.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:25 smithi096 ceph-mon[39645]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:25:26.532 DEBUG:teuthology.orchestra.run.smithi204:osd.5> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.5.service 2024-09-13T14:25:26.534 INFO:tasks.cephadm:Deploying osd.6 on smithi204 with /dev/vg_nvme/lv_2... 2024-09-13T14:25:26.535 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T14:25:27.069 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:26 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:27.069 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:26 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:27.069 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:26 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:27.070 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:26 smithi204 ceph-mon[32546]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:25:27.070 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:26 smithi204 ceph-mon[32546]: osdmap e35: 6 total, 5 up, 6 in 2024-09-13T14:25:27.070 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:26 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:25:27.070 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:26 smithi204 ceph-mon[32546]: from='osd.5 [v2:172.21.15.204:6808/580729752,v1:172.21.15.204:6809/580729752]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:27.070 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:26 smithi204 ceph-mon[32546]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:27.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:27.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[39645]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:25:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[39645]: osdmap e35: 6 total, 5 up, 6 in 2024-09-13T14:25:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:25:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[39645]: from='osd.5 [v2:172.21.15.204:6808/580729752,v1:172.21.15.204:6809/580729752]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[39645]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:27.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:27.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:27.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:27.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[29229]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:25:27.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[29229]: osdmap e35: 6 total, 5 up, 6 in 2024-09-13T14:25:27.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:25:27.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[29229]: from='osd.5 [v2:172.21.15.204:6808/580729752,v1:172.21.15.204:6809/580729752]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:27.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:26 smithi096 ceph-mon[29229]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:27.447 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 14:25:27 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T14:25:27.163+0000 7fabc1e78640 -1 osd.5 0 waiting for initial osdmap 2024-09-13T14:25:27.447 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 14:25:27 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T14:25:27.170+0000 7fabbac89640 -1 osd.5 36 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:25:27.901 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:25:28.144 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:27 smithi204 ceph-mon[32546]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:28.144 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:27 smithi204 ceph-mon[32546]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-09-13T14:25:28.144 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:27 smithi204 ceph-mon[32546]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T14:25:28.144 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:27 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:25:28.144 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:27 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:25:28.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:27 smithi096 ceph-mon[29229]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:28.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:27 smithi096 ceph-mon[29229]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-09-13T14:25:28.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:27 smithi096 ceph-mon[29229]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T14:25:28.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:27 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:25:28.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:27 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:25:28.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:27 smithi096 ceph-mon[39645]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:25:28.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:27 smithi096 ceph-mon[39645]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-09-13T14:25:28.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:27 smithi096 ceph-mon[39645]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T14:25:28.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:27 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:25:28.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:27 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:25:29.417 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:29 smithi204 ceph-mon[32546]: purged_snaps scrub starts 2024-09-13T14:25:29.417 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:29 smithi204 ceph-mon[32546]: purged_snaps scrub ok 2024-09-13T14:25:29.417 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:29 smithi204 ceph-mon[32546]: osd.5 [v2:172.21.15.204:6808/580729752,v1:172.21.15.204:6809/580729752] boot 2024-09-13T14:25:29.417 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:29 smithi204 ceph-mon[32546]: osdmap e37: 6 total, 6 up, 6 in 2024-09-13T14:25:29.418 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:29 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:25:29.418 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:29 smithi204 ceph-mon[32546]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:29 smithi096 ceph-mon[29229]: purged_snaps scrub starts 2024-09-13T14:25:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:29 smithi096 ceph-mon[29229]: purged_snaps scrub ok 2024-09-13T14:25:29.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:29 smithi096 ceph-mon[29229]: osd.5 [v2:172.21.15.204:6808/580729752,v1:172.21.15.204:6809/580729752] boot 2024-09-13T14:25:29.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:29 smithi096 ceph-mon[29229]: osdmap e37: 6 total, 6 up, 6 in 2024-09-13T14:25:29.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:29 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:25:29.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:29 smithi096 ceph-mon[29229]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:29 smithi096 ceph-mon[39645]: purged_snaps scrub starts 2024-09-13T14:25:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:29 smithi096 ceph-mon[39645]: purged_snaps scrub ok 2024-09-13T14:25:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:29 smithi096 ceph-mon[39645]: osd.5 [v2:172.21.15.204:6808/580729752,v1:172.21.15.204:6809/580729752] boot 2024-09-13T14:25:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:29 smithi096 ceph-mon[39645]: osdmap e37: 6 total, 6 up, 6 in 2024-09-13T14:25:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:29 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:25:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:29 smithi096 ceph-mon[39645]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:30.446 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:30 smithi204 ceph-mon[32546]: osdmap e38: 6 total, 6 up, 6 in 2024-09-13T14:25:30.446 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:30 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:30.446 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:30 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:30.447 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:30 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:30.447 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:30 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:30.447 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:30 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:30.447 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:30 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:30.447 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:30 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:30.447 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:30 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:30.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[29229]: osdmap e38: 6 total, 6 up, 6 in 2024-09-13T14:25:30.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:30.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:30.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:30.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:30.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:30.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:30.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:30.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:30.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[39645]: osdmap e38: 6 total, 6 up, 6 in 2024-09-13T14:25:30.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:30.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:30.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:30.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:30.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:30.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:30.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:30.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:30 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:31.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:31 smithi204 ceph-mon[32546]: Detected new or changed devices on smithi204 2024-09-13T14:25:31.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:31 smithi204 ceph-mon[32546]: Adjusting osd_memory_target on smithi204 to 8544M 2024-09-13T14:25:31.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:31 smithi204 ceph-mon[32546]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T14:25:31.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:31 smithi204 ceph-mon[32546]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:31.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:31 smithi096 ceph-mon[29229]: Detected new or changed devices on smithi204 2024-09-13T14:25:31.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:31 smithi096 ceph-mon[29229]: Adjusting osd_memory_target on smithi204 to 8544M 2024-09-13T14:25:31.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:31 smithi096 ceph-mon[29229]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T14:25:31.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:31 smithi096 ceph-mon[29229]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:31.474 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:31 smithi096 ceph-mon[39645]: Detected new or changed devices on smithi204 2024-09-13T14:25:31.475 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:31 smithi096 ceph-mon[39645]: Adjusting osd_memory_target on smithi204 to 8544M 2024-09-13T14:25:31.475 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:31 smithi096 ceph-mon[39645]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T14:25:31.475 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:31 smithi096 ceph-mon[39645]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:31.516 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:25:31.537 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi204:vg_nvme/lv_2 2024-09-13T14:25:31.781 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:25:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:33 smithi204 ceph-mon[32546]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:33.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:33 smithi096 ceph-mon[29229]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:33.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:33 smithi096 ceph-mon[39645]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:34.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:34 smithi204 ceph-mon[32546]: from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:34.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:34 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:25:34.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:34 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:25:34.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:34 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:34.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:34 smithi096 ceph-mon[29229]: from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:34.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:34 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:25:34.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:34 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:25:34.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:34 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:34.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:34 smithi096 ceph-mon[39645]: from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:34.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:34 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:25:34.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:34 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:25:34.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:34 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:35.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:35 smithi096 ceph-mon[29229]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:35.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:35 smithi096 ceph-mon[39645]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:35.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:35 smithi204 ceph-mon[32546]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:36.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:36 smithi096 ceph-mon[29229]: from='client.? 172.21.15.204:0/2007678914' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8404e088-3523-4c21-b7c2-0a31b22e0279"}]: dispatch 2024-09-13T14:25:36.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:36 smithi096 ceph-mon[29229]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8404e088-3523-4c21-b7c2-0a31b22e0279"}]: dispatch 2024-09-13T14:25:36.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:36 smithi096 ceph-mon[29229]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8404e088-3523-4c21-b7c2-0a31b22e0279"}]': finished 2024-09-13T14:25:36.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:36 smithi096 ceph-mon[29229]: osdmap e40: 7 total, 6 up, 7 in 2024-09-13T14:25:36.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:36 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:25:36.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:36 smithi096 ceph-mon[29229]: from='client.? 172.21.15.204:0/609005167' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:25:36.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:36 smithi096 ceph-mon[39645]: from='client.? 172.21.15.204:0/2007678914' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8404e088-3523-4c21-b7c2-0a31b22e0279"}]: dispatch 2024-09-13T14:25:36.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:36 smithi096 ceph-mon[39645]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8404e088-3523-4c21-b7c2-0a31b22e0279"}]: dispatch 2024-09-13T14:25:36.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:36 smithi096 ceph-mon[39645]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8404e088-3523-4c21-b7c2-0a31b22e0279"}]': finished 2024-09-13T14:25:36.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:36 smithi096 ceph-mon[39645]: osdmap e40: 7 total, 6 up, 7 in 2024-09-13T14:25:36.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:36 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:25:36.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:36 smithi096 ceph-mon[39645]: from='client.? 172.21.15.204:0/609005167' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:25:36.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:36 smithi204 ceph-mon[32546]: from='client.? 172.21.15.204:0/2007678914' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8404e088-3523-4c21-b7c2-0a31b22e0279"}]: dispatch 2024-09-13T14:25:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:36 smithi204 ceph-mon[32546]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8404e088-3523-4c21-b7c2-0a31b22e0279"}]: dispatch 2024-09-13T14:25:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:36 smithi204 ceph-mon[32546]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8404e088-3523-4c21-b7c2-0a31b22e0279"}]': finished 2024-09-13T14:25:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:36 smithi204 ceph-mon[32546]: osdmap e40: 7 total, 6 up, 7 in 2024-09-13T14:25:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:36 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:25:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:36 smithi204 ceph-mon[32546]: from='client.? 172.21.15.204:0/609005167' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:25:37.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:37 smithi204 ceph-mon[32546]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:38.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:37 smithi096 ceph-mon[29229]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:38.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:37 smithi096 ceph-mon[39645]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:39.857 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:39 smithi204 ceph-mon[32546]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:40.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:39 smithi096 ceph-mon[29229]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:40.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:39 smithi096 ceph-mon[39645]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:41.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:41 smithi204 ceph-mon[32546]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:42.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:41 smithi096 ceph-mon[29229]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:42.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:41 smithi096 ceph-mon[39645]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:42.956 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:42 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T14:25:42.957 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:42 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:42.957 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:42 smithi204 ceph-mon[32546]: Deploying daemon osd.6 on smithi204 2024-09-13T14:25:43.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:42 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T14:25:43.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:42 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:43.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:42 smithi096 ceph-mon[29229]: Deploying daemon osd.6 on smithi204 2024-09-13T14:25:43.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:42 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T14:25:43.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:42 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:43.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:42 smithi096 ceph-mon[39645]: Deploying daemon osd.6 on smithi204 2024-09-13T14:25:43.771 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:43 smithi204 ceph-mon[32546]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:44.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:43 smithi096 ceph-mon[29229]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:44.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:43 smithi096 ceph-mon[39645]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:45.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:45 smithi204 ceph-mon[32546]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:46.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:45 smithi096 ceph-mon[29229]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:46.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:45 smithi096 ceph-mon[39645]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:47.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:47 smithi204 ceph-mon[32546]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:47.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:47 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:47.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:47 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:47.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:47 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:48.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:47 smithi096 ceph-mon[29229]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:48.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:47 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:48.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:47 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:48.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:47 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:48.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:47 smithi096 ceph-mon[39645]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:48.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:47 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:48.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:47 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:48.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:47 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:49.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:49 smithi204 ceph-mon[32546]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:50.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:49 smithi096 ceph-mon[29229]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:50.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:49 smithi096 ceph-mon[39645]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:50.705 INFO:teuthology.orchestra.run.smithi204.stdout:Created osd(s) 6 on host 'smithi204' 2024-09-13T14:25:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[29229]: from='osd.6 [v2:172.21.15.204:6816/118365497,v1:172.21.15.204:6817/118365497]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:25:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[29229]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:25:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[29229]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:51.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:51.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:51.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[39645]: from='osd.6 [v2:172.21.15.204:6816/118365497,v1:172.21.15.204:6817/118365497]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:25:51.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[39645]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:25:51.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[39645]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:51.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:51.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:51.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:51 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:51.338 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:51 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:51.338 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:51 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:51.338 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:51 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:51.338 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:51 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:51.338 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:51 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:51.338 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:51 smithi204 ceph-mon[32546]: from='osd.6 [v2:172.21.15.204:6816/118365497,v1:172.21.15.204:6817/118365497]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:25:51.338 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:51 smithi204 ceph-mon[32546]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:25:51.338 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:51 smithi204 ceph-mon[32546]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:51.338 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:51 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:51.338 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:51 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:51.338 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:51 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:51.362 DEBUG:teuthology.orchestra.run.smithi204:osd.6> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.6.service 2024-09-13T14:25:51.365 INFO:tasks.cephadm:Deploying osd.7 on smithi204 with /dev/vg_nvme/lv_1... 2024-09-13T14:25:51.365 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T14:25:52.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:52 smithi096 ceph-mon[29229]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:25:52.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:52 smithi096 ceph-mon[29229]: osdmap e41: 7 total, 6 up, 7 in 2024-09-13T14:25:52.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:52 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:25:52.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:52 smithi096 ceph-mon[29229]: from='osd.6 [v2:172.21.15.204:6816/118365497,v1:172.21.15.204:6817/118365497]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:52.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:52 smithi096 ceph-mon[29229]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:52.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:52 smithi096 ceph-mon[39645]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:25:52.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:52 smithi096 ceph-mon[39645]: osdmap e41: 7 total, 6 up, 7 in 2024-09-13T14:25:52.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:52 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:25:52.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:52 smithi096 ceph-mon[39645]: from='osd.6 [v2:172.21.15.204:6816/118365497,v1:172.21.15.204:6817/118365497]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:52.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:52 smithi096 ceph-mon[39645]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:52.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:52 smithi204 ceph-mon[32546]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:25:52.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:52 smithi204 ceph-mon[32546]: osdmap e41: 7 total, 6 up, 7 in 2024-09-13T14:25:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:52 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:25:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:52 smithi204 ceph-mon[32546]: from='osd.6 [v2:172.21.15.204:6816/118365497,v1:172.21.15.204:6817/118365497]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:52 smithi204 ceph-mon[32546]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:25:52.469 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 14:25:52 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T14:25:52.069+0000 7efeeadf5640 -1 osd.6 0 waiting for initial osdmap 2024-09-13T14:25:52.469 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 14:25:52 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T14:25:52.075+0000 7efee7c30640 -1 osd.6 42 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:25:52.485 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:25:53.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:53 smithi204 ceph-mon[32546]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-09-13T14:25:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:53 smithi204 ceph-mon[32546]: osdmap e42: 7 total, 6 up, 7 in 2024-09-13T14:25:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:53 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:25:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:53 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:25:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:53 smithi204 ceph-mon[32546]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:53.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:53 smithi096 ceph-mon[29229]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-09-13T14:25:53.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:53 smithi096 ceph-mon[29229]: osdmap e42: 7 total, 6 up, 7 in 2024-09-13T14:25:53.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:53 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:25:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:53 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:25:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:53 smithi096 ceph-mon[29229]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:53.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:53 smithi096 ceph-mon[39645]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-09-13T14:25:53.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:53 smithi096 ceph-mon[39645]: osdmap e42: 7 total, 6 up, 7 in 2024-09-13T14:25:53.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:53 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:25:53.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:53 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:25:53.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:53 smithi096 ceph-mon[39645]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:25:54.079 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:54 smithi204 ceph-mon[32546]: purged_snaps scrub starts 2024-09-13T14:25:54.079 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:54 smithi204 ceph-mon[32546]: purged_snaps scrub ok 2024-09-13T14:25:54.079 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:54 smithi204 ceph-mon[32546]: osd.6 [v2:172.21.15.204:6816/118365497,v1:172.21.15.204:6817/118365497] boot 2024-09-13T14:25:54.079 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:54 smithi204 ceph-mon[32546]: osdmap e43: 7 total, 7 up, 7 in 2024-09-13T14:25:54.079 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:54 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:25:54.079 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:54 smithi204 ceph-mon[32546]: osdmap e44: 7 total, 7 up, 7 in 2024-09-13T14:25:54.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:54 smithi096 ceph-mon[29229]: purged_snaps scrub starts 2024-09-13T14:25:54.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:54 smithi096 ceph-mon[29229]: purged_snaps scrub ok 2024-09-13T14:25:54.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:54 smithi096 ceph-mon[29229]: osd.6 [v2:172.21.15.204:6816/118365497,v1:172.21.15.204:6817/118365497] boot 2024-09-13T14:25:54.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:54 smithi096 ceph-mon[29229]: osdmap e43: 7 total, 7 up, 7 in 2024-09-13T14:25:54.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:54 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:25:54.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:54 smithi096 ceph-mon[29229]: osdmap e44: 7 total, 7 up, 7 in 2024-09-13T14:25:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:54 smithi096 ceph-mon[39645]: purged_snaps scrub starts 2024-09-13T14:25:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:54 smithi096 ceph-mon[39645]: purged_snaps scrub ok 2024-09-13T14:25:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:54 smithi096 ceph-mon[39645]: osd.6 [v2:172.21.15.204:6816/118365497,v1:172.21.15.204:6817/118365497] boot 2024-09-13T14:25:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:54 smithi096 ceph-mon[39645]: osdmap e43: 7 total, 7 up, 7 in 2024-09-13T14:25:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:54 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:25:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:54 smithi096 ceph-mon[39645]: osdmap e44: 7 total, 7 up, 7 in 2024-09-13T14:25:55.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:55 smithi204 ceph-mon[32546]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:25:55.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:55.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:55 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:55 smithi204 ceph-mon[32546]: osdmap e45: 7 total, 7 up, 7 in 2024-09-13T14:25:55.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[29229]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:25:55.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:55.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:55.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:55.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:55.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:55.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:55.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:55.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:55.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:55.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[29229]: osdmap e45: 7 total, 7 up, 7 in 2024-09-13T14:25:55.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[39645]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:25:55.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:55.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:55.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:55.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:55.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:25:55.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:55.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:55.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:55.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:25:55.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:55 smithi096 ceph-mon[39645]: osdmap e45: 7 total, 7 up, 7 in 2024-09-13T14:25:56.340 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:56 smithi204 ceph-mon[32546]: Detected new or changed devices on smithi204 2024-09-13T14:25:56.340 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:56 smithi204 ceph-mon[32546]: Adjusting osd_memory_target on smithi204 to 5696M 2024-09-13T14:25:56.346 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-09-13T14:25:56.367 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi204:vg_nvme/lv_1 2024-09-13T14:25:56.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:56 smithi096 ceph-mon[29229]: Detected new or changed devices on smithi204 2024-09-13T14:25:56.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:56 smithi096 ceph-mon[29229]: Adjusting osd_memory_target on smithi204 to 5696M 2024-09-13T14:25:56.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:56 smithi096 ceph-mon[39645]: Detected new or changed devices on smithi204 2024-09-13T14:25:56.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:56 smithi096 ceph-mon[39645]: Adjusting osd_memory_target on smithi204 to 5696M 2024-09-13T14:25:56.612 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:25:57.413 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:57 smithi204 ceph-mon[32546]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:25:57.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:57 smithi096 ceph-mon[29229]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:25:57.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:57 smithi096 ceph-mon[39645]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:25:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:59 smithi096 ceph-mon[29229]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:25:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:59 smithi096 ceph-mon[29229]: from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:59 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:25:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:59 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:25:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:25:59 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:59 smithi096 ceph-mon[39645]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:25:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:59 smithi096 ceph-mon[39645]: from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:59 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:25:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:59 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:25:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:25:59 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:59.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:59 smithi204 ceph-mon[32546]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:25:59.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:59 smithi204 ceph-mon[32546]: from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:59.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:59 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:25:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:59 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:25:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:25:59 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:01.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:01 smithi096 ceph-mon[29229]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:01 smithi096 ceph-mon[29229]: from='client.? 172.21.15.204:0/218165508' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "93b23a35-0059-4e03-a2ba-28fee6431892"}]: dispatch 2024-09-13T14:26:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:01 smithi096 ceph-mon[29229]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "93b23a35-0059-4e03-a2ba-28fee6431892"}]: dispatch 2024-09-13T14:26:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:01 smithi096 ceph-mon[29229]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "93b23a35-0059-4e03-a2ba-28fee6431892"}]': finished 2024-09-13T14:26:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:01 smithi096 ceph-mon[29229]: osdmap e46: 8 total, 7 up, 8 in 2024-09-13T14:26:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:01 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:01 smithi096 ceph-mon[29229]: from='client.? 172.21.15.204:0/2276020647' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:26:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:01 smithi096 ceph-mon[39645]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:01 smithi096 ceph-mon[39645]: from='client.? 172.21.15.204:0/218165508' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "93b23a35-0059-4e03-a2ba-28fee6431892"}]: dispatch 2024-09-13T14:26:01.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:01 smithi096 ceph-mon[39645]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "93b23a35-0059-4e03-a2ba-28fee6431892"}]: dispatch 2024-09-13T14:26:01.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:01 smithi096 ceph-mon[39645]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "93b23a35-0059-4e03-a2ba-28fee6431892"}]': finished 2024-09-13T14:26:01.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:01 smithi096 ceph-mon[39645]: osdmap e46: 8 total, 7 up, 8 in 2024-09-13T14:26:01.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:01 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:01.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:01 smithi096 ceph-mon[39645]: from='client.? 172.21.15.204:0/2276020647' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:26:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:01 smithi204 ceph-mon[32546]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:01 smithi204 ceph-mon[32546]: from='client.? 172.21.15.204:0/218165508' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "93b23a35-0059-4e03-a2ba-28fee6431892"}]: dispatch 2024-09-13T14:26:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:01 smithi204 ceph-mon[32546]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "93b23a35-0059-4e03-a2ba-28fee6431892"}]: dispatch 2024-09-13T14:26:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:01 smithi204 ceph-mon[32546]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "93b23a35-0059-4e03-a2ba-28fee6431892"}]': finished 2024-09-13T14:26:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:01 smithi204 ceph-mon[32546]: osdmap e46: 8 total, 7 up, 8 in 2024-09-13T14:26:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:01 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:01 smithi204 ceph-mon[32546]: from='client.? 172.21.15.204:0/2276020647' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:26:03.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:03 smithi096 ceph-mon[29229]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:03.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:03 smithi096 ceph-mon[39645]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:03.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:03 smithi204 ceph-mon[32546]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:05.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:05 smithi096 ceph-mon[29229]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:05.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:05 smithi096 ceph-mon[39645]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:05.675 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:05 smithi204 ceph-mon[32546]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:07.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:07 smithi096 ceph-mon[29229]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:07.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:07 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T14:26:07.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:07 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:07.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:07 smithi096 ceph-mon[29229]: Deploying daemon osd.7 on smithi204 2024-09-13T14:26:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:07 smithi096 ceph-mon[39645]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:07 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T14:26:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:07 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:07 smithi096 ceph-mon[39645]: Deploying daemon osd.7 on smithi204 2024-09-13T14:26:07.632 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:07 smithi204 ceph-mon[32546]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:07.633 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:07 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T14:26:07.633 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:07 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:07.633 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:07 smithi204 ceph-mon[32546]: Deploying daemon osd.7 on smithi204 2024-09-13T14:26:09.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:09 smithi096 ceph-mon[29229]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:09 smithi096 ceph-mon[39645]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:09.577 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:09 smithi204 ceph-mon[32546]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:11.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:11 smithi096 ceph-mon[29229]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:11.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:11 smithi096 ceph-mon[39645]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:11.574 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:11 smithi204 ceph-mon[32546]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:12.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:12 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:12.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:12 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:12.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:12 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:12.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:12 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:12.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:12 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:12.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:12 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:12.641 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:12 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:12.641 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:12 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:12.641 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:12 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:13.546 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:13 smithi204 ceph-mon[32546]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:13.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:13 smithi096 ceph-mon[29229]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:13.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:13 smithi096 ceph-mon[39645]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:15.018 INFO:teuthology.orchestra.run.smithi204.stdout:Created osd(s) 7 on host 'smithi204' 2024-09-13T14:26:15.642 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:15 smithi204 ceph-mon[32546]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:15.642 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:15 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:15.642 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:15 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:15.643 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:15 smithi204 ceph-mon[32546]: from='osd.7 [v2:172.21.15.204:6824/3529639424,v1:172.21.15.204:6825/3529639424]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:26:15.643 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:15 smithi204 ceph-mon[32546]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:26:15.667 DEBUG:teuthology.orchestra.run.smithi204:osd.7> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.7.service 2024-09-13T14:26:15.670 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-13T14:26:15.670 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T14:26:15.698 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:15 smithi096 ceph-mon[29229]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:15.698 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:15 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:15.698 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:15 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:15.698 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:15 smithi096 ceph-mon[29229]: from='osd.7 [v2:172.21.15.204:6824/3529639424,v1:172.21.15.204:6825/3529639424]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:26:15.698 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:15 smithi096 ceph-mon[29229]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:26:15.699 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:15 smithi096 ceph-mon[39645]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:15.699 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:15 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:15.699 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:15 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:15.699 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:15 smithi096 ceph-mon[39645]: from='osd.7 [v2:172.21.15.204:6824/3529639424,v1:172.21.15.204:6825/3529639424]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:26:15.699 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:15 smithi096 ceph-mon[39645]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:26:15.928 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:17.025 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:16 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:17.025 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:16 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:17.025 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:16 smithi204 ceph-mon[32546]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:26:17.026 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:16 smithi204 ceph-mon[32546]: osdmap e47: 8 total, 7 up, 8 in 2024-09-13T14:26:17.026 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:16 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:17.026 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:16 smithi204 ceph-mon[32546]: from='osd.7 [v2:172.21.15.204:6824/3529639424,v1:172.21.15.204:6825/3529639424]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:26:17.026 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:16 smithi204 ceph-mon[32546]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:26:17.049 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:16 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:17.049 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:16 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:17.050 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:16 smithi096 ceph-mon[29229]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:26:17.050 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:16 smithi096 ceph-mon[29229]: osdmap e47: 8 total, 7 up, 8 in 2024-09-13T14:26:17.050 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:16 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:17.050 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:16 smithi096 ceph-mon[29229]: from='osd.7 [v2:172.21.15.204:6824/3529639424,v1:172.21.15.204:6825/3529639424]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:26:17.050 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:16 smithi096 ceph-mon[29229]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:26:17.051 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:16 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:17.051 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:16 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:17.051 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:16 smithi096 ceph-mon[39645]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:26:17.051 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:16 smithi096 ceph-mon[39645]: osdmap e47: 8 total, 7 up, 8 in 2024-09-13T14:26:17.051 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:16 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:17.052 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:16 smithi096 ceph-mon[39645]: from='osd.7 [v2:172.21.15.204:6824/3529639424,v1:172.21.15.204:6825/3529639424]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:26:17.052 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:16 smithi096 ceph-mon[39645]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-09-13T14:26:17.469 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 14:26:17 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T14:26:17.023+0000 7fe67d85c640 -1 osd.7 0 waiting for initial osdmap 2024-09-13T14:26:17.469 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 14:26:17 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T14:26:17.029+0000 7fe678693640 -1 osd.7 48 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:26:17.717 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:26:18.053 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:17 smithi096 ceph-mon[29229]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:18.054 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:17 smithi096 ceph-mon[29229]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-09-13T14:26:18.054 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:17 smithi096 ceph-mon[29229]: osdmap e48: 8 total, 7 up, 8 in 2024-09-13T14:26:18.054 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:17 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:18.054 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:17 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:18.054 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:17 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/259041301' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:26:18.054 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:17 smithi096 ceph-mon[39645]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:18.054 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:17 smithi096 ceph-mon[39645]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-09-13T14:26:18.054 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:17 smithi096 ceph-mon[39645]: osdmap e48: 8 total, 7 up, 8 in 2024-09-13T14:26:18.054 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:17 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:18.055 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:17 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:18.055 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:17 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/259041301' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:26:18.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:17 smithi204 ceph-mon[32546]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:26:18.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:17 smithi204 ceph-mon[32546]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-09-13T14:26:18.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:17 smithi204 ceph-mon[32546]: osdmap e48: 8 total, 7 up, 8 in 2024-09-13T14:26:18.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:17 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:18.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:17 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:18.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:17 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/259041301' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:26:18.423 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":48,"num_osds":8,"num_up_osds":7,"osd_up_since":1726237553,"num_in_osds":8,"osd_in_since":1726237560,"num_remapped_pgs":0} 2024-09-13T14:26:19.173 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:19 smithi204 ceph-mon[32546]: purged_snaps scrub starts 2024-09-13T14:26:19.174 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:19 smithi204 ceph-mon[32546]: purged_snaps scrub ok 2024-09-13T14:26:19.174 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:19 smithi204 ceph-mon[32546]: osd.7 [v2:172.21.15.204:6824/3529639424,v1:172.21.15.204:6825/3529639424] boot 2024-09-13T14:26:19.174 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:19 smithi204 ceph-mon[32546]: osdmap e49: 8 total, 8 up, 8 in 2024-09-13T14:26:19.174 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:19 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:19.174 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:19 smithi204 ceph-mon[32546]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:19.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:19 smithi096 ceph-mon[29229]: purged_snaps scrub starts 2024-09-13T14:26:19.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:19 smithi096 ceph-mon[29229]: purged_snaps scrub ok 2024-09-13T14:26:19.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:19 smithi096 ceph-mon[29229]: osd.7 [v2:172.21.15.204:6824/3529639424,v1:172.21.15.204:6825/3529639424] boot 2024-09-13T14:26:19.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:19 smithi096 ceph-mon[29229]: osdmap e49: 8 total, 8 up, 8 in 2024-09-13T14:26:19.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:19 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:19.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:19 smithi096 ceph-mon[29229]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:19.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:19 smithi096 ceph-mon[39645]: purged_snaps scrub starts 2024-09-13T14:26:19.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:19 smithi096 ceph-mon[39645]: purged_snaps scrub ok 2024-09-13T14:26:19.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:19 smithi096 ceph-mon[39645]: osd.7 [v2:172.21.15.204:6824/3529639424,v1:172.21.15.204:6825/3529639424] boot 2024-09-13T14:26:19.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:19 smithi096 ceph-mon[39645]: osdmap e49: 8 total, 8 up, 8 in 2024-09-13T14:26:19.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:19 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:19.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:19 smithi096 ceph-mon[39645]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:19.425 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T14:26:19.673 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:20.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: osdmap e50: 8 total, 8 up, 8 in 2024-09-13T14:26:20.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: Detected new or changed devices on smithi204 2024-09-13T14:26:20.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:20.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:20.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:20.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:20.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:20.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:20.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: Adjusting osd_memory_target on smithi204 to 4272M 2024-09-13T14:26:20.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:20.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:20.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:20.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:20.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:20.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:20.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:20.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:20.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: osdmap e50: 8 total, 8 up, 8 in 2024-09-13T14:26:20.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: Detected new or changed devices on smithi204 2024-09-13T14:26:20.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:20.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:20.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:20.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:20.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:20.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:20.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: Adjusting osd_memory_target on smithi204 to 4272M 2024-09-13T14:26:20.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:20.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:20.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:20.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:20.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:20.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:20.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:20.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:20 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:20.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: osdmap e50: 8 total, 8 up, 8 in 2024-09-13T14:26:20.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: Detected new or changed devices on smithi204 2024-09-13T14:26:20.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:20.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:20.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:20.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:20.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:20.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:20.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: Adjusting osd_memory_target on smithi204 to 4272M 2024-09-13T14:26:20.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:20.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:20.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:20.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:20.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:20.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:20.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:20.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:20 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:21.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:21 smithi096 ceph-mon[39645]: osdmap e51: 8 total, 8 up, 8 in 2024-09-13T14:26:21.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:21 smithi096 ceph-mon[39645]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:21.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:21 smithi096 ceph-mon[29229]: osdmap e51: 8 total, 8 up, 8 in 2024-09-13T14:26:21.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:21 smithi096 ceph-mon[29229]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:21.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:21 smithi204 ceph-mon[32546]: osdmap e51: 8 total, 8 up, 8 in 2024-09-13T14:26:21.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:21 smithi204 ceph-mon[32546]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:21.567 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:26:22.214 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:22 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3427031916' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:26:22.215 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:22 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3427031916' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:26:22.240 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":51,"num_osds":8,"num_up_osds":8,"osd_up_since":1726237578,"num_in_osds":8,"osd_in_since":1726237560,"num_remapped_pgs":0} 2024-09-13T14:26:22.240 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:26:22.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:22 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3427031916' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:26:22.482 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:23.238 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:23 smithi096 ceph-mon[29229]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:23.238 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:23 smithi096 ceph-mon[39645]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:23.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:23 smithi204 ceph-mon[32546]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:24.140 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:26:24.140 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":51,"fsid":"51e50ed6-71db-11ef-bceb-c7b262605968","created":"2024-09-13T14:21:16.367777+0000","modified":"2024-09-13T14:26:20.025036+0000","last_up_change":"2024-09-13T14:26:18.016817+0000","last_in_change":"2024-09-13T14:26:00.677688+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:24:20.514144+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6802","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6803","nonce":766102065}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6804","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6805","nonce":766102065}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6808","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6809","nonce":766102065}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6806","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6807","nonce":766102065}]},"public_addr":"172.21.15.96:6803/766102065","cluster_addr":"172.21.15.96:6805/766102065","heartbeat_back_addr":"172.21.15.96:6809/766102065","heartbeat_front_addr":"172.21.15.96:6807/766102065","state":["exists","up"]},{"osd":1,"uuid":"1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6810","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6811","nonce":401053665}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6812","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6813","nonce":401053665}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6816","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6817","nonce":401053665}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6814","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6815","nonce":401053665}]},"public_addr":"172.21.15.96:6811/401053665","cluster_addr":"172.21.15.96:6813/401053665","heartbeat_back_addr":"172.21.15.96:6817/401053665","heartbeat_front_addr":"172.21.15.96:6815/401053665","state":["exists","up"]},{"osd":2,"uuid":"6a92a5b3-bdc7-499a-8b0f-2b0936fd9598","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6818","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6819","nonce":2386475672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6820","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6821","nonce":2386475672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6824","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6825","nonce":2386475672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6822","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6823","nonce":2386475672}]},"public_addr":"172.21.15.96:6819/2386475672","cluster_addr":"172.21.15.96:6821/2386475672","heartbeat_back_addr":"172.21.15.96:6825/2386475672","heartbeat_front_addr":"172.21.15.96:6823/2386475672","state":["exists","up"]},{"osd":3,"uuid":"1230646c-934e-4eb4-b4b9-16be52ffba66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6826","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6827","nonce":282100645}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6828","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6829","nonce":282100645}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6832","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6833","nonce":282100645}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6830","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6831","nonce":282100645}]},"public_addr":"172.21.15.96:6827/282100645","cluster_addr":"172.21.15.96:6829/282100645","heartbeat_back_addr":"172.21.15.96:6833/282100645","heartbeat_front_addr":"172.21.15.96:6831/282100645","state":["exists","up"]},{"osd":4,"uuid":"14115269-3d41-4ebf-8613-71ed59ec0f29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6800","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6801","nonce":1188704632}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6802","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6803","nonce":1188704632}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6806","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6807","nonce":1188704632}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6804","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6805","nonce":1188704632}]},"public_addr":"172.21.15.204:6801/1188704632","cluster_addr":"172.21.15.204:6803/1188704632","heartbeat_back_addr":"172.21.15.204:6807/1188704632","heartbeat_front_addr":"172.21.15.204:6805/1188704632","state":["exists","up"]},{"osd":5,"uuid":"15148a0c-df44-419e-b0f0-2b1600c207f4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":38,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6808","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6809","nonce":580729752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6810","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6811","nonce":580729752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6814","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6815","nonce":580729752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6812","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6813","nonce":580729752}]},"public_addr":"172.21.15.204:6809/580729752","cluster_addr":"172.21.15.204:6811/580729752","heartbeat_back_addr":"172.21.15.204:6815/580729752","heartbeat_front_addr":"172.21.15.204:6813/580729752","state":["exists","up"]},{"osd":6,"uuid":"8404e088-3523-4c21-b7c2-0a31b22e0279","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":44,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6816","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6817","nonce":118365497}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6818","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6819","nonce":118365497}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6822","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6823","nonce":118365497}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6820","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6821","nonce":118365497}]},"public_addr":"172.21.15.204:6817/118365497","cluster_addr":"172.21.15.204:6819/118365497","heartbeat_back_addr":"172.21.15.204:6823/118365497","heartbeat_front_addr":"172.21.15.204:6821/118365497","state":["exists","up"]},{"osd":7,"uuid":"93b23a35-0059-4e03-a2ba-28fee6431892","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6824","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6825","nonce":3529639424}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6826","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6827","nonce":3529639424}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6830","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6831","nonce":3529639424}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6828","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6829","nonce":3529639424}]},"public_addr":"172.21.15.204:6825/3529639424","cluster_addr":"172.21.15.204:6827/3529639424","heartbeat_back_addr":"172.21.15.204:6831/3529639424","heartbeat_front_addr":"172.21.15.204:6829/3529639424","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:28.841233+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:53.021149+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:17.120825+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:41.339674+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:02.571454+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:26.608523+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:51.119331+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:26:16.050687+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.96:6801/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/1494034928":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/19119516":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/1644412166":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6801/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:6800/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:6800/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/376655558":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/1790922234":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/2292899363":"2024-09-14T14:21:45.592521+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:26:24.461 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:24 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1241690118' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:26:24.462 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:24 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1241690118' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:26:24.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:24 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1241690118' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:26:24.752 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-13T14:24:20.514144+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-13T14:26:24.753 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-13T14:26:24.994 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:25.448 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:25 smithi096 ceph-mon[29229]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:25.448 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:25 smithi096 ceph-mon[39645]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:25 smithi204 ceph-mon[32546]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:26.720 INFO:teuthology.orchestra.run.smithi096.stdout:pg_num: 1 2024-09-13T14:26:27.308 INFO:tasks.cephadm:Adding prometheus.a on smithi204 2024-09-13T14:26:27.308 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch apply prometheus '1;smithi204=a' 2024-09-13T14:26:27.564 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:26:27.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:27 smithi096 ceph-mon[29229]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:27.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:27 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1951773907' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T14:26:27.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:27 smithi096 ceph-mon[39645]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:27.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:27 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1951773907' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T14:26:27.685 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:27 smithi204 ceph-mon[32546]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:27.685 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:27 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1951773907' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T14:26:29.334 INFO:teuthology.orchestra.run.smithi204.stdout:Scheduled prometheus update... 2024-09-13T14:26:29.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:29 smithi096 ceph-mon[39645]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:29 smithi096 ceph-mon[29229]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:29.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:29 smithi204 ceph-mon[32546]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:29.936 DEBUG:teuthology.orchestra.run.smithi204:prometheus.a> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@prometheus.a.service 2024-09-13T14:26:29.939 INFO:tasks.cephadm:Adding node-exporter.a on smithi096 2024-09-13T14:26:29.939 INFO:tasks.cephadm:Adding node-exporter.b on smithi204 2024-09-13T14:26:29.939 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch apply node-exporter '2;smithi096=a;smithi204=b' 2024-09-13T14:26:30.242 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:26:30.657 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:30 smithi204 ceph-mon[32546]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi204=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:30.657 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:30 smithi204 ceph-mon[32546]: Saving service prometheus spec with placement smithi204=a;count:1 2024-09-13T14:26:30.657 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:30 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:30.658 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:30 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:30.658 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:30 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:30.658 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:30 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:30.658 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:30 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:30.658 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:30 smithi204 ceph-mon[32546]: Deploying daemon prometheus.a on smithi204 2024-09-13T14:26:30.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[29229]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi204=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:30.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[29229]: Saving service prometheus spec with placement smithi204=a;count:1 2024-09-13T14:26:30.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:30.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:30.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:30.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:30.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:30.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[29229]: Deploying daemon prometheus.a on smithi204 2024-09-13T14:26:30.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[39645]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi204=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:30.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[39645]: Saving service prometheus spec with placement smithi204=a;count:1 2024-09-13T14:26:30.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:30.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:30.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:30.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:30.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:30.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:30 smithi096 ceph-mon[39645]: Deploying daemon prometheus.a on smithi204 2024-09-13T14:26:31.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:31 smithi204 ceph-mon[32546]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:31.725 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:31 smithi096 ceph-mon[29229]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:31.725 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:31 smithi096 ceph-mon[39645]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:32.557 INFO:teuthology.orchestra.run.smithi204.stdout:Scheduled node-exporter update... 2024-09-13T14:26:33.579 DEBUG:teuthology.orchestra.run.smithi096:node-exporter.a> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@node-exporter.a.service 2024-09-13T14:26:33.582 DEBUG:teuthology.orchestra.run.smithi204:node-exporter.b> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@node-exporter.b.service 2024-09-13T14:26:33.584 INFO:tasks.cephadm:Adding alertmanager.a on smithi096 2024-09-13T14:26:33.584 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch apply alertmanager '1;smithi096=a' 2024-09-13T14:26:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:33 smithi096 ceph-mon[29229]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:33 smithi096 ceph-mon[29229]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi096=a;smithi204=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:33.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:33 smithi096 ceph-mon[29229]: Saving service node-exporter spec with placement smithi096=a;smithi204=b;count:2 2024-09-13T14:26:33.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:33 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:33.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:33 smithi096 ceph-mon[39645]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:33.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:33 smithi096 ceph-mon[39645]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi096=a;smithi204=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:33.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:33 smithi096 ceph-mon[39645]: Saving service node-exporter spec with placement smithi096=a;smithi204=b;count:2 2024-09-13T14:26:33.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:33 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:33.865 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:26:33.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:33 smithi204 ceph-mon[32546]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:33.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:33 smithi204 ceph-mon[32546]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi096=a;smithi204=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:33.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:33 smithi204 ceph-mon[32546]: Saving service node-exporter spec with placement smithi096=a;smithi204=b;count:2 2024-09-13T14:26:33.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:33 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:35.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:35 smithi096 ceph-mon[29229]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:35.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:35 smithi096 ceph-mon[39645]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:35 smithi204 ceph-mon[32546]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:37.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:37 smithi096 ceph-mon[29229]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:37.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:37 smithi096 ceph-mon[39645]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:37.831 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:37 smithi204 ceph-mon[32546]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:38.277 INFO:teuthology.orchestra.run.smithi204.stdout:Scheduled alertmanager update... 2024-09-13T14:26:39.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:39 smithi096 ceph-mon[29229]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:39.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:39 smithi096 ceph-mon[29229]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi096=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:39.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:39 smithi096 ceph-mon[29229]: Saving service alertmanager spec with placement smithi096=a;count:1 2024-09-13T14:26:39.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:39 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:39.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:39 smithi096 ceph-mon[39645]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:39.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:39 smithi096 ceph-mon[39645]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi096=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:39.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:39 smithi096 ceph-mon[39645]: Saving service alertmanager spec with placement smithi096=a;count:1 2024-09-13T14:26:39.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:39 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:39.660 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:39 smithi204 ceph-mon[32546]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:39.660 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:39 smithi204 ceph-mon[32546]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi096=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:39.660 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:39 smithi204 ceph-mon[32546]: Saving service alertmanager spec with placement smithi096=a;count:1 2024-09-13T14:26:39.660 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:39 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:40.107 DEBUG:teuthology.orchestra.run.smithi096:alertmanager.a> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@alertmanager.a.service 2024-09-13T14:26:40.110 INFO:tasks.cephadm:Adding grafana.a on smithi204 2024-09-13T14:26:40.110 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph orch apply grafana '1;smithi204=a' 2024-09-13T14:26:41.379 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:41 smithi204 ceph-mon[32546]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:41.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:41 smithi096 ceph-mon[29229]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:41.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:41 smithi096 ceph-mon[39645]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:41.596 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:26:43.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:43 smithi096 ceph-mon[29229]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:43.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:43 smithi096 ceph-mon[39645]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:43.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:43 smithi204 ceph-mon[32546]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:44.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 podman[59419]: 2024-09-13 14:26:44.381391984 +0000 UTC m=+0.142374954 container create 9afae272a3c42ce71981b95df456be86761c98096d07c124a253d72d94d3956f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:26:44.835 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 podman[59419]: 2024-09-13 14:26:44.618420745 +0000 UTC m=+0.379403717 container init 9afae272a3c42ce71981b95df456be86761c98096d07c124a253d72d94d3956f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:26:44.836 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 podman[59419]: 2024-09-13 14:26:44.621284471 +0000 UTC m=+0.382267455 container start 9afae272a3c42ce71981b95df456be86761c98096d07c124a253d72d94d3956f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:26:44.836 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.645Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T14:26:44.836 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.645Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T14:26:44.836 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.645Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi204 (none))" 2024-09-13T14:26:44.836 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.645Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:26:44.836 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.645Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:26:44.837 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.646Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:26:44.837 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.647Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T14:26:44.837 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.647Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:26:44.837 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.647Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:26:44.837 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.649Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:26:44.837 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.649Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.505µs 2024-09-13T14:26:44.837 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.649Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:26:44.838 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.650Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-09-13T14:26:44.838 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.650Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=16.894µs wal_replay_duration=1.313968ms wbl_replay_duration=321ns total_replay_duration=1.352798ms 2024-09-13T14:26:44.838 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.651Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:26:44.838 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.651Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T14:26:44.838 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.651Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:26:44.838 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.666Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.814894ms db_storage=782ns remote_storage=1.054µs web_handler=392ns query_engine=468ns scrape=436.795µs scrape_sd=21.779µs notify=722ns notify_sd=1.15µs rules=14.020782ms tracing=28.74µs 2024-09-13T14:26:44.838 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.666Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T14:26:44.839 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:26:44.666Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:26:45.154 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 bash[59419]: 9afae272a3c42ce71981b95df456be86761c98096d07c124a253d72d94d3956f 2024-09-13T14:26:45.155 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:26:44 smithi204 systemd[1]: Started Ceph prometheus.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:26:45.324 INFO:teuthology.orchestra.run.smithi204.stdout:Scheduled grafana update... 2024-09-13T14:26:45.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:45 smithi204 ceph-mon[32546]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:45.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:45 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T14:26:45.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:45 smithi096 ceph-mon[39645]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:45.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:45.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:45.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:45.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:45 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T14:26:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:45 smithi096 ceph-mon[29229]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:45 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T14:26:45.917 DEBUG:teuthology.orchestra.run.smithi204:grafana.a> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@grafana.a.service 2024-09-13T14:26:45.920 INFO:tasks.cephadm:Setting up client nodes... 2024-09-13T14:26:45.920 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T14:26:46.168 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ignoring --setuser ceph since I am not root 2024-09-13T14:26:46.168 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ignoring --setgroup ceph since I am not root 2024-09-13T14:26:46.168 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:46 smithi204 ceph-mgr[34406]: -- 172.21.15.204:0/1202878051 <== mon.2 v2:172.21.15.96:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f2f9e38340 con 0x55f2f9e45000 2024-09-13T14:26:46.169 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:46.167+0000 7fa010af3ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:26:46.198 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ignoring --setuser ceph since I am not root 2024-09-13T14:26:46.198 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ignoring --setgroup ceph since I am not root 2024-09-13T14:26:46.198 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-mgr[29462]: -- 172.21.15.96:0/2220280825 <== mon.0 v2:172.21.15.96:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561b84518340 con 0x561b84524800 2024-09-13T14:26:46.198 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:46.196+0000 7f89fcd51ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:26:46.200 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:46.448 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-mon[29229]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi204=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:46.448 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-mon[29229]: Saving service grafana spec with placement smithi204=a;count:1 2024-09-13T14:26:46.449 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:46.449 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-mon[29229]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T14:26:46.449 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-mon[29229]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-13T14:26:46.449 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-mon[39645]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi204=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:46.449 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-mon[39645]: Saving service grafana spec with placement smithi204=a;count:1 2024-09-13T14:26:46.449 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:46.449 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-mon[39645]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T14:26:46.449 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-mon[39645]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-13T14:26:46.450 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:46.254+0000 7f89fcd51ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:26:46.450 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:46.319+0000 7f89fcd51ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:26:46.450 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:46.447+0000 7f89fcd51ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:26:46.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:46 smithi204 ceph-mon[32546]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi204=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:46.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:46 smithi204 ceph-mon[32546]: Saving service grafana spec with placement smithi204=a;count:1 2024-09-13T14:26:46.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:46 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' 2024-09-13T14:26:46.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:46 smithi204 ceph-mon[32546]: from='mgr.14150 172.21.15.96:0/4177091065' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T14:26:46.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:46 smithi204 ceph-mon[32546]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-13T14:26:46.469 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:46.222+0000 7fa010af3ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:26:46.469 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:46.287+0000 7fa010af3ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:26:46.469 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:46.412+0000 7fa010af3ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:26:46.803 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:46.549+0000 7fa010af3ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:26:46.803 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:46.801+0000 7fa010af3ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:26:46.822 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:46.592+0000 7f89fcd51ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:26:47.218 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:46.861+0000 7fa010af3ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:26:47.218 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:47 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:47.022+0000 7fa010af3ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:26:47.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:46.851+0000 7f89fcd51ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:26:47.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:46 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:46.910+0000 7f89fcd51ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:26:47.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:47.074+0000 7f89fcd51ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:26:47.998 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:47.932+0000 7f89fcd51ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:26:47.998 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:47.997+0000 7f89fcd51ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:26:48.119 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:47 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:47.861+0000 7fa010af3ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:26:48.119 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:47 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:47.926+0000 7fa010af3ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:26:48.347 INFO:teuthology.orchestra.run.smithi096.stdout:[client.0] 2024-09-13T14:26:48.347 INFO:teuthology.orchestra.run.smithi096.stdout: key = AQCoS+Rm2epvFBAAqIlmn03cBvoeZsVqTa7NJA== 2024-09-13T14:26:48.636 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:48 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/418249062' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:26:48.636 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:48 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/418249062' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:26:48.636 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:48 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/418249062' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:26:48.636 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:48 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/418249062' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:26:48.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:48 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/418249062' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:26:48.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:48 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/418249062' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:26:48.719 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:48.631+0000 7fa010af3ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:26:48.985 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:48.736+0000 7f89fcd51ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:26:48.985 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:48.855+0000 7f89fcd51ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:26:49.048 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:48.742+0000 7fa010af3ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:26:49.048 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:48.864+0000 7fa010af3ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:26:49.049 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:48.923+0000 7fa010af3ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:26:49.049 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:49.047+0000 7fa010af3ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:26:49.050 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:26:49.050 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-13T14:26:49.050 DEBUG:teuthology.orchestra.run.smithi096:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-13T14:26:49.092 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T14:26:49.252 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:48.984+0000 7f89fcd51ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:26:49.252 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:49 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:49.048+0000 7f89fcd51ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:26:49.252 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:49 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:49.182+0000 7f89fcd51ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:26:49.252 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:49 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:49.250+0000 7f89fcd51ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:26:49.333 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:49.117+0000 7fa010af3ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:26:49.333 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:49.179+0000 7fa010af3ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:26:49.333 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:49.332+0000 7fa010af3ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:26:49.355 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:26:49.514 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:49 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:49.310+0000 7f89fcd51ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:26:49.515 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:49 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:49.448+0000 7f89fcd51ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:26:49.515 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:49 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:49.513+0000 7f89fcd51ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:26:49.600 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:49.404+0000 7fa010af3ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:26:49.600 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:49.472+0000 7fa010af3ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:26:49.600 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:49.560+0000 7fa010af3ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:26:49.777 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:49 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:49.580+0000 7f89fcd51ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:26:49.777 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:49 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:49.664+0000 7f89fcd51ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:26:49.777 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:49 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:49.720+0000 7f89fcd51ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:26:49.777 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:49 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:49.776+0000 7f89fcd51ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:26:49.968 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:49.624+0000 7fa010af3ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:26:49.968 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:49.685+0000 7fa010af3ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:26:50.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-mon[32546]: Standby manager daemon x restarted 2024-09-13T14:26:50.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-mon[32546]: Standby manager daemon x started 2024-09-13T14:26:50.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/3385943232' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:26:50.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/3385943232' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:26:50.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/3385943232' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:26:50.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/3385943232' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:26:50.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-mon[32546]: Active manager daemon y restarted 2024-09-13T14:26:50.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-mon[32546]: Activating manager daemon y 2024-09-13T14:26:50.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-mon[32546]: osdmap e52: 8 total, 8 up, 8 in 2024-09-13T14:26:50.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-mon[32546]: mgrmap e17: y(active, starting, since 0.0106394s), standbys: x 2024-09-13T14:26:50.719 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:50.472+0000 7fa010af3ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:26:50.720 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: [13/Sep/2024:14:26:50] ENGINE Bus STARTING 2024-09-13T14:26:50.720 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: CherryPy Checker: 2024-09-13T14:26:50.720 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: The Application mounted at '' has an empty config. 2024-09-13T14:26:50.720 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: 2024-09-13T14:26:50.720 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: [13/Sep/2024:14:26:50] ENGINE Serving on http://:::9283 2024-09-13T14:26:50.720 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: [13/Sep/2024:14:26:50] ENGINE Bus STARTED 2024-09-13T14:26:50.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[29229]: Standby manager daemon x restarted 2024-09-13T14:26:50.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[29229]: Standby manager daemon x started 2024-09-13T14:26:50.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/3385943232' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:26:50.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/3385943232' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:26:50.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/3385943232' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:26:50.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/3385943232' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:26:50.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[29229]: Active manager daemon y restarted 2024-09-13T14:26:50.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[29229]: Activating manager daemon y 2024-09-13T14:26:50.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[29229]: osdmap e52: 8 total, 8 up, 8 in 2024-09-13T14:26:50.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[29229]: mgrmap e17: y(active, starting, since 0.0106394s), standbys: x 2024-09-13T14:26:50.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[39645]: Standby manager daemon x restarted 2024-09-13T14:26:50.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[39645]: Standby manager daemon x started 2024-09-13T14:26:50.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/3385943232' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:26:50.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/3385943232' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:26:50.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/3385943232' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:26:50.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/3385943232' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:26:50.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[39645]: Active manager daemon y restarted 2024-09-13T14:26:50.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[39645]: Activating manager daemon y 2024-09-13T14:26:50.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[39645]: osdmap e52: 8 total, 8 up, 8 in 2024-09-13T14:26:50.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-mon[39645]: mgrmap e17: y(active, starting, since 0.0106394s), standbys: x 2024-09-13T14:26:50.824 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:50.512+0000 7f89fcd51ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:26:50.824 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:26:50] ENGINE Bus STARTING 2024-09-13T14:26:50.824 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: CherryPy Checker: 2024-09-13T14:26:50.824 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: The Application mounted at '' has an empty config. 2024-09-13T14:26:50.824 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:50.824 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:26:50] ENGINE Serving on http://:::9283 2024-09-13T14:26:50.825 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:26:50] ENGINE Bus STARTED 2024-09-13T14:26:51.271 INFO:teuthology.orchestra.run.smithi204.stdout:[client.1] 2024-09-13T14:26:51.271 INFO:teuthology.orchestra.run.smithi204.stdout: key = AQCrS+Rm3B3oDxAAiQao/7NltkYI/cEEh2DFrw== 2024-09-13T14:26:51.587 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:26:51.588 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:26:51.588 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:26:51.588 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:26:51.588 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:26:51.588 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:26:51.588 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:26:51.588 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:26:51.588 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:26:51.588 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:26:51.588 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:26:51.589 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:26:51.589 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:51.589 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:26:51.589 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:26:51.589 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:26:51.589 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: Manager daemon y is now available 2024-09-13T14:26:51.589 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:51.589 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:51.589 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:51.589 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:26:51.589 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:26:51.590 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='client.? 172.21.15.204:0/2917248078' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:26:51.590 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:26:51.590 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:51 smithi204 ceph-mon[32546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:26:51.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:26:51.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:26:51.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:26:51.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:26:51.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:26:51.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:26:51.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:26:51.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:26:51.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:26:51.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:26:51.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:26:51.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:26:51.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:51.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:26:51.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:26:51.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:26:51.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: Manager daemon y is now available 2024-09-13T14:26:51.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:51.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:51.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:51.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:26:51.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:26:51.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='client.? 172.21.15.204:0/2917248078' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:26:51.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:26:51.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[29229]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:26:51.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:26:51.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:26:51.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:26:51.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:26:51.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:26:51.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:26:51.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:26:51.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:26:51.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:26:51.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:26:51.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:26:51.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:26:51.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:26:51.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:26:51.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:26:51.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:26:51.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: Manager daemon y is now available 2024-09-13T14:26:51.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:51.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:51.828 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:51.828 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:26:51.828 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:26:51.828 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='client.? 172.21.15.204:0/2917248078' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:26:51.828 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:26:51.828 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-mon[39645]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:26:51.828 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:51.724+0000 7f899fb14640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:26:51] ENGINE Error in HTTPServer.serve 2024-09-13T14:26:51.828 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: Traceback (most recent call last): 2024-09-13T14:26:51.828 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:26:51.829 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: self._connections.run(self.expiration_interval) 2024-09-13T14:26:51.829 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:26:51.829 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: self._run(expiration_interval) 2024-09-13T14:26:51.829 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:26:51.829 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:26:51.829 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:26:51.829 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:26:51.829 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:26:51.829 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: s = self.context.wrap_socket( 2024-09-13T14:26:51.829 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:26:51.829 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: return self.sslsocket_class._create( 2024-09-13T14:26:51.830 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:26:51.830 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: self.do_handshake() 2024-09-13T14:26:51.830 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:26:51.830 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: self._sslobj.do_handshake() 2024-09-13T14:26:51.830 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:26:51.830 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:26:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:26:52.217 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:26:52 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:26:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:26:52.243 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-09-13T14:26:52.244 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-13T14:26:52.244 DEBUG:teuthology.orchestra.run.smithi204:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-13T14:26:52.281 INFO:tasks.cephadm:Creating RBD pool 2024-09-13T14:26:52.281 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool create rbd 8 2024-09-13T14:26:52.523 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:52.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[39645]: mgrmap e18: y(active, since 1.02584s), standbys: x 2024-09-13T14:26:52.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[39645]: [13/Sep/2024:14:26:51] ENGINE Bus STARTING 2024-09-13T14:26:52.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[39645]: [13/Sep/2024:14:26:51] ENGINE Serving on https://172.21.15.96:7150 2024-09-13T14:26:52.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[39645]: [13/Sep/2024:14:26:51] ENGINE Bus STARTED 2024-09-13T14:26:52.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[39645]: [13/Sep/2024:14:26:51] ENGINE Error in HTTPServer.serve 2024-09-13T14:26:52.573 INFO:journalctl@ceph.mon.c.smithi096.stdout: Traceback (most recent call last): 2024-09-13T14:26:52.573 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:26:52.573 INFO:journalctl@ceph.mon.c.smithi096.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:26:52.573 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:26:52.573 INFO:journalctl@ceph.mon.c.smithi096.stdout: self._run(expiration_interval) 2024-09-13T14:26:52.573 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:26:52.573 INFO:journalctl@ceph.mon.c.smithi096.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:26:52.573 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:26:52.573 INFO:journalctl@ceph.mon.c.smithi096.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:26:52.573 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:26:52.574 INFO:journalctl@ceph.mon.c.smithi096.stdout: s = self.context.wrap_socket( 2024-09-13T14:26:52.574 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:26:52.574 INFO:journalctl@ceph.mon.c.smithi096.stdout: return self.sslsocket_class._create( 2024-09-13T14:26:52.574 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:26:52.574 INFO:journalctl@ceph.mon.c.smithi096.stdout: self.do_handshake() 2024-09-13T14:26:52.574 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:26:52.574 INFO:journalctl@ceph.mon.c.smithi096.stdout: self._sslobj.do_handshake() 2024-09-13T14:26:52.574 INFO:journalctl@ceph.mon.c.smithi096.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:26:52.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:52.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:52.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:52.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[29229]: mgrmap e18: y(active, since 1.02584s), standbys: x 2024-09-13T14:26:52.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[29229]: [13/Sep/2024:14:26:51] ENGINE Bus STARTING 2024-09-13T14:26:52.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[29229]: [13/Sep/2024:14:26:51] ENGINE Serving on https://172.21.15.96:7150 2024-09-13T14:26:52.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[29229]: [13/Sep/2024:14:26:51] ENGINE Bus STARTED 2024-09-13T14:26:52.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[29229]: [13/Sep/2024:14:26:51] ENGINE Error in HTTPServer.serve 2024-09-13T14:26:52.575 INFO:journalctl@ceph.mon.a.smithi096.stdout: Traceback (most recent call last): 2024-09-13T14:26:52.575 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:26:52.575 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:26:52.575 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:26:52.575 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._run(expiration_interval) 2024-09-13T14:26:52.575 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:26:52.575 INFO:journalctl@ceph.mon.a.smithi096.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:26:52.575 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:26:52.575 INFO:journalctl@ceph.mon.a.smithi096.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:26:52.575 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:26:52.576 INFO:journalctl@ceph.mon.a.smithi096.stdout: s = self.context.wrap_socket( 2024-09-13T14:26:52.576 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:26:52.576 INFO:journalctl@ceph.mon.a.smithi096.stdout: return self.sslsocket_class._create( 2024-09-13T14:26:52.576 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:26:52.576 INFO:journalctl@ceph.mon.a.smithi096.stdout: self.do_handshake() 2024-09-13T14:26:52.576 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:26:52.576 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._sslobj.do_handshake() 2024-09-13T14:26:52.576 INFO:journalctl@ceph.mon.a.smithi096.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:26:52.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:52.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:52.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:52 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:52.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:52 smithi204 ceph-mon[32546]: mgrmap e18: y(active, since 1.02584s), standbys: x 2024-09-13T14:26:52.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:52 smithi204 ceph-mon[32546]: [13/Sep/2024:14:26:51] ENGINE Bus STARTING 2024-09-13T14:26:52.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:52 smithi204 ceph-mon[32546]: [13/Sep/2024:14:26:51] ENGINE Serving on https://172.21.15.96:7150 2024-09-13T14:26:52.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:52 smithi204 ceph-mon[32546]: [13/Sep/2024:14:26:51] ENGINE Bus STARTED 2024-09-13T14:26:52.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:52 smithi204 ceph-mon[32546]: [13/Sep/2024:14:26:51] ENGINE Error in HTTPServer.serve 2024-09-13T14:26:52.969 INFO:journalctl@ceph.mon.b.smithi204.stdout: Traceback (most recent call last): 2024-09-13T14:26:52.969 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:26:52.969 INFO:journalctl@ceph.mon.b.smithi204.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:26:52.969 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:26:52.969 INFO:journalctl@ceph.mon.b.smithi204.stdout: self._run(expiration_interval) 2024-09-13T14:26:52.969 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:26:52.970 INFO:journalctl@ceph.mon.b.smithi204.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:26:52.970 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:26:52.970 INFO:journalctl@ceph.mon.b.smithi204.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:26:52.970 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:26:52.970 INFO:journalctl@ceph.mon.b.smithi204.stdout: s = self.context.wrap_socket( 2024-09-13T14:26:52.970 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:26:52.970 INFO:journalctl@ceph.mon.b.smithi204.stdout: return self.sslsocket_class._create( 2024-09-13T14:26:52.970 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:26:52.970 INFO:journalctl@ceph.mon.b.smithi204.stdout: self.do_handshake() 2024-09-13T14:26:52.970 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:26:52.970 INFO:journalctl@ceph.mon.b.smithi204.stdout: self._sslobj.do_handshake() 2024-09-13T14:26:52.970 INFO:journalctl@ceph.mon.b.smithi204.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:26:52.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:52 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:52.971 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:52 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:52.971 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:52 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:53.818 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:53 smithi096 ceph-mon[39645]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:53.818 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:53 smithi096 ceph-mon[39645]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-13T14:26:53.818 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:53 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:53.818 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:53 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:53.819 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:53 smithi096 ceph-mon[29229]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:53.819 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:53 smithi096 ceph-mon[29229]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-13T14:26:53.819 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:53 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:53.819 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:53 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:53.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:53 smithi204 ceph-mon[32546]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:53.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:53 smithi204 ceph-mon[32546]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-13T14:26:53.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:53 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:53.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:53 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:54.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:54 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:54.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:54 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:54.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:54 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:54.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:54 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:54.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:54 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:54.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:54 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:54.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:54 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:54.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:54 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:54.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:54 smithi204 ceph-mon[32546]: Adjusting osd_memory_target on smithi204 to 4016M 2024-09-13T14:26:54.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:54 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:54.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:54 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:54.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:54 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:54.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:54 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3346181776' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:26:55.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:55.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:55.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:55.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[29229]: Adjusting osd_memory_target on smithi204 to 4016M 2024-09-13T14:26:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3346181776' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:26:55.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:55.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:55.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:55.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:55.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:55.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:55.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:55.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:26:55.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[39645]: Adjusting osd_memory_target on smithi204 to 4016M 2024-09-13T14:26:55.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:55.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:55.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:55.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:54 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3346181776' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:26:55.113 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'rbd' created 2024-09-13T14:26:55.811 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- rbd pool init rbd 2024-09-13T14:26:56.068 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:56.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:26:56.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:26:56.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:26:56.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:26:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:26:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:26:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:26:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:26:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: Deploying daemon alertmanager.a on smithi096 2024-09-13T14:26:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-13T14:26:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3346181776' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T14:26:56.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[39645]: osdmap e53: 8 total, 8 up, 8 in 2024-09-13T14:26:56.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:26:56.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:26:56.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:26:56.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:26:56.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:56.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:26:56.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:26:56.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:26:56.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:26:56.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:56.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:56.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:56.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:56.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:56.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: Deploying daemon alertmanager.a on smithi096 2024-09-13T14:26:56.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-13T14:26:56.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3346181776' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T14:26:56.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:55 smithi096 ceph-mon[29229]: osdmap e53: 8 total, 8 up, 8 in 2024-09-13T14:26:56.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:26:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:26:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:26:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:26:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:26:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:26:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:26:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:26:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:56.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:26:56.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: Deploying daemon alertmanager.a on smithi096 2024-09-13T14:26:56.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-13T14:26:56.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3346181776' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T14:26:56.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:55 smithi204 ceph-mon[32546]: osdmap e53: 8 total, 8 up, 8 in 2024-09-13T14:26:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:57 smithi204 ceph-mon[32546]: osdmap e54: 8 total, 8 up, 8 in 2024-09-13T14:26:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:57 smithi204 ceph-mon[32546]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:57.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:57 smithi096 ceph-mon[29229]: osdmap e54: 8 total, 8 up, 8 in 2024-09-13T14:26:57.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:57 smithi096 ceph-mon[29229]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:57.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:57 smithi096 ceph-mon[39645]: osdmap e54: 8 total, 8 up, 8 in 2024-09-13T14:26:57.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:57 smithi096 ceph-mon[39645]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:58.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:58 smithi096 ceph-mon[29229]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:26:58.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:58 smithi096 ceph-mon[29229]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T14:26:58.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:58 smithi096 ceph-mon[39645]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:26:58.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:58 smithi096 ceph-mon[39645]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T14:26:58.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:58 smithi204 ceph-mon[32546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:26:58.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:58 smithi204 ceph-mon[32546]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T14:26:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:59 smithi204 ceph-mon[32546]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T14:26:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:26:59 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1303212305' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:26:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:59 smithi096 ceph-mon[29229]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T14:26:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:26:59 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1303212305' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:26:59.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:59 smithi096 ceph-mon[39645]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T14:26:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:26:59 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1303212305' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:27:00.384 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:00 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1303212305' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T14:27:00.384 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:00 smithi096 ceph-mon[29229]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T14:27:00.384 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:00 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1303212305' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T14:27:00.384 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:00 smithi096 ceph-mon[39645]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T14:27:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:00 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1303212305' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T14:27:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:00 smithi204 ceph-mon[32546]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T14:27:01.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:01 smithi204 ceph-mon[32546]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T14:27:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:01 smithi204 ceph-mon[32546]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 16 op/s 2024-09-13T14:27:01.516 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:01 smithi096 ceph-mon[29229]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T14:27:01.517 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:01 smithi096 ceph-mon[29229]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 16 op/s 2024-09-13T14:27:01.517 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:01 smithi096 ceph-mon[39645]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T14:27:01.517 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:01 smithi096 ceph-mon[39645]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 16 op/s 2024-09-13T14:27:01.851 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-13T14:27:02.400 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:02 smithi096 ceph-mon[29229]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:27:02.401 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:02 smithi096 ceph-mon[29229]: Cluster is now healthy 2024-09-13T14:27:02.401 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:02 smithi096 ceph-mon[29229]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T14:27:02.401 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:02 smithi096 ceph-mon[39645]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:27:02.401 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:02 smithi096 ceph-mon[39645]: Cluster is now healthy 2024-09-13T14:27:02.401 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:02 smithi096 ceph-mon[39645]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T14:27:02.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:27:02 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:27:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:27:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:02 smithi204 ceph-mon[32546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:27:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:02 smithi204 ceph-mon[32546]: Cluster is now healthy 2024-09-13T14:27:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:02 smithi204 ceph-mon[32546]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T14:27:03.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:03 smithi096 ceph-mon[29229]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 377 B/s wr, 0 op/s 2024-09-13T14:27:03.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:03 smithi096 ceph-mon[39645]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 377 B/s wr, 0 op/s 2024-09-13T14:27:03.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:03 smithi204 ceph-mon[32546]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 377 B/s wr, 0 op/s 2024-09-13T14:27:04.045 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:05.831 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:05 smithi096 ceph-mon[39645]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-09-13T14:27:05.831 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:05 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:05.831 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:05 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:05.831 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:05 smithi096 ceph-mon[29229]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-09-13T14:27:05.831 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:05 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:05.831 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:05 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:05 smithi204 ceph-mon[32546]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-09-13T14:27:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:05 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:05 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:05.970 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:27:06.659 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3473004627' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T14:27:06.660 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:06.660 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:06.660 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:06.660 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:06.660 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3473004627' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T14:27:06.660 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:06.660 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:06.660 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:06.660 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:06.967 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:27:06] ENGINE Bus STOPPING 2024-09-13T14:27:06.967 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:27:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:27:06.967 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:27:06] ENGINE Bus STOPPED 2024-09-13T14:27:06.967 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:27:06] ENGINE Bus STARTING 2024-09-13T14:27:06.967 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:27:06] ENGINE Serving on http://:::9283 2024-09-13T14:27:06.968 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:27:06 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:27:06] ENGINE Bus STARTED 2024-09-13T14:27:06.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:06 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3473004627' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T14:27:06.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:06 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:06.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:06 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:06.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:06 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:06.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:06 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:07.287 INFO:teuthology.orchestra.run.smithi096.stdout:[{"version":18,"timestamp":"2024-09-13T14:26:54.092432+0000","name":"","changes":[{"name":"osd/host:smithi204/osd_memory_target","previous_value":"4479518924","new_value":"4211083468"}]}] 2024-09-13T14:27:07.288 INFO:tasks.ceph_manager:config epoch is 18 2024-09-13T14:27:07.288 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T14:27:07.288 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T14:27:07.288 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T14:27:07.533 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:07.892 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:07 smithi096 ceph-mon[29229]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 276 B/s wr, 0 op/s 2024-09-13T14:27:07.893 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:07 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:07.893 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:07 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:07.893 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:07 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:27:07.893 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:07 smithi096 ceph-mon[29229]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:27:07.893 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:07 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:07.893 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:07 smithi096 ceph-mon[29229]: Deploying daemon grafana.a on smithi204 2024-09-13T14:27:07.893 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:07 smithi096 ceph-mon[39645]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 276 B/s wr, 0 op/s 2024-09-13T14:27:07.893 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:07 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:07.893 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:07 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:07.894 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:07 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:27:07.894 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:07 smithi096 ceph-mon[39645]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:27:07.894 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:07 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:07.894 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:07 smithi096 ceph-mon[39645]: Deploying daemon grafana.a on smithi204 2024-09-13T14:27:07.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:07 smithi204 ceph-mon[32546]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 276 B/s wr, 0 op/s 2024-09-13T14:27:07.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:07 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:07.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:07 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:07.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:07 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:27:07.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:07 smithi204 ceph-mon[32546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:27:07.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:07 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:07.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:07 smithi204 ceph-mon[32546]: Deploying daemon grafana.a on smithi204 2024-09-13T14:27:08.624 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:27:08 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[68212]: ts=2024-09-13T14:27:08.386Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000702247s 2024-09-13T14:27:09.322 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:27:09.893 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:09 smithi096 ceph-mon[29229]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 243 B/s wr, 0 op/s 2024-09-13T14:27:09.893 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:09 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/4103437861' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:27:09.894 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:09 smithi096 ceph-mon[39645]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 243 B/s wr, 0 op/s 2024-09-13T14:27:09.894 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:09 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/4103437861' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:27:09.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:09 smithi204 ceph-mon[32546]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 243 B/s wr, 0 op/s 2024-09-13T14:27:09.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:09 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/4103437861' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:27:10.041 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":20,"flags":0,"active_gid":24335,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6800","nonce":776211071},{"type":"v1","addr":"172.21.15.96:6801","nonce":776211071}]},"active_addr":"172.21.15.96:6801/776211071","active_change":"2024-09-13T14:26:50.514430+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24331,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.96:8443/","prometheus":"http://172.21.15.96:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":52,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.96:0","nonce":902309444}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.96:0","nonce":3782790019}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.96:0","nonce":3050864675}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.96:0","nonce":1652886381}]}]} 2024-09-13T14:27:10.048 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T14:27:10.048 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T14:27:10.048 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:27:10.292 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:11.728 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:11 smithi096 ceph-mon[29229]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:27:11.728 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:11 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:11.729 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:11 smithi096 ceph-mon[29229]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T14:27:11.729 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:11 smithi096 ceph-mon[39645]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:27:11.729 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:11 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:11.729 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:11 smithi096 ceph-mon[39645]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T14:27:11.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:11 smithi204 ceph-mon[32546]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:27:11.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:11 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:11.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:11 smithi204 ceph-mon[32546]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T14:27:12.072 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:27:12.072 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":59,"fsid":"51e50ed6-71db-11ef-bceb-c7b262605968","created":"2024-09-13T14:21:16.367777+0000","modified":"2024-09-13T14:27:11.241780+0000","last_up_change":"2024-09-13T14:26:18.016817+0000","last_in_change":"2024-09-13T14:26:00.677688+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:24:20.514144+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:26:54.629267+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6802","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6803","nonce":766102065}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6804","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6805","nonce":766102065}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6808","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6809","nonce":766102065}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6806","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6807","nonce":766102065}]},"public_addr":"172.21.15.96:6803/766102065","cluster_addr":"172.21.15.96:6805/766102065","heartbeat_back_addr":"172.21.15.96:6809/766102065","heartbeat_front_addr":"172.21.15.96:6807/766102065","state":["exists","up"]},{"osd":1,"uuid":"1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6810","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6811","nonce":401053665}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6812","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6813","nonce":401053665}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6816","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6817","nonce":401053665}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6814","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6815","nonce":401053665}]},"public_addr":"172.21.15.96:6811/401053665","cluster_addr":"172.21.15.96:6813/401053665","heartbeat_back_addr":"172.21.15.96:6817/401053665","heartbeat_front_addr":"172.21.15.96:6815/401053665","state":["exists","up"]},{"osd":2,"uuid":"6a92a5b3-bdc7-499a-8b0f-2b0936fd9598","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6818","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6819","nonce":2386475672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6820","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6821","nonce":2386475672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6824","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6825","nonce":2386475672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6822","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6823","nonce":2386475672}]},"public_addr":"172.21.15.96:6819/2386475672","cluster_addr":"172.21.15.96:6821/2386475672","heartbeat_back_addr":"172.21.15.96:6825/2386475672","heartbeat_front_addr":"172.21.15.96:6823/2386475672","state":["exists","up"]},{"osd":3,"uuid":"1230646c-934e-4eb4-b4b9-16be52ffba66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6826","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6827","nonce":282100645}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6828","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6829","nonce":282100645}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6832","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6833","nonce":282100645}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6830","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6831","nonce":282100645}]},"public_addr":"172.21.15.96:6827/282100645","cluster_addr":"172.21.15.96:6829/282100645","heartbeat_back_addr":"172.21.15.96:6833/282100645","heartbeat_front_addr":"172.21.15.96:6831/282100645","state":["exists","up"]},{"osd":4,"uuid":"14115269-3d41-4ebf-8613-71ed59ec0f29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6800","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6801","nonce":1188704632}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6802","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6803","nonce":1188704632}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6806","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6807","nonce":1188704632}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6804","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6805","nonce":1188704632}]},"public_addr":"172.21.15.204:6801/1188704632","cluster_addr":"172.21.15.204:6803/1188704632","heartbeat_back_addr":"172.21.15.204:6807/1188704632","heartbeat_front_addr":"172.21.15.204:6805/1188704632","state":["exists","up"]},{"osd":5,"uuid":"15148a0c-df44-419e-b0f0-2b1600c207f4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6808","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6809","nonce":580729752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6810","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6811","nonce":580729752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6814","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6815","nonce":580729752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6812","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6813","nonce":580729752}]},"public_addr":"172.21.15.204:6809/580729752","cluster_addr":"172.21.15.204:6811/580729752","heartbeat_back_addr":"172.21.15.204:6815/580729752","heartbeat_front_addr":"172.21.15.204:6813/580729752","state":["exists","up"]},{"osd":6,"uuid":"8404e088-3523-4c21-b7c2-0a31b22e0279","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6816","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6817","nonce":118365497}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6818","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6819","nonce":118365497}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6822","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6823","nonce":118365497}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6820","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6821","nonce":118365497}]},"public_addr":"172.21.15.204:6817/118365497","cluster_addr":"172.21.15.204:6819/118365497","heartbeat_back_addr":"172.21.15.204:6823/118365497","heartbeat_front_addr":"172.21.15.204:6821/118365497","state":["exists","up"]},{"osd":7,"uuid":"93b23a35-0059-4e03-a2ba-28fee6431892","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6824","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6825","nonce":3529639424}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6826","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6827","nonce":3529639424}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6830","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6831","nonce":3529639424}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6828","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6829","nonce":3529639424}]},"public_addr":"172.21.15.204:6825/3529639424","cluster_addr":"172.21.15.204:6827/3529639424","heartbeat_back_addr":"172.21.15.204:6831/3529639424","heartbeat_front_addr":"172.21.15.204:6829/3529639424","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:28.841233+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:53.021149+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:17.120825+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:41.339674+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:02.571454+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:26.608523+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:51.119331+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:26:16.050687+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.96:0/1790922234":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/260172418":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/2292899363":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/960238088":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/378263291":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/376655558":"2024-09-14T14:21:45.592521+0000","172.21.15.96:6800/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6800/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:6801/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/1644412166":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6801/2749644659":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/19119516":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/752011643":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/1494034928":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6801/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6800/2749644659":"2024-09-14T14:26:50.514276+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:27:12.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:27:12 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:27:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:27:12.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:12 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/625466081' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:27:13.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:12 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/625466081' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:27:13.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:12 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/625466081' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:27:13.138 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T14:27:13.138 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:27:13.386 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:13.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:13 smithi096 ceph-mon[39645]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:13.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:13 smithi096 ceph-mon[29229]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:13.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:13 smithi204 ceph-mon[32546]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:15.218 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:27:15.274 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":59,"fsid":"51e50ed6-71db-11ef-bceb-c7b262605968","created":"2024-09-13T14:21:16.367777+0000","modified":"2024-09-13T14:27:11.241780+0000","last_up_change":"2024-09-13T14:26:18.016817+0000","last_in_change":"2024-09-13T14:26:00.677688+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:24:20.514144+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:26:54.629267+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6802","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6803","nonce":766102065}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6804","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6805","nonce":766102065}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6808","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6809","nonce":766102065}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6806","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6807","nonce":766102065}]},"public_addr":"172.21.15.96:6803/766102065","cluster_addr":"172.21.15.96:6805/766102065","heartbeat_back_addr":"172.21.15.96:6809/766102065","heartbeat_front_addr":"172.21.15.96:6807/766102065","state":["exists","up"]},{"osd":1,"uuid":"1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6810","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6811","nonce":401053665}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6812","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6813","nonce":401053665}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6816","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6817","nonce":401053665}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6814","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6815","nonce":401053665}]},"public_addr":"172.21.15.96:6811/401053665","cluster_addr":"172.21.15.96:6813/401053665","heartbeat_back_addr":"172.21.15.96:6817/401053665","heartbeat_front_addr":"172.21.15.96:6815/401053665","state":["exists","up"]},{"osd":2,"uuid":"6a92a5b3-bdc7-499a-8b0f-2b0936fd9598","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6818","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6819","nonce":2386475672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6820","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6821","nonce":2386475672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6824","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6825","nonce":2386475672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6822","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6823","nonce":2386475672}]},"public_addr":"172.21.15.96:6819/2386475672","cluster_addr":"172.21.15.96:6821/2386475672","heartbeat_back_addr":"172.21.15.96:6825/2386475672","heartbeat_front_addr":"172.21.15.96:6823/2386475672","state":["exists","up"]},{"osd":3,"uuid":"1230646c-934e-4eb4-b4b9-16be52ffba66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6826","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6827","nonce":282100645}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6828","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6829","nonce":282100645}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6832","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6833","nonce":282100645}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6830","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6831","nonce":282100645}]},"public_addr":"172.21.15.96:6827/282100645","cluster_addr":"172.21.15.96:6829/282100645","heartbeat_back_addr":"172.21.15.96:6833/282100645","heartbeat_front_addr":"172.21.15.96:6831/282100645","state":["exists","up"]},{"osd":4,"uuid":"14115269-3d41-4ebf-8613-71ed59ec0f29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6800","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6801","nonce":1188704632}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6802","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6803","nonce":1188704632}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6806","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6807","nonce":1188704632}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6804","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6805","nonce":1188704632}]},"public_addr":"172.21.15.204:6801/1188704632","cluster_addr":"172.21.15.204:6803/1188704632","heartbeat_back_addr":"172.21.15.204:6807/1188704632","heartbeat_front_addr":"172.21.15.204:6805/1188704632","state":["exists","up"]},{"osd":5,"uuid":"15148a0c-df44-419e-b0f0-2b1600c207f4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6808","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6809","nonce":580729752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6810","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6811","nonce":580729752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6814","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6815","nonce":580729752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6812","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6813","nonce":580729752}]},"public_addr":"172.21.15.204:6809/580729752","cluster_addr":"172.21.15.204:6811/580729752","heartbeat_back_addr":"172.21.15.204:6815/580729752","heartbeat_front_addr":"172.21.15.204:6813/580729752","state":["exists","up"]},{"osd":6,"uuid":"8404e088-3523-4c21-b7c2-0a31b22e0279","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6816","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6817","nonce":118365497}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6818","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6819","nonce":118365497}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6822","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6823","nonce":118365497}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6820","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6821","nonce":118365497}]},"public_addr":"172.21.15.204:6817/118365497","cluster_addr":"172.21.15.204:6819/118365497","heartbeat_back_addr":"172.21.15.204:6823/118365497","heartbeat_front_addr":"172.21.15.204:6821/118365497","state":["exists","up"]},{"osd":7,"uuid":"93b23a35-0059-4e03-a2ba-28fee6431892","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6824","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6825","nonce":3529639424}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6826","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6827","nonce":3529639424}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6830","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6831","nonce":3529639424}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6828","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6829","nonce":3529639424}]},"public_addr":"172.21.15.204:6825/3529639424","cluster_addr":"172.21.15.204:6827/3529639424","heartbeat_back_addr":"172.21.15.204:6831/3529639424","heartbeat_front_addr":"172.21.15.204:6829/3529639424","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:28.841233+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:53.021149+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:17.120825+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:41.339674+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:02.571454+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:26.608523+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:51.119331+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:26:16.050687+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.96:0/1790922234":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/260172418":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/2292899363":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/960238088":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/378263291":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/376655558":"2024-09-14T14:21:45.592521+0000","172.21.15.96:6800/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6800/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:6801/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/1644412166":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6801/2749644659":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/19119516":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/752011643":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/1494034928":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6801/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6800/2749644659":"2024-09-14T14:26:50.514276+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:27:15.786 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:15 smithi096 ceph-mon[29229]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:15.787 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:15 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1644859275' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:27:15.787 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:15 smithi096 ceph-mon[39645]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:15.787 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:15 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1644859275' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:27:15.812 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T14:27:15.812 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T14:27:15.813 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T14:27:15.813 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T14:27:15.813 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T14:27:15.814 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T14:27:15.814 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T14:27:15.815 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T14:27:15.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:15 smithi204 ceph-mon[32546]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:15.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:15 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1644859275' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:27:16.263 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:16.263 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:16.644 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:27:16 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[68212]: ts=2024-09-13T14:27:16.389Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003121746s 2024-09-13T14:27:17.622 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:17 smithi096 ceph-mon[29229]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:17.622 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:17 smithi096 ceph-mon[39645]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:17.635 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:17.636 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:17.636 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:17.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:17 smithi204 ceph-mon[32546]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:18.968 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 systemd[1]: Started Ceph grafana.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:27:19.165 INFO:teuthology.orchestra.run.smithi096.stdout:111669149730 2024-09-13T14:27:19.166 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T14:27:19.221 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=settings t=2024-09-13T14:27:18.970716543Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-13T14:27:19.221 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=settings t=2024-09-13T14:27:18.970896526Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-13T14:27:19.221 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=settings t=2024-09-13T14:27:18.970916444Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T14:27:19.222 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=settings t=2024-09-13T14:27:18.970924099Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T14:27:19.222 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=settings t=2024-09-13T14:27:18.970938792Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T14:27:19.222 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=settings t=2024-09-13T14:27:18.970945702Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T14:27:19.222 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=settings t=2024-09-13T14:27:18.970952074Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T14:27:19.222 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=settings t=2024-09-13T14:27:18.97095854Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T14:27:19.222 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=settings t=2024-09-13T14:27:18.970965338Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T14:27:19.222 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=settings t=2024-09-13T14:27:18.970971669Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T14:27:19.222 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=settings t=2024-09-13T14:27:18.97097808Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T14:27:19.222 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=settings t=2024-09-13T14:27:18.970984439Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T14:27:19.222 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=settings t=2024-09-13T14:27:18.970990583Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T14:27:19.222 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=settings t=2024-09-13T14:27:18.970996952Z level=info msg="App mode production" 2024-09-13T14:27:19.222 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=sqlstore t=2024-09-13T14:27:18.971047807Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T14:27:19.222 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=sqlstore t=2024-09-13T14:27:18.971071668Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T14:27:19.223 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.971970812Z level=info msg="Starting DB migrations" 2024-09-13T14:27:19.223 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.972046057Z level=info msg="Executing migration" id="create migration_log table" 2024-09-13T14:27:19.223 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.972717008Z level=info msg="Executing migration" id="create user table" 2024-09-13T14:27:19.224 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.973170155Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-13T14:27:19.224 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.974013498Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-13T14:27:19.224 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.977005311Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-13T14:27:19.224 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.977662368Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-13T14:27:19.224 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.980315598Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-13T14:27:19.224 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.982976754Z level=info msg="Executing migration" id="create user table v2" 2024-09-13T14:27:19.224 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.985951958Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-13T14:27:19.224 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.986349922Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-13T14:27:19.224 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.989071819Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-13T14:27:19.225 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.991663459Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-13T14:27:19.225 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.994204793Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-13T14:27:19.225 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.997167325Z level=info msg="Executing migration" id="Update user table charset" 2024-09-13T14:27:19.225 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:18 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:18.999826087Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-13T14:27:19.225 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.002372433Z level=info msg="Executing migration" id="Add missing user data" 2024-09-13T14:27:19.226 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.005062387Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-13T14:27:19.226 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.005501571Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-13T14:27:19.226 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.008167129Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-13T14:27:19.226 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.008648212Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-13T14:27:19.226 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.01104169Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-13T14:27:19.226 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.013722996Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-13T14:27:19.226 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.014203679Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-13T14:27:19.226 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.016975447Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-13T14:27:19.226 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.019369727Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-13T14:27:19.226 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.021998402Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-13T14:27:19.227 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.024605179Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-13T14:27:19.227 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.027219528Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-13T14:27:19.227 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.029788868Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-13T14:27:19.227 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.032250469Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-13T14:27:19.227 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.034729476Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-13T14:27:19.227 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.037383453Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-13T14:27:19.227 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.040097967Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-13T14:27:19.227 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.042607334Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-13T14:27:19.227 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.04517551Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-13T14:27:19.227 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.047826404Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-13T14:27:19.227 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.050485166Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-13T14:27:19.227 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.053164586Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-13T14:27:19.227 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.055654784Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-13T14:27:19.228 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.056020935Z level=info msg="Executing migration" id="create star table" 2024-09-13T14:27:19.228 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.056397271Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-13T14:27:19.228 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.056809641Z level=info msg="Executing migration" id="create org table v1" 2024-09-13T14:27:19.228 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.057213834Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-13T14:27:19.228 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.057618717Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-13T14:27:19.228 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.057995984Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-13T14:27:19.228 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.058401414Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-13T14:27:19.228 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.05880681Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-13T14:27:19.228 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.059203354Z level=info msg="Executing migration" id="Update org table charset" 2024-09-13T14:27:19.228 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.059499439Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-13T14:27:19.228 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.059770351Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-13T14:27:19.228 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.060096247Z level=info msg="Executing migration" id="create dashboard table" 2024-09-13T14:27:19.228 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.060464268Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-13T14:27:19.229 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.060854934Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-13T14:27:19.229 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.061268481Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-13T14:27:19.229 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.061679173Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-13T14:27:19.229 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.062158365Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-13T14:27:19.229 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.06298427Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-13T14:27:19.229 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.063839003Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-13T14:27:19.229 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.064623354Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-13T14:27:19.229 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.065055918Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-13T14:27:19.229 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.06584083Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-13T14:27:19.229 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.066622767Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-13T14:27:19.229 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.067032807Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-13T14:27:19.229 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.06734341Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-13T14:27:19.229 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.067840217Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-13T14:27:19.230 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.0683559Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-13T14:27:19.230 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.068856307Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-13T14:27:19.230 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.069636312Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-13T14:27:19.230 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.070165076Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-13T14:27:19.230 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.07098355Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-13T14:27:19.230 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.071889279Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-13T14:27:19.230 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.072358422Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-13T14:27:19.231 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.072629425Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-13T14:27:19.231 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.073170775Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-13T14:27:19.231 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.074012761Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-13T14:27:19.231 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.074933245Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-13T14:27:19.231 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.075708434Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-13T14:27:19.231 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.076013476Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-13T14:27:19.231 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.076443461Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-13T14:27:19.231 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.076855335Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-13T14:27:19.231 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.077171424Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-13T14:27:19.231 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.077699879Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-13T14:27:19.231 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.078580847Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-13T14:27:19.231 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.079513431Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-13T14:27:19.231 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.080388309Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-13T14:27:19.231 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.08298909Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-13T14:27:19.232 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.086005856Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-13T14:27:19.232 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.08637362Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-13T14:27:19.232 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.089214267Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-13T14:27:19.232 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.089777412Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-13T14:27:19.232 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.092121122Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-13T14:27:19.232 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.094821066Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-13T14:27:19.232 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.095157529Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-13T14:27:19.232 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.097917557Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-13T14:27:19.232 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.100619183Z level=info msg="Executing migration" id="create data_source table" 2024-09-13T14:27:19.232 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.103450929Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-13T14:27:19.232 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.104050243Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-13T14:27:19.232 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.104469748Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-13T14:27:19.233 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.107071345Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-13T14:27:19.233 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.107509554Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-13T14:27:19.233 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.109812619Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-13T14:27:19.233 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.110295802Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-13T14:27:19.233 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.112890607Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-13T14:27:19.233 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.115505638Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-13T14:27:19.233 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.118230991Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-13T14:27:19.233 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.118826781Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-13T14:27:19.233 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.121599377Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-13T14:27:19.233 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.124030734Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-13T14:27:19.233 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.126454485Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-13T14:27:19.233 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.127008735Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-13T14:27:19.233 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.129721941Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-13T14:27:19.233 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.132478432Z level=info msg="Executing migration" id="Add uid column" 2024-09-13T14:27:19.234 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.134991306Z level=info msg="Executing migration" id="Update uid value" 2024-09-13T14:27:19.234 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.135331589Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-13T14:27:19.234 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.138076743Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-13T14:27:19.234 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.138611271Z level=info msg="Executing migration" id="create api_key table" 2024-09-13T14:27:19.234 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.140647286Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-13T14:27:19.234 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.14315449Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-13T14:27:19.234 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.14361909Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-13T14:27:19.234 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.146312122Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-13T14:27:19.234 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.148742992Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-13T14:27:19.234 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.149550307Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-13T14:27:19.234 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.150453139Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-13T14:27:19.234 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.151691049Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-13T14:27:19.234 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.152490756Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-13T14:27:19.235 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.153426487Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-13T14:27:19.235 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.154120914Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-13T14:27:19.235 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.155073522Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-13T14:27:19.235 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.155689786Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-13T14:27:19.235 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.156068049Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-13T14:27:19.235 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.156910927Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-13T14:27:19.235 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.157709061Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-13T14:27:19.235 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.158509785Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-13T14:27:19.235 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.159345538Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-13T14:27:19.235 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.16011562Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-13T14:27:19.235 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.160908386Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-13T14:27:19.235 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.161806251Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-13T14:27:19.236 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.162532563Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-13T14:27:19.236 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.163245173Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-13T14:27:19.236 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.164083354Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-13T14:27:19.236 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.165125965Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-13T14:27:19.236 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.16575862Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-13T14:27:19.236 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.166609281Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-13T14:27:19.237 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.167428545Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-13T14:27:19.237 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.168370919Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-13T14:27:19.238 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.169071168Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-13T14:27:19.238 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.170068291Z level=info msg="Executing migration" id="create quota table v1" 2024-09-13T14:27:19.238 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.170832765Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-13T14:27:19.238 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.171802928Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-13T14:27:19.238 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.172146837Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-13T14:27:19.238 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.172534924Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-13T14:27:19.238 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.173365218Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-13T14:27:19.238 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.174306899Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-13T14:27:19.239 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.174740376Z level=info msg="Executing migration" id="create session table" 2024-09-13T14:27:19.239 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.175185764Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-13T14:27:19.239 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.175988578Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-13T14:27:19.239 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.176330717Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-13T14:27:19.239 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.17673044Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-13T14:27:19.239 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.177671175Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-13T14:27:19.240 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.17828769Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-13T14:27:19.240 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.178591904Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-13T14:27:19.240 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.17886612Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-13T14:27:19.240 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.179172517Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-13T14:27:19.240 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.179573183Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-13T14:27:19.240 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.179845068Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-13T14:27:19.241 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.180461209Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-13T14:27:19.241 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.183010639Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-13T14:27:19.241 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.183644219Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-13T14:27:19.241 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.186569596Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-13T14:27:19.241 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.189152603Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-13T14:27:19.241 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.189818401Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-13T14:27:19.241 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.192610921Z level=info msg="Executing migration" id="create alert table v1" 2024-09-13T14:27:19.241 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.195533264Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-13T14:27:19.241 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.198103113Z level=info msg="Executing migration" id="add index alert state" 2024-09-13T14:27:19.241 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.20087612Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-13T14:27:19.241 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.201354048Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-13T14:27:19.241 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.204115029Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-13T14:27:19.241 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.206783748Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-13T14:27:19.242 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.209460559Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-13T14:27:19.242 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.211025839Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-13T14:27:19.242 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.211804855Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-13T14:27:19.242 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.212596397Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-13T14:27:19.242 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.213373821Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-13T14:27:19.242 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.214149583Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-13T14:27:19.242 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.214973846Z level=info msg="Executing migration" id="Add column is_default" 2024-09-13T14:27:19.242 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.215907249Z level=info msg="Executing migration" id="Add column frequency" 2024-09-13T14:27:19.242 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.21669342Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-13T14:27:19.242 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.217525529Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-13T14:27:19.242 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.218386894Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-13T14:27:19.242 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.219228272Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-13T14:27:19.242 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.219893441Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-13T14:27:19.243 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.220799916Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-13T14:27:19.243 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.221468712Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-13T14:27:19.243 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.22225661Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-13T14:27:19.243 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.223039236Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-13T14:27:19.243 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.223864025Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-13T14:27:19.243 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.224663658Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-13T14:27:19.243 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.225476707Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-13T14:27:19.472 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.22630846Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-13T14:27:19.472 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.227089399Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-13T14:27:19.473 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.227926267Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-13T14:27:19.473 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.228726594Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-13T14:27:19.473 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.229545677Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-13T14:27:19.473 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.230331055Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-13T14:27:19.473 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.231160452Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-13T14:27:19.473 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.231980815Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-13T14:27:19.474 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.232814546Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-13T14:27:19.474 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.233642272Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-13T14:27:19.474 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.234486394Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-13T14:27:19.474 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.235407856Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-13T14:27:19.474 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.236124121Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-13T14:27:19.474 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.236975496Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-13T14:27:19.474 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.237785008Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-13T14:27:19.475 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.238642777Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-13T14:27:19.475 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.241322409Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-13T14:27:19.475 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.244243186Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-13T14:27:19.475 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.246944823Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-13T14:27:19.478 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.249421328Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-13T14:27:19.478 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.252330194Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-13T14:27:19.478 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.254990715Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-13T14:27:19.479 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.257644204Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-13T14:27:19.479 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.260369298Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-13T14:27:19.479 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.262871962Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-13T14:27:19.479 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.2655119Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-13T14:27:19.479 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.26823315Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-13T14:27:19.480 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.271025094Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-13T14:27:19.480 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.273665586Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-13T14:27:19.480 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.276281626Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-13T14:27:19.480 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.278827968Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-13T14:27:19.480 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.281529529Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-13T14:27:19.480 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.284302138Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-13T14:27:19.480 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.287018359Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-13T14:27:19.481 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.289686408Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-13T14:27:19.481 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.292244813Z level=info msg="Executing migration" id="Move region to single row" 2024-09-13T14:27:19.481 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.294861887Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-13T14:27:19.481 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.297445394Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-13T14:27:19.481 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.300088826Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-13T14:27:19.481 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.302652417Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-13T14:27:19.481 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.305249144Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-13T14:27:19.481 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.307945776Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-13T14:27:19.481 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.310693425Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-13T14:27:19.481 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.313200683Z level=info msg="Executing migration" id="create test_data table" 2024-09-13T14:27:19.481 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.315857126Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-13T14:27:19.481 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.318418283Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-13T14:27:19.481 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.321048681Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-13T14:27:19.482 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.323679222Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-13T14:27:19.482 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.326451551Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-13T14:27:19.482 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.329153449Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-13T14:27:19.482 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.331730172Z level=info msg="Executing migration" id="create team table" 2024-09-13T14:27:19.482 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.332365382Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-13T14:27:19.482 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.333177847Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-13T14:27:19.482 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.334007091Z level=info msg="Executing migration" id="create team member table" 2024-09-13T14:27:19.482 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.334791749Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-13T14:27:19.482 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.335596812Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-13T14:27:19.482 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.336395518Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-13T14:27:19.482 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.337298784Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-13T14:27:19.482 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.338264517Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-13T14:27:19.482 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.339044843Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-13T14:27:19.482 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.339840685Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-13T14:27:19.483 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.340639608Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-13T14:27:19.483 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.341475465Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-13T14:27:19.483 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.342288183Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-13T14:27:19.483 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.343019749Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-13T14:27:19.483 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.34381048Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-13T14:27:19.483 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.344688294Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-13T14:27:19.483 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.345392808Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-13T14:27:19.483 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.34617416Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-13T14:27:19.483 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.346625139Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-13T14:27:19.483 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.347406793Z level=info msg="Executing migration" id="create tag table" 2024-09-13T14:27:19.483 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.34782698Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-13T14:27:19.483 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.348268407Z level=info msg="Executing migration" id="create login attempt table" 2024-09-13T14:27:19.483 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.349080302Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-13T14:27:19.484 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.349519264Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-13T14:27:19.484 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.350312777Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-13T14:27:19.484 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.352218459Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-13T14:27:19.484 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.352616859Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-13T14:27:19.484 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.353467533Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-13T14:27:19.484 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.353801794Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-13T14:27:19.484 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.354604842Z level=info msg="Executing migration" id="create user auth table" 2024-09-13T14:27:19.484 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.355410194Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-13T14:27:19.485 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.356174439Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-13T14:27:19.485 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.35651684Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-13T14:27:19.485 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.357383043Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-13T14:27:19.485 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.358193916Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-13T14:27:19.485 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.360996682Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-13T14:27:19.485 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.361826834Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-13T14:27:19.485 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.364650628Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-13T14:27:19.485 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.367135887Z level=info msg="Executing migration" id="create server_lock table" 2024-09-13T14:27:19.485 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.370188763Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-13T14:27:19.485 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.370753742Z level=info msg="Executing migration" id="create user auth token table" 2024-09-13T14:27:19.485 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.373650737Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-13T14:27:19.485 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.376107025Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-13T14:27:19.485 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.378868497Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-13T14:27:19.486 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.37938413Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-13T14:27:19.486 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.382086516Z level=info msg="Executing migration" id="create cache_data table" 2024-09-13T14:27:19.486 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.384779962Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-13T14:27:19.486 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.387395277Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-13T14:27:19.486 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.387840731Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-13T14:27:19.486 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.390892164Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-13T14:27:19.486 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.393483874Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-13T14:27:19.486 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.396039923Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-13T14:27:19.486 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.398731028Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-13T14:27:19.486 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.401404271Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-13T14:27:19.486 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.404063503Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-13T14:27:19.486 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.406413189Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-13T14:27:19.486 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.406873361Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-13T14:27:19.487 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.407287519Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-13T14:27:19.487 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.40772052Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-13T14:27:19.487 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.408146678Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-13T14:27:19.487 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.409007294Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-13T14:27:19.487 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.40961824Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-13T14:27:19.487 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.409888415Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-13T14:27:19.488 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.410298067Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-13T14:27:19.488 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.410688227Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-13T14:27:19.488 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.411113289Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-13T14:27:19.488 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.411409066Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-13T14:27:19.488 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.411834461Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-13T14:27:19.488 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.412284902Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-13T14:27:19.488 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.412702345Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-13T14:27:19.488 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.413125406Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-13T14:27:19.489 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.413943258Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-13T14:27:19.489 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.414355169Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-13T14:27:19.489 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.414728168Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-13T14:27:19.489 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.418176723Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-13T14:27:19.489 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.421568745Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-13T14:27:19.489 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.422003768Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-13T14:27:19.489 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.422414413Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-13T14:27:19.489 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.423218714Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-13T14:27:19.489 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.423626867Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-13T14:27:19.489 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.424046794Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-13T14:27:19.490 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.424467706Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-13T14:27:19.490 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.424895326Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-13T14:27:19.490 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.425212847Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-13T14:27:19.490 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.425983036Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-13T14:27:19.490 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.426767438Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-13T14:27:19.490 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.427613694Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-13T14:27:19.490 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.428029772Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-13T14:27:19.490 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.428449984Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-13T14:27:19.490 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.429242733Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-13T14:27:19.490 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.430026293Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-13T14:27:19.490 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.43045221Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-13T14:27:19.491 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.431257543Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-13T14:27:19.491 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.432048783Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-13T14:27:19.491 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.432363033Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-13T14:27:19.491 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.432801934Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-13T14:27:19.491 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.433233372Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-13T14:27:19.491 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.433660896Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-13T14:27:19.492 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.435562344Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-13T14:27:19.492 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.436702304Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-13T14:27:19.492 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.438408227Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-13T14:27:19.492 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.440832835Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-13T14:27:19.492 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.441689703Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-13T14:27:19.492 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.444441839Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-13T14:27:19.493 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.447081492Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-13T14:27:19.493 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.449616516Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-13T14:27:19.493 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.452235962Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-13T14:27:19.493 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.454844418Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-13T14:27:19.493 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.457346031Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-13T14:27:19.493 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.458215Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-13T14:27:19.494 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.459248592Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-13T14:27:19.494 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.460201794Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-13T14:27:19.494 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.460905732Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-13T14:27:19.494 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.461777314Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-13T14:27:19.494 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.462177883Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-13T14:27:19.494 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.463064916Z level=info msg="Executing migration" id="create alert_image table" 2024-09-13T14:27:19.495 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.463886271Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-13T14:27:19.495 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.464670742Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-13T14:27:19.495 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.464954542Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-13T14:27:19.495 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.465352442Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-13T14:27:19.495 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.465749992Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T14:27:19.496 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.465822635Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T14:27:19.496 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.466135072Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-13T14:27:19.496 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.466544386Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-13T14:27:19.496 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.467335612Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-13T14:27:19.496 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.467472776Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-13T14:27:19.496 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.467591832Z level=info msg="alerts found to migrate" alerts=0 2024-09-13T14:27:19.497 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.468032447Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-13T14:27:19.497 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.468803628Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-13T14:27:19.497 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.469714745Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-13T14:27:19.497 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.470377006Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-13T14:27:19.497 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.471472228Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-13T14:27:19.497 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.472070345Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-13T14:27:19.498 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.472909469Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-13T14:27:19.498 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.473727927Z level=info msg="Executing migration" id="create data_keys table" 2024-09-13T14:27:19.498 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.474517834Z level=info msg="Executing migration" id="create secrets table" 2024-09-13T14:27:19.498 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.474962891Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-13T14:27:19.847 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:19.847 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:19.847 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:19.969 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.479453932Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-13T14:27:19.969 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.4804171Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-13T14:27:19.969 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.481230491Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-13T14:27:19.970 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.48549342Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-13T14:27:19.970 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.489709531Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-13T14:27:19.970 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.490559174Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-13T14:27:19.970 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.491434572Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-13T14:27:19.970 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.49177582Z level=info msg="Executing migration" id="create permission table" 2024-09-13T14:27:19.970 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.492639098Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-13T14:27:19.970 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.493428192Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-13T14:27:19.970 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.494302695Z level=info msg="Executing migration" id="create role table" 2024-09-13T14:27:19.970 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.494721245Z level=info msg="Executing migration" id="add column display_name" 2024-09-13T14:27:19.970 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.497570527Z level=info msg="Executing migration" id="add column group_name" 2024-09-13T14:27:19.970 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.500169269Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-13T14:27:19.970 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.502993953Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-13T14:27:19.970 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.503508358Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-13T14:27:19.971 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.506277094Z level=info msg="Executing migration" id="create team role table" 2024-09-13T14:27:19.971 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.509190935Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-13T14:27:19.971 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.511876741Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-13T14:27:19.971 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.512652817Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-13T14:27:19.971 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.515503654Z level=info msg="Executing migration" id="create user role table" 2024-09-13T14:27:19.972 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.51831301Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-13T14:27:19.972 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.520914976Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-13T14:27:19.972 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.521414067Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-13T14:27:19.972 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.524187201Z level=info msg="Executing migration" id="create builtin role table" 2024-09-13T14:27:19.972 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.526624096Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-13T14:27:19.972 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.529327734Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-13T14:27:19.972 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.531989116Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-13T14:27:19.972 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.53443516Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-13T14:27:19.972 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.53497748Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-13T14:27:19.972 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.535529301Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-13T14:27:19.972 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.535950146Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-13T14:27:19.972 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.537728373Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-13T14:27:19.973 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.540382924Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-13T14:27:19.973 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.54094142Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-13T14:27:19.973 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.541922659Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-13T14:27:19.973 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.542387993Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-13T14:27:19.973 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.542869479Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-13T14:27:19.973 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.543176237Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-13T14:27:19.973 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.543575501Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-13T14:27:19.973 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.544017594Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-13T14:27:19.973 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.544403188Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-13T14:27:19.973 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.544695232Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-13T14:27:19.973 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.545068009Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-13T14:27:19.973 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.545465975Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-13T14:27:19.973 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.545908396Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-13T14:27:19.974 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.546865339Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-13T14:27:19.974 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.547184404Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-13T14:27:19.974 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.547604898Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-13T14:27:19.974 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.548041677Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-13T14:27:19.974 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.548497836Z level=info msg="Executing migration" id="add correlation config column" 2024-09-13T14:27:19.974 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.549552159Z level=info msg="Executing migration" id="create entity_events table" 2024-09-13T14:27:19.974 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.550010656Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-13T14:27:19.974 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.55275545Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T14:27:19.974 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.55302539Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T14:27:19.974 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.553389341Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T14:27:19.974 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.553494298Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T14:27:19.974 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.553790318Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-13T14:27:19.974 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.554198663Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-13T14:27:19.974 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.55481244Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-13T14:27:19.975 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.555262158Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T14:27:19.975 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.555694385Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-13T14:27:19.975 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.556112113Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T14:27:19.975 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.556527661Z level=info msg="Executing migration" id="Drop public config table" 2024-09-13T14:27:19.975 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.556881817Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-13T14:27:19.975 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.557329093Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-13T14:27:19.975 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.557728046Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T14:27:19.975 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.558186119Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-13T14:27:19.975 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.56057985Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-13T14:27:19.975 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.563623907Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-13T14:27:19.975 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.566165982Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-13T14:27:19.975 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.567163821Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-13T14:27:19.975 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.569827787Z level=info msg="Executing migration" id="add share column" 2024-09-13T14:27:19.975 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.572503122Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-13T14:27:19.976 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.575069767Z level=info msg="Executing migration" id="create file table" 2024-09-13T14:27:19.976 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.575619171Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-13T14:27:19.976 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.578169951Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-13T14:27:19.976 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.580908613Z level=info msg="Executing migration" id="create file_meta table" 2024-09-13T14:27:19.976 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.581666243Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-13T14:27:19.976 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.582144019Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-13T14:27:19.976 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.58307638Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-13T14:27:19.977 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.583656505Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-13T14:27:19.977 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.584386738Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-13T14:27:19.977 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.584857126Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-13T14:27:19.977 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.585882342Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-13T14:27:19.977 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.586766198Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-13T14:27:19.977 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.58747221Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-13T14:27:19.977 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.588354251Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-13T14:27:19.977 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.588884573Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-13T14:27:19.977 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.589699949Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-13T14:27:19.977 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.590757129Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-13T14:27:19.977 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.59178955Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-13T14:27:19.977 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.592230111Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-13T14:27:19.977 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.601735512Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-13T14:27:19.978 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.602548652Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-13T14:27:19.978 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.603358699Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-13T14:27:19.978 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.606890076Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-13T14:27:19.978 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.607644665Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-13T14:27:19.978 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.608514893Z level=info msg="Executing migration" id="create folder table" 2024-09-13T14:27:19.978 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.609136127Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-13T14:27:19.978 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.609579005Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-13T14:27:19.978 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.610019005Z level=info msg="Executing migration" id="Update folder title length" 2024-09-13T14:27:19.978 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.610896647Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-13T14:27:19.978 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=migrator t=2024-09-13T14:27:19.611606026Z level=info msg="migrations completed" performed=485 skipped=0 duration=639.578652ms 2024-09-13T14:27:19.978 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=sqlstore t=2024-09-13T14:27:19.612006901Z level=info msg="Created default organization" 2024-09-13T14:27:19.978 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=plugin.loader t=2024-09-13T14:27:19.669136702Z level=info msg="Plugin registered" pluginID=input 2024-09-13T14:27:19.978 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=plugin.loader t=2024-09-13T14:27:19.69675392Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-13T14:27:19.979 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=plugin.loader t=2024-09-13T14:27:19.696775548Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-13T14:27:19.979 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=secrets t=2024-09-13T14:27:19.696809785Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T14:27:19.979 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=query_data t=2024-09-13T14:27:19.698498598Z level=info msg="Query Service initialization" 2024-09-13T14:27:19.979 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=live.push_http t=2024-09-13T14:27:19.70078577Z level=info msg="Live Push Gateway initialization" 2024-09-13T14:27:20.207 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:19 smithi096 ceph-mon[29229]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:20.208 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:19 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:20.208 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:19 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:20.208 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:19 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:20.208 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:19 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:20.208 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:19 smithi096 ceph-mon[29229]: Deploying daemon node-exporter.a on smithi096 2024-09-13T14:27:20.208 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:19 smithi096 ceph-mon[39645]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:20.208 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:19 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:20.208 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:19 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:20.209 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:19 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:20.209 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:19 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:20.209 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:19 smithi096 ceph-mon[39645]: Deploying daemon node-exporter.a on smithi096 2024-09-13T14:27:20.437 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-mon[32546]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:20.438 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:20.438 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:20.438 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:20.438 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:20.438 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:19 smithi204 ceph-mon[32546]: Deploying daemon node-exporter.a on smithi096 2024-09-13T14:27:20.473 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:20.718 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=infra.usagestats.collector t=2024-09-13T14:27:20.437447367Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T14:27:20.718 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=provisioning.datasources t=2024-09-13T14:27:20.437833159Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-13T14:27:20.719 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=provisioning.datasources t=2024-09-13T14:27:20.470242008Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T14:27:20.719 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=provisioning.datasources t=2024-09-13T14:27:20.502351956Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-13T14:27:20.719 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=provisioning.alerting t=2024-09-13T14:27:20.523734282Z level=info msg="starting to provision alerting" 2024-09-13T14:27:20.719 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=provisioning.alerting t=2024-09-13T14:27:20.523755392Z level=info msg="finished to provision alerting" 2024-09-13T14:27:20.719 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=grafanaStorageLogger t=2024-09-13T14:27:20.52389086Z level=info msg="storage starting" 2024-09-13T14:27:20.719 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=ngalert.state.manager t=2024-09-13T14:27:20.523903055Z level=info msg="Warming state cache for startup" 2024-09-13T14:27:20.719 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=http.server t=2024-09-13T14:27:20.529252256Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T14:27:20.719 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=ngalert.state.manager t=2024-09-13T14:27:20.534304408Z level=info msg="State cache has been initialized" states=0 duration=10.399321ms 2024-09-13T14:27:20.719 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=ticker t=2024-09-13T14:27:20.534371859Z level=info msg=starting first_tick=2024-09-13T14:27:30Z 2024-09-13T14:27:20.719 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=ngalert.multiorg.alertmanager t=2024-09-13T14:27:20.534364391Z level=info msg="starting MultiOrg Alertmanager" 2024-09-13T14:27:20.719 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=sqlstore.transactions t=2024-09-13T14:27:20.535099896Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:27:20.719 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[61466]: logger=sqlstore.transactions t=2024-09-13T14:27:20.56902636Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:27:21.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:20 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:21.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:20 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:21.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:20 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:21.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:20 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:21.436 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:21.436 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:20 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:22.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:21 smithi096 ceph-mon[29229]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:22.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:21 smithi096 ceph-mon[39645]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:22.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:21 smithi204 ceph-mon[32546]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:22.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:27:22 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:27:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:27:24.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:23 smithi096 ceph-mon[29229]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:24.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:23 smithi096 ceph-mon[39645]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:24.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:23 smithi204 ceph-mon[32546]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:26.226 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:25 smithi096 ceph-mon[29229]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:26.227 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:25 smithi096 ceph-mon[39645]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:26.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:25 smithi204 ceph-mon[32546]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:26.982 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:26 smithi096 podman[69770]: 2024-09-13 14:27:26.881306011 +0000 UTC m=+3.931890151 container create 4e13850242054193620c4422811e60f54fdcb07b990ec8218c0e671132db0de8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:27:28.001 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 podman[69770]: 2024-09-13 14:27:27.668607555 +0000 UTC m=+4.719191650 container init 4e13850242054193620c4422811e60f54fdcb07b990ec8218c0e671132db0de8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:27:28.001 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 podman[69770]: 2024-09-13 14:27:27.671479644 +0000 UTC m=+4.722063746 container start 4e13850242054193620c4422811e60f54fdcb07b990ec8218c0e671132db0de8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:27:28.001 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.676Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T14:27:28.001 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.676Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T14:27:28.001 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.676Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:27:28.001 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.676Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:27:28.002 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.676Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:27:28.002 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.676Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:27:28.002 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:27:28.002 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:27:28.002 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:27:28.002 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:27:28.002 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:27:28.003 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:27:28.003 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:27:28.003 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:27:28.003 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:27:28.004 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:27:28.004 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:27:28.004 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:27:28.004 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:27:28.004 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:27:28.004 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:27:28.004 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:27:28.004 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:27:28.005 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:27:28.005 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:27:28.005 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:27:28.005 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:27:28.005 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:27:28.005 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:27:28.005 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:27:28.006 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:27:28.006 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:27:28.006 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:27:28.006 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:27:28.006 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:27:28.006 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:27:28.006 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:27:28.007 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:27:28.007 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:27:28.007 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:27:28.007 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:27:28.007 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:27:28.007 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:27:28.008 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:27:28.008 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:27:28.008 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:27:28.008 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:27:28.008 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:27:28.008 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:27:28.009 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:27:28.009 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:27:28.009 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:27:28.009 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[69971]: ts=2024-09-13T14:27:27.677Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:27:28.009 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 bash[69770]: 4e13850242054193620c4422811e60f54fdcb07b990ec8218c0e671132db0de8 2024-09-13T14:27:28.010 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:27:27 smithi096 systemd[1]: Started Ceph node-exporter.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:27:28.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:28 smithi096 ceph-mon[39645]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:28.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:28 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:28.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:28 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:28.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:28 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:28.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:28 smithi096 ceph-mon[29229]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:28.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:28 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:28.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:28 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:28.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:28 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:28.366 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:28 smithi204 ceph-mon[32546]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:28.366 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:28 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:28.366 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:28 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:28.366 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:28 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:28.719 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:28 smithi204 systemd[1]: Starting Ceph node-exporter.b for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:27:29.009 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:28 smithi204 bash[61728]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-13T14:27:29.080 INFO:teuthology.orchestra.run.smithi096.stdout:158913789978 2024-09-13T14:27:29.080 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T14:27:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:29 smithi096 ceph-mon[29229]: Deploying daemon node-exporter.b on smithi204 2024-09-13T14:27:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:29 smithi096 ceph-mon[39645]: Deploying daemon node-exporter.b on smithi204 2024-09-13T14:27:29.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:29 smithi204 ceph-mon[32546]: Deploying daemon node-exporter.b on smithi204 2024-09-13T14:27:29.469 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:29 smithi204 bash[61728]: Getting image source signatures 2024-09-13T14:27:29.469 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:29 smithi204 bash[61728]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-13T14:27:29.469 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:29 smithi204 bash[61728]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-13T14:27:29.469 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:29 smithi204 bash[61728]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-13T14:27:30.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:30 smithi096 ceph-mon[39645]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:30.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:30 smithi096 ceph-mon[29229]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:30.136 INFO:teuthology.orchestra.run.smithi096.stdout:133143986207 2024-09-13T14:27:30.136 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T14:27:30.450 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:30 smithi204 ceph-mon[32546]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:30.718 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:30 smithi204 bash[61728]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-13T14:27:30.718 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:30 smithi204 bash[61728]: Writing manifest to image destination 2024-09-13T14:27:31.157 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:30 smithi204 podman[61728]: 2024-09-13 14:27:30.839809182 +0000 UTC m=+2.194073283 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-13T14:27:31.157 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:30 smithi204 podman[61728]: 2024-09-13 14:27:30.955134014 +0000 UTC m=+2.309398152 container create 0fc850b0a1c6c26327d1e7050af9dbf25e0cec7faf3507ac7331b24cc6e22cda (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:27:31.469 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 podman[61728]: 2024-09-13 14:27:31.157268028 +0000 UTC m=+2.511532120 container init 0fc850b0a1c6c26327d1e7050af9dbf25e0cec7faf3507ac7331b24cc6e22cda (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:27:31.469 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 podman[61728]: 2024-09-13 14:27:31.160517698 +0000 UTC m=+2.514781796 container start 0fc850b0a1c6c26327d1e7050af9dbf25e0cec7faf3507ac7331b24cc6e22cda (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:27:31.469 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.165Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T14:27:31.469 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.165Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T14:27:31.469 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.165Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:27:31.469 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.165Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:27:31.469 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:27:31.469 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:27:31.469 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:27:31.469 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:27:31.469 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:27:31.470 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:27:31.470 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:27:31.470 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:27:31.470 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:27:31.470 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:27:31.470 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:27:31.470 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:27:31.471 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:27:31.471 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:27:31.471 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:27:31.471 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:27:31.471 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:27:31.471 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:27:31.471 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:27:31.471 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:27:31.471 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:27:31.471 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:27:31.471 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:27:31.471 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:27:31.471 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:27:31.471 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:27:31.472 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:27:31.472 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:27:31.472 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:27:31.472 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:27:31.472 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:27:31.472 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:27:31.472 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:27:31.472 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:27:31.472 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:27:31.473 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:27:31.473 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:27:31.473 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:27:31.473 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:27:31.473 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:27:31.473 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:27:31.473 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:27:31.473 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:27:31.473 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:27:31.473 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:27:31.473 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:27:31.473 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.166Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:27:31.473 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.167Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:27:31.474 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[61805]: ts=2024-09-13T14:27:31.167Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:27:31.474 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 bash[61728]: 0fc850b0a1c6c26327d1e7050af9dbf25e0cec7faf3507ac7331b24cc6e22cda 2024-09-13T14:27:31.474 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:27:31 smithi204 systemd[1]: Started Ceph node-exporter.b for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:27:32.119 INFO:teuthology.orchestra.run.smithi096.stdout:111669149732 2024-09-13T14:27:32.130 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:32 smithi096 ceph-mon[29229]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:32.130 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:32 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:32.130 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:32 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:32.130 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:32 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:32.130 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:32 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:32.130 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:32 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:32.130 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:32 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:32.131 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:32 smithi096 ceph-mon[39645]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:32.131 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:32 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:32.131 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:32 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:32.131 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:32 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:32.131 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:32 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:32.131 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:32 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:32.131 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:32 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:32.304 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:32 smithi204 ceph-mon[32546]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:32.304 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:32 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:32.304 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:32 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:32.304 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:32 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:32.304 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:32 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:32.304 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:32 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:32.304 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:32 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:32.305 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:27:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:27:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:27:33.290 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:33 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1365011703' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:27:33.290 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:33 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1365011703' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:27:33.301 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:33 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1365011703' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:27:33.316 INFO:teuthology.orchestra.run.smithi096.stdout:34359738418 2024-09-13T14:27:33.316 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T14:27:34.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:34 smithi204 ceph-mon[32546]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:34.242 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:34 smithi096 ceph-mon[39645]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:34.243 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:34 smithi096 ceph-mon[29229]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:34.626 INFO:teuthology.orchestra.run.smithi096.stdout:184683593750 2024-09-13T14:27:34.626 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T14:27:34.928 INFO:teuthology.orchestra.run.smithi096.stdout:55834574894 2024-09-13T14:27:34.928 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T14:27:36.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:36 smithi096 ceph-mon[39645]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:36.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:36 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:36.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:36 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:36.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:36 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:36.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:36 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:36 smithi096 ceph-mon[29229]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:36 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:36 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:36 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:36 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:36.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:36 smithi204 ceph-mon[32546]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:36 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:36 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:36 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:36 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:36.771 INFO:teuthology.orchestra.run.smithi096.stdout:210453397521 2024-09-13T14:27:36.772 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T14:27:37.171 INFO:teuthology.orchestra.run.smithi096.stdout:77309411370 2024-09-13T14:27:37.171 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T14:27:37.748 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:37.748 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:37.749 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:37.749 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:37.786 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149730 got 111669149732 for osd.3 2024-09-13T14:27:37.787 DEBUG:teuthology.parallel:result is None 2024-09-13T14:27:37.805 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:37.816 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:38.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:38 smithi096 ceph-mon[29229]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:38.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:38 smithi096 ceph-mon[39645]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:38.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:38 smithi204 ceph-mon[32546]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:39.391 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:39 smithi096 ceph-mon[29229]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:39.392 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:39 smithi096 ceph-mon[39645]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:39.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:39 smithi204 ceph-mon[32546]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:41.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:41 smithi096 ceph-mon[29229]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:41.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:41 smithi096 ceph-mon[39645]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:41.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:41 smithi204 ceph-mon[32546]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:42.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:27:42 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:27:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:27:43.765 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:43 smithi096 ceph-mon[29229]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:43.765 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:43 smithi096 ceph-mon[39645]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:43.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:43 smithi204 ceph-mon[32546]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:44.556 INFO:teuthology.orchestra.run.smithi096.stdout:55834574897 2024-09-13T14:27:44.756 INFO:teuthology.orchestra.run.smithi096.stdout:158913789981 2024-09-13T14:27:44.756 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:44 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1188405191' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:27:44.756 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:44 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1188405191' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:27:44.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:44 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1188405191' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:27:45.382 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913789978 got 158913789981 for osd.5 2024-09-13T14:27:45.382 DEBUG:teuthology.parallel:result is None 2024-09-13T14:27:45.794 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:45 smithi096 ceph-mon[29229]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:45.794 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:45 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/600148796' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:27:45.794 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:45 smithi096 ceph-mon[39645]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:45.794 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:45 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/600148796' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:27:45.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:45 smithi204 ceph-mon[32546]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:45.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:45 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/600148796' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:27:46.158 INFO:teuthology.orchestra.run.smithi096.stdout:184683593752 2024-09-13T14:27:46.927 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:46 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/19900824' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:27:46.927 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:46 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/19900824' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:27:46.951 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593750 got 184683593752 for osd.6 2024-09-13T14:27:46.952 DEBUG:teuthology.parallel:result is None 2024-09-13T14:27:46.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:46 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/19900824' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:27:47.715 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574894 got 55834574897 for osd.1 2024-09-13T14:27:47.715 DEBUG:teuthology.parallel:result is None 2024-09-13T14:27:47.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:47 smithi204 ceph-mon[32546]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:48.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:47 smithi096 ceph-mon[29229]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:48.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:47 smithi096 ceph-mon[39645]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:48.673 INFO:teuthology.orchestra.run.smithi096.stdout:210453397523 2024-09-13T14:27:48.673 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:48 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2581023051' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:27:48.674 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:48 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2581023051' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:27:48.870 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:48 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2581023051' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:27:49.402 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397521 got 210453397523 for osd.7 2024-09-13T14:27:49.402 DEBUG:teuthology.parallel:result is None 2024-09-13T14:27:49.669 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:49 smithi096 ceph-mon[29229]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:49.669 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:49 smithi096 ceph-mon[39645]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:49.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:49 smithi204 ceph-mon[32546]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:50.069 INFO:teuthology.orchestra.run.smithi096.stdout:34359738422 2024-09-13T14:27:50.264 INFO:teuthology.orchestra.run.smithi096.stdout:133143986211 2024-09-13T14:27:50.836 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:50 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2684819369' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:27:50.836 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:50 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3696396364' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:27:50.837 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:50 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:50.837 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:50 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2684819369' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:27:50.837 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:50 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3696396364' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:27:50.837 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:50 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:50.860 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986207 got 133143986211 for osd.4 2024-09-13T14:27:50.860 DEBUG:teuthology.parallel:result is None 2024-09-13T14:27:50.886 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:50.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:50 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2684819369' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:27:50.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:50 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3696396364' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:27:50.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:50 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:51.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:51 smithi096 ceph-mon[29229]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:51.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:51 smithi096 ceph-mon[39645]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:51.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:51 smithi204 ceph-mon[32546]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:52.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:27:52 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:27:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:27:52.661 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738418 got 34359738422 for osd.0 2024-09-13T14:27:52.661 DEBUG:teuthology.parallel:result is None 2024-09-13T14:27:53.818 INFO:teuthology.orchestra.run.smithi096.stdout:77309411374 2024-09-13T14:27:53.882 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:53 smithi096 ceph-mon[29229]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:53.883 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:53 smithi096 ceph-mon[39645]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:53.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:53 smithi204 ceph-mon[32546]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:54.577 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411370 got 77309411374 for osd.2 2024-09-13T14:27:54.578 DEBUG:teuthology.parallel:result is None 2024-09-13T14:27:54.578 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:27:54.578 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:27:54.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:54 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2155603183' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:27:54.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:54 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2155603183' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:27:54.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:54 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2155603183' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:27:55.015 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:56.063 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:55 smithi096 ceph-mon[39645]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:56.063 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:55 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:56.063 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:55 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:56.063 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:55 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:56.063 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:55 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:56.063 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:55 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:56.064 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:55 smithi096 ceph-mon[29229]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:56.064 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:55 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:56.064 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:55 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:56.064 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:55 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:56.064 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:55 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:56.064 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:55 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:56.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:55 smithi204 ceph-mon[32546]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:56.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:55 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:55 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:55 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:55 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:55 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:27:57.113 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:27:57.114 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:27:57.123 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:56 smithi096 ceph-mon[29229]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:27:57.123 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:56 smithi096 ceph-mon[29229]: Reconfiguring daemon alertmanager.a on smithi096 2024-09-13T14:27:57.124 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:56 smithi096 ceph-mon[39645]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:27:57.124 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:56 smithi096 ceph-mon[39645]: Reconfiguring daemon alertmanager.a on smithi096 2024-09-13T14:27:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:56 smithi204 ceph-mon[32546]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:27:57.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:56 smithi204 ceph-mon[32546]: Reconfiguring daemon alertmanager.a on smithi096 2024-09-13T14:27:58.216 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:57 smithi096 ceph-mon[29229]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:58.216 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:57 smithi096 ceph-mon[29229]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:27:58.216 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:57 smithi096 ceph-mon[39645]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:58.217 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:57 smithi096 ceph-mon[39645]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:27:58.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:57 smithi204 ceph-mon[32546]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:58.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:57 smithi204 ceph-mon[32546]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:27:59.572 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:27:59 smithi096 systemd[1]: Stopping Ceph alertmanager.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:28:00.127 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":43,"stamp":"2024-09-13T14:27:56.542640+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":115,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82,"ondisk_log_size":82,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385160,"kb_used_data":6312,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576056,"statfs":{"total":767960285184,"available":765517881344,"internally_reserved":0,"allocated":6463488,"data_stored":3738818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002362"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.372809+0000","last_change":"2024-09-13T14:27:01.132344+0000","last_active":"2024-09-13T14:27:11.372809+0000","last_peered":"2024-09-13T14:27:11.372809+0000","last_clean":"2024-09-13T14:27:11.372809+0000","last_became_active":"2024-09-13T14:26:56.113467+0000","last_became_peered":"2024-09-13T14:26:56.113467+0000","last_unstale":"2024-09-13T14:27:11.372809+0000","last_undegraded":"2024-09-13T14:27:11.372809+0000","last_fullsized":"2024-09-13T14:27:11.372809+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:57:56.604631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023893800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.245402+0000","last_change":"2024-09-13T14:27:01.127728+0000","last_active":"2024-09-13T14:27:11.245402+0000","last_peered":"2024-09-13T14:27:11.245402+0000","last_clean":"2024-09-13T14:27:11.245402+0000","last_became_active":"2024-09-13T14:26:56.124603+0000","last_became_peered":"2024-09-13T14:26:56.124603+0000","last_unstale":"2024-09-13T14:27:11.245402+0000","last_undegraded":"2024-09-13T14:27:11.245402+0000","last_fullsized":"2024-09-13T14:27:11.245402+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:46:33.019632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000373223,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.372886+0000","last_change":"2024-09-13T14:27:01.908723+0000","last_active":"2024-09-13T14:27:11.372886+0000","last_peered":"2024-09-13T14:27:11.372886+0000","last_clean":"2024-09-13T14:27:11.372886+0000","last_became_active":"2024-09-13T14:26:56.124401+0000","last_became_peered":"2024-09-13T14:26:56.124401+0000","last_unstale":"2024-09-13T14:27:11.372886+0000","last_undegraded":"2024-09-13T14:27:11.372886+0000","last_fullsized":"2024-09-13T14:27:11.372886+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:46.437020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027405700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.245382+0000","last_change":"2024-09-13T14:27:01.127754+0000","last_active":"2024-09-13T14:27:11.245382+0000","last_peered":"2024-09-13T14:27:11.245382+0000","last_clean":"2024-09-13T14:27:11.245382+0000","last_became_active":"2024-09-13T14:26:56.112967+0000","last_became_peered":"2024-09-13T14:26:56.112967+0000","last_unstale":"2024-09-13T14:27:11.245382+0000","last_undegraded":"2024-09-13T14:27:11.245382+0000","last_fullsized":"2024-09-13T14:27:11.245382+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:40:20.095122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000367501,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":21,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.373063+0000","last_change":"2024-09-13T14:27:01.131429+0000","last_active":"2024-09-13T14:27:11.373063+0000","last_peered":"2024-09-13T14:27:11.373063+0000","last_clean":"2024-09-13T14:27:11.373063+0000","last_became_active":"2024-09-13T14:26:56.114214+0000","last_became_peered":"2024-09-13T14:26:56.114214+0000","last_unstale":"2024-09-13T14:27:11.373063+0000","last_undegraded":"2024-09-13T14:27:11.373063+0000","last_fullsized":"2024-09-13T14:27:11.373063+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:43.625300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00095382599999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.244501+0000","last_change":"2024-09-13T14:27:01.127122+0000","last_active":"2024-09-13T14:27:11.244501+0000","last_peered":"2024-09-13T14:27:11.244501+0000","last_clean":"2024-09-13T14:27:11.244501+0000","last_became_active":"2024-09-13T14:26:56.178609+0000","last_became_peered":"2024-09-13T14:26:56.178609+0000","last_unstale":"2024-09-13T14:27:11.244501+0000","last_undegraded":"2024-09-13T14:27:11.244501+0000","last_fullsized":"2024-09-13T14:27:11.244501+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:23:16.906453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036252799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.372985+0000","last_change":"2024-09-13T14:27:01.908723+0000","last_active":"2024-09-13T14:27:11.372985+0000","last_peered":"2024-09-13T14:27:11.372985+0000","last_clean":"2024-09-13T14:27:11.372985+0000","last_became_active":"2024-09-13T14:26:56.115228+0000","last_became_peered":"2024-09-13T14:26:56.115228+0000","last_unstale":"2024-09-13T14:27:11.372985+0000","last_undegraded":"2024-09-13T14:27:11.372985+0000","last_fullsized":"2024-09-13T14:27:11.372985+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:48.775507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027767100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.372818+0000","last_change":"2024-09-13T14:27:01.130820+0000","last_active":"2024-09-13T14:27:11.372818+0000","last_peered":"2024-09-13T14:27:11.372818+0000","last_clean":"2024-09-13T14:27:11.372818+0000","last_became_active":"2024-09-13T14:26:56.113569+0000","last_became_peered":"2024-09-13T14:26:56.113569+0000","last_unstale":"2024-09-13T14:27:11.372818+0000","last_undegraded":"2024-09-13T14:27:11.372818+0000","last_fullsized":"2024-09-13T14:27:11.372818+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:47.705568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034224499999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"52'79","reported_seq":49,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:27:01.908406+0000","last_change":"2024-09-13T14:26:20.041272+0000","last_active":"2024-09-13T14:27:01.908406+0000","last_peered":"2024-09-13T14:27:01.908406+0000","last_clean":"2024-09-13T14:27:01.908406+0000","last_became_active":"2024-09-13T14:26:20.035983+0000","last_became_peered":"2024-09-13T14:26:20.035983+0000","last_unstale":"2024-09-13T14:27:01.908406+0000","last_undegraded":"2024-09-13T14:27:01.908406+0000","last_fullsized":"2024-09-13T14:27:01.908406+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:24:20.875473+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:32:28.938482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397525,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.55700000000000005}]},{"osd":1,"last update":"Fri Sep 13 14:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.621}]},{"osd":2,"last update":"Fri Sep 13 14:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.0880000000000001,"5min":1.0880000000000001,"15min":1.0880000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.51400000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.437},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.58499999999999996}]},{"osd":4,"last update":"Fri Sep 13 14:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.65400000000000003}]},{"osd":5,"last update":"Fri Sep 13 14:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.421,"5min":1.421,"15min":1.421},"last":0.66900000000000004}]},{"osd":6,"last update":"Fri Sep 13 14:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.69099999999999995}]}]},{"osd":6,"up_from":43,"seq":184683593754,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.496}]},{"osd":1,"last update":"Fri Sep 13 14:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.56200000000000006}]},{"osd":2,"last update":"Fri Sep 13 14:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.48299999999999998}]},{"osd":3,"last update":"Fri Sep 13 14:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.52400000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.46100000000000002}]},{"osd":5,"last update":"Fri Sep 13 14:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.435},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.57399999999999995}]},{"osd":7,"last update":"Fri Sep 13 14:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.59199999999999997}]}]},{"osd":5,"up_from":37,"seq":158913789983,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":245863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.52900000000000003,"15min":0.53700000000000003},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.65500000000000003,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51800000000000002,"15min":0.51900000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.68000000000000005,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.40500000000000003}]},{"osd":1,"last update":"Fri Sep 13 14:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.51700000000000002,"15min":0.51900000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.53200000000000003,"15min":0.53600000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.39200000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.52800000000000002,"15min":0.53200000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.441},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.52400000000000002,"15min":0.52600000000000002},"min":{"1min":0.39400000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.64400000000000002,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.372}]},{"osd":3,"last update":"Fri Sep 13 14:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.53000000000000003,"15min":0.53400000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71199999999999997,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.38},{"interface":"front","average":{"1min":0.5,"5min":0.52800000000000002,"15min":0.53200000000000003},"min":{"1min":0.35699999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.63700000000000001,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.42399999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.36699999999999999,"15min":0.35699999999999998},"min":{"1min":0.27700000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.373,"15min":0.36599999999999999},"min":{"1min":0.311,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.30299999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.433},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.28699999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:27:22 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.41399999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986212,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:27:11 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.55300000000000005,"15min":0.56299999999999994},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.68300000000000005,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.53700000000000003,"15min":0.53500000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.45400000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:27:11 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.52300000000000002,"15min":0.52700000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.67400000000000004,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.55600000000000005,"15min":0.56200000000000006},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.66600000000000004,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.63400000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:27:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.505},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.53700000000000003,"15min":0.54000000000000004},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.67600000000000005,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.68600000000000005}]},{"osd":3,"last update":"Fri Sep 13 14:27:11 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.53000000000000003,"15min":0.53200000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.65900000000000003,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.55100000000000005,"15min":0.55500000000000005},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.71199999999999997,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.62}]},{"osd":5,"last update":"Fri Sep 13 14:27:32 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.38300000000000001,"15min":0.38},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.56200000000000006,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.378,"15min":0.375},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.54400000000000004}]},{"osd":6,"last update":"Fri Sep 13 14:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.48299999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.60499999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149737,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297924,"kb_used_data":568,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447228,"statfs":{"total":95995035648,"available":95689961472,"internally_reserved":0,"allocated":581632,"data_stored":245844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.40300000000000002,"15min":0.41999999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.441,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.33900000000000002,"5min":0.378,"15min":0.39000000000000001},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.47199999999999998,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.31900000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.41299999999999998,"15min":0.42299999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.51900000000000002,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.39100000000000001,"15min":0.40100000000000002},"min":{"1min":0.27200000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.50700000000000001,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.24299999999999999}]},{"osd":2,"last update":"Fri Sep 13 14:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.436,"15min":0.45100000000000001},"min":{"1min":0.28399999999999997,"5min":0.248,"15min":0.248},"max":{"1min":0.57699999999999996,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.41199999999999998,"15min":0.42799999999999999},"min":{"1min":0.23999999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.53400000000000003,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.435}]},{"osd":4,"last update":"Fri Sep 13 14:27:07 2024","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.60299999999999998,"15min":0.57999999999999996},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.3979999999999997,"5min":4.3979999999999997,"15min":4.3979999999999997},"last":0.379},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.623,"15min":0.59399999999999997},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.4660000000000002,"5min":4.4660000000000002,"15min":4.4660000000000002},"last":0.51800000000000002}]},{"osd":5,"last update":"Fri Sep 13 14:27:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.70999999999999996,"15min":0.74099999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.71499999999999997,"5min":4.4139999999999997,"15min":4.4139999999999997},"last":0.37},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.70399999999999996,"15min":0.73199999999999998},"min":{"1min":0.35399999999999998,"5min":0.312,"15min":0.312},"max":{"1min":0.748,"5min":4.3799999999999999,"15min":4.3799999999999999},"last":0.53800000000000003}]},{"osd":6,"last update":"Fri Sep 13 14:26:57 2024","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.4329999999999998,"5min":4.4329999999999998,"15min":4.4329999999999998},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.40999999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:27:20 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.35399999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297924,"kb_used_data":568,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447228,"statfs":{"total":95995035648,"available":95689961472,"internally_reserved":0,"allocated":581632,"data_stored":245844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:27:20 2024","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":0.34100000000000003,"15min":0.33700000000000002},"min":{"1min":0.24099999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.53100000000000003,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.37,"15min":0.36399999999999999},"min":{"1min":0.251,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.32100000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:27:20 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.36699999999999999,"15min":0.36199999999999999},"min":{"1min":0.28199999999999997,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.39300000000000002,"15min":0.38900000000000001},"min":{"1min":0.28699999999999998,"5min":0.249,"15min":0.249},"max":{"1min":0.55900000000000005,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.28699999999999998}]},{"osd":3,"last update":"Fri Sep 13 14:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.40600000000000003,"15min":0.41299999999999998},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.503,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.38,"5min":0.42499999999999999,"15min":0.42999999999999999},"min":{"1min":0.29999999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.53700000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.36499999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:27:09 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.38200000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.52800000000000002,"15min":0.52400000000000002},"min":{"1min":0.45100000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.50900000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.53100000000000003,"15min":0.53800000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.75800000000000001,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.55100000000000005,"15min":0.55400000000000005},"min":{"1min":0.42299999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.74199999999999999,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.42099999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:26:58 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.499},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.47099999999999997}]},{"osd":7,"last update":"Fri Sep 13 14:27:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.437}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":589824,"data_stored":246107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:26:58 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.31900000000000001,"15min":0.29299999999999998},"min":{"1min":0.24299999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.34200000000000003,"15min":0.314},"min":{"1min":0.27500000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.32600000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:27:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.377,"15min":0.36299999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":1.0800000000000001,"5min":1.0800000000000001,"15min":1.0800000000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.36499999999999999,"15min":0.35099999999999998},"min":{"1min":0.22900000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.254}]},{"osd":3,"last update":"Fri Sep 13 14:27:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.433,"15min":0.441},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.41099999999999998,"15min":0.41199999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.313}]},{"osd":4,"last update":"Fri Sep 13 14:27:08 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.53900000000000003,"15min":0.53700000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.71699999999999997,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.71699999999999997}]},{"osd":5,"last update":"Fri Sep 13 14:27:32 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.57899999999999996,"15min":0.58199999999999996},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.71399999999999997,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.44800000000000001,"5min":0.433,"15min":0.433},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.66000000000000003}]},{"osd":6,"last update":"Fri Sep 13 14:26:54 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.79200000000000004}]},{"osd":7,"last update":"Fri Sep 13 14:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.747},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.58899999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:27:01 2024","interfaces":[{"interface":"back","average":{"1min":0.34100000000000003,"5min":0.32800000000000001,"15min":0.316},"min":{"1min":0.23799999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.499,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.38,"5min":0.33700000000000002,"15min":0.313},"min":{"1min":0.22500000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.54800000000000004,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.28899999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:27:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.36499999999999999,"15min":0.35499999999999998},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.35599999999999998,"15min":0.34799999999999998},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.54400000000000004,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.30399999999999999}]},{"osd":3,"last update":"Fri Sep 13 14:26:47 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.40999999999999998,"15min":0.40899999999999997},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40400000000000003,"15min":0.40500000000000003},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.32500000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:27:08 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.54400000000000004,"15min":0.54300000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55800000000000005,"15min":0.56000000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.73099999999999998,"5min":0.755,"15min":0.755},"last":0.63}]},{"osd":5,"last update":"Fri Sep 13 14:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.39400000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.71699999999999997,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.66500000000000004}]},{"osd":6,"last update":"Fri Sep 13 14:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.59899999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:27:18 2024","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":1.6539999999999999,"15min":1.6539999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":19.390000000000001,"5min":19.390000000000001,"15min":19.390000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.60799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:28:00.129 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:28:00.273 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:27:59 smithi096 ceph-mon[29229]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:00.273 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:27:59 smithi096 ceph-mon[39645]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:27:59 smithi204 ceph-mon[32546]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:00.572 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:00 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[68212]: ts=2024-09-13T14:28:00.272Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:28:00.572 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:00 smithi096 podman[72439]: 2024-09-13 14:28:00.302682894 +0000 UTC m=+0.123506915 container died 1a4f8415ae35087f5f5d4dc0ab3fe8cca210dbb7257700d377661ffcb6934902 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:28:01.014 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:00 smithi096 podman[72439]: 2024-09-13 14:28:00.781024936 +0000 UTC m=+0.601848950 container cleanup 1a4f8415ae35087f5f5d4dc0ab3fe8cca210dbb7257700d377661ffcb6934902 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:28:01.014 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:00 smithi096 bash[72439]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T14:28:01.211 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:01.322 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:01 smithi096 podman[72450]: 2024-09-13 14:28:01.013894915 +0000 UTC m=+0.714763675 container remove 1a4f8415ae35087f5f5d4dc0ab3fe8cca210dbb7257700d377661ffcb6934902 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:28:01.322 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:01 smithi096 podman[72450]: 2024-09-13 14:28:01.091045165 +0000 UTC m=+0.791913927 volume remove a952e346280a1f8afa2e29d46981108c60285e5eecad5ce12965b53ccc55b230 2024-09-13T14:28:01.669 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:01 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T14:28:01.670 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:01 smithi096 systemd[1]: Stopped Ceph alertmanager.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:28:01.670 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:01 smithi096 systemd[1]: Starting Ceph alertmanager.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:28:02.256 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:01 smithi096 ceph-mon[29229]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:02.257 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:01 smithi096 ceph-mon[39645]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:02.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:01 smithi204 ceph-mon[32546]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:02.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:28:02 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:28:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:28:02.916 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:28:02.917 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:28:03.190 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:02 smithi096 podman[72662]: 2024-09-13 14:28:02.811065385 +0000 UTC m=+0.016234267 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-13T14:28:03.190 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:02 smithi096 podman[72662]: 2024-09-13 14:28:02.929906712 +0000 UTC m=+0.135075589 volume create 6ecc162b67749b319951cbd4a559f2655f4af512d418429fd3a9870b59d17ab5 2024-09-13T14:28:03.492 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:03 smithi096 podman[72662]: 2024-09-13 14:28:03.189604579 +0000 UTC m=+0.394773447 container create c2fb084ac3b6e330733e37f7bd38b11aca67a7db23c80816f46203a383e6a51b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:28:03.492 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:03 smithi096 podman[72662]: 2024-09-13 14:28:03.492181101 +0000 UTC m=+0.697349978 container init c2fb084ac3b6e330733e37f7bd38b11aca67a7db23c80816f46203a383e6a51b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:28:03.822 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:03 smithi096 podman[72662]: 2024-09-13 14:28:03.495733939 +0000 UTC m=+0.700902807 container start c2fb084ac3b6e330733e37f7bd38b11aca67a7db23c80816f46203a383e6a51b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:28:03.822 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[72700]: ts=2024-09-13T14:28:03.507Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-13T14:28:03.822 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[72700]: ts=2024-09-13T14:28:03.507Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-13T14:28:03.823 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[72700]: ts=2024-09-13T14:28:03.508Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.96 port=9094 2024-09-13T14:28:03.823 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[72700]: ts=2024-09-13T14:28:03.510Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T14:28:03.823 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[72700]: ts=2024-09-13T14:28:03.537Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:28:03.823 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[72700]: ts=2024-09-13T14:28:03.538Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:28:03.823 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[72700]: ts=2024-09-13T14:28:03.540Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-13T14:28:03.823 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[72700]: ts=2024-09-13T14:28:03.540Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T14:28:03.823 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:03 smithi096 bash[72662]: c2fb084ac3b6e330733e37f7bd38b11aca67a7db23c80816f46203a383e6a51b 2024-09-13T14:28:03.823 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:03 smithi096 systemd[1]: Started Ceph alertmanager.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:28:04.054 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":46,"stamp":"2024-09-13T14:28:02.543902+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":115,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82,"ondisk_log_size":82,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385160,"kb_used_data":6312,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576056,"statfs":{"total":767960285184,"available":765517881344,"internally_reserved":0,"allocated":6463488,"data_stored":3738818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002634"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.372809+0000","last_change":"2024-09-13T14:27:01.132344+0000","last_active":"2024-09-13T14:27:11.372809+0000","last_peered":"2024-09-13T14:27:11.372809+0000","last_clean":"2024-09-13T14:27:11.372809+0000","last_became_active":"2024-09-13T14:26:56.113467+0000","last_became_peered":"2024-09-13T14:26:56.113467+0000","last_unstale":"2024-09-13T14:27:11.372809+0000","last_undegraded":"2024-09-13T14:27:11.372809+0000","last_fullsized":"2024-09-13T14:27:11.372809+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:57:56.604631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023893800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.245402+0000","last_change":"2024-09-13T14:27:01.127728+0000","last_active":"2024-09-13T14:27:11.245402+0000","last_peered":"2024-09-13T14:27:11.245402+0000","last_clean":"2024-09-13T14:27:11.245402+0000","last_became_active":"2024-09-13T14:26:56.124603+0000","last_became_peered":"2024-09-13T14:26:56.124603+0000","last_unstale":"2024-09-13T14:27:11.245402+0000","last_undegraded":"2024-09-13T14:27:11.245402+0000","last_fullsized":"2024-09-13T14:27:11.245402+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:46:33.019632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000373223,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.372886+0000","last_change":"2024-09-13T14:27:01.908723+0000","last_active":"2024-09-13T14:27:11.372886+0000","last_peered":"2024-09-13T14:27:11.372886+0000","last_clean":"2024-09-13T14:27:11.372886+0000","last_became_active":"2024-09-13T14:26:56.124401+0000","last_became_peered":"2024-09-13T14:26:56.124401+0000","last_unstale":"2024-09-13T14:27:11.372886+0000","last_undegraded":"2024-09-13T14:27:11.372886+0000","last_fullsized":"2024-09-13T14:27:11.372886+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:46.437020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027405700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.245382+0000","last_change":"2024-09-13T14:27:01.127754+0000","last_active":"2024-09-13T14:27:11.245382+0000","last_peered":"2024-09-13T14:27:11.245382+0000","last_clean":"2024-09-13T14:27:11.245382+0000","last_became_active":"2024-09-13T14:26:56.112967+0000","last_became_peered":"2024-09-13T14:26:56.112967+0000","last_unstale":"2024-09-13T14:27:11.245382+0000","last_undegraded":"2024-09-13T14:27:11.245382+0000","last_fullsized":"2024-09-13T14:27:11.245382+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:40:20.095122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000367501,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":21,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.373063+0000","last_change":"2024-09-13T14:27:01.131429+0000","last_active":"2024-09-13T14:27:11.373063+0000","last_peered":"2024-09-13T14:27:11.373063+0000","last_clean":"2024-09-13T14:27:11.373063+0000","last_became_active":"2024-09-13T14:26:56.114214+0000","last_became_peered":"2024-09-13T14:26:56.114214+0000","last_unstale":"2024-09-13T14:27:11.373063+0000","last_undegraded":"2024-09-13T14:27:11.373063+0000","last_fullsized":"2024-09-13T14:27:11.373063+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:43.625300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00095382599999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.244501+0000","last_change":"2024-09-13T14:27:01.127122+0000","last_active":"2024-09-13T14:27:11.244501+0000","last_peered":"2024-09-13T14:27:11.244501+0000","last_clean":"2024-09-13T14:27:11.244501+0000","last_became_active":"2024-09-13T14:26:56.178609+0000","last_became_peered":"2024-09-13T14:26:56.178609+0000","last_unstale":"2024-09-13T14:27:11.244501+0000","last_undegraded":"2024-09-13T14:27:11.244501+0000","last_fullsized":"2024-09-13T14:27:11.244501+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:23:16.906453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036252799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.372985+0000","last_change":"2024-09-13T14:27:01.908723+0000","last_active":"2024-09-13T14:27:11.372985+0000","last_peered":"2024-09-13T14:27:11.372985+0000","last_clean":"2024-09-13T14:27:11.372985+0000","last_became_active":"2024-09-13T14:26:56.115228+0000","last_became_peered":"2024-09-13T14:26:56.115228+0000","last_unstale":"2024-09-13T14:27:11.372985+0000","last_undegraded":"2024-09-13T14:27:11.372985+0000","last_fullsized":"2024-09-13T14:27:11.372985+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:48.775507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027767100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:27:11.372818+0000","last_change":"2024-09-13T14:27:01.130820+0000","last_active":"2024-09-13T14:27:11.372818+0000","last_peered":"2024-09-13T14:27:11.372818+0000","last_clean":"2024-09-13T14:27:11.372818+0000","last_became_active":"2024-09-13T14:26:56.113569+0000","last_became_peered":"2024-09-13T14:26:56.113569+0000","last_unstale":"2024-09-13T14:27:11.372818+0000","last_undegraded":"2024-09-13T14:27:11.372818+0000","last_fullsized":"2024-09-13T14:27:11.372818+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:47.705568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034224499999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"52'79","reported_seq":49,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:27:01.908406+0000","last_change":"2024-09-13T14:26:20.041272+0000","last_active":"2024-09-13T14:27:01.908406+0000","last_peered":"2024-09-13T14:27:01.908406+0000","last_clean":"2024-09-13T14:27:01.908406+0000","last_became_active":"2024-09-13T14:26:20.035983+0000","last_became_peered":"2024-09-13T14:26:20.035983+0000","last_unstale":"2024-09-13T14:27:01.908406+0000","last_undegraded":"2024-09-13T14:27:01.908406+0000","last_fullsized":"2024-09-13T14:27:01.908406+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:24:20.875473+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:32:28.938482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397527,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.52300000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.443},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.48499999999999999}]},{"osd":2,"last update":"Fri Sep 13 14:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.0880000000000001,"5min":1.0880000000000001,"15min":1.0880000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.42099999999999999}]},{"osd":3,"last update":"Fri Sep 13 14:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.498},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.46300000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.35999999999999999}]},{"osd":5,"last update":"Fri Sep 13 14:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.421,"5min":1.421,"15min":1.421},"last":0.47299999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.51600000000000001}]}]},{"osd":6,"up_from":43,"seq":184683593756,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:27:56 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.52000000000000002,"15min":0.51600000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.70299999999999996,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.375,"5min":0.69999999999999996,"15min":0.58799999999999997},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":17.888000000000002,"5min":17.888000000000002,"15min":17.888000000000002},"last":0.36499999999999999}]},{"osd":1,"last update":"Fri Sep 13 14:27:56 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.67000000000000004,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.52400000000000002,"15min":0.52500000000000002},"min":{"1min":0.38800000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.68600000000000005,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.51000000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:27:56 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.51200000000000001,"15min":0.51300000000000001},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.56299999999999994,"15min":0.56699999999999995},"min":{"1min":0.40300000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.67400000000000004,"5min":0.874,"15min":0.874},"last":0.39600000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:27:56 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.53100000000000003,"15min":0.52900000000000003},"min":{"1min":0.42199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.71799999999999997,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.3700000000000001,"5min":0.71399999999999997,"15min":0.60499999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":17.952999999999999,"5min":17.952999999999999,"15min":17.952999999999999},"last":0.41099999999999998}]},{"osd":4,"last update":"Fri Sep 13 14:27:56 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.45300000000000001,"15min":0.45500000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.44700000000000001,"15min":0.44600000000000001},"min":{"1min":0.32900000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.76000000000000001,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.56799999999999995}]},{"osd":5,"last update":"Fri Sep 13 14:27:56 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.44600000000000001,"15min":0.442},"min":{"1min":0.312,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.5,"15min":0.502},"min":{"1min":0.30599999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.71699999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.629}]},{"osd":7,"last update":"Fri Sep 13 14:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.61899999999999999}]}]},{"osd":5,"up_from":37,"seq":158913789985,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":245863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.52900000000000003,"15min":0.53700000000000003},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.65500000000000003,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51800000000000002,"15min":0.51900000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.68000000000000005,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.51300000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.51700000000000002,"15min":0.51900000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.504,"5min":0.53200000000000003,"15min":0.53600000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.39500000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.52800000000000002,"15min":0.53200000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.52400000000000002,"15min":0.52600000000000002},"min":{"1min":0.39400000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.64400000000000002,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.46600000000000003}]},{"osd":3,"last update":"Fri Sep 13 14:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.53000000000000003,"15min":0.53400000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71199999999999997,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.5,"5min":0.52800000000000002,"15min":0.53200000000000003},"min":{"1min":0.35699999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.63700000000000001,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.40699999999999997}]},{"osd":4,"last update":"Fri Sep 13 14:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.36699999999999999,"15min":0.35699999999999998},"min":{"1min":0.27700000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.373,"15min":0.36599999999999999},"min":{"1min":0.311,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.41799999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:27:55 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.43099999999999999,"15min":0.42299999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.438,"5min":0.40999999999999998,"15min":0.40600000000000003},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.501}]},{"osd":7,"last update":"Fri Sep 13 14:27:22 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.44},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.52200000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986214,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:27:11 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.55300000000000005,"15min":0.56299999999999994},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.68300000000000005,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.53700000000000003,"15min":0.53500000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.46899999999999997}]},{"osd":1,"last update":"Fri Sep 13 14:27:11 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.52300000000000002,"15min":0.52700000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.67400000000000004,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.55600000000000005,"15min":0.56200000000000006},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.66600000000000004,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.44700000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:27:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.504},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.53700000000000003,"15min":0.54000000000000004},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.67600000000000005,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.58499999999999996}]},{"osd":3,"last update":"Fri Sep 13 14:27:11 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.53000000000000003,"15min":0.53200000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.65900000000000003,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.55100000000000005,"15min":0.55500000000000005},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.71199999999999997,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.48699999999999999}]},{"osd":5,"last update":"Fri Sep 13 14:27:32 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.38300000000000001,"15min":0.38},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.56200000000000006,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.378,"15min":0.375},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.38500000000000001}]},{"osd":6,"last update":"Fri Sep 13 14:27:56 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.44900000000000001,"15min":0.44500000000000001},"min":{"1min":0.307,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.624},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.442,"15min":0.437},"min":{"1min":0.34499999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.61099999999999999}]},{"osd":7,"last update":"Fri Sep 13 14:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.51900000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149739,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297924,"kb_used_data":568,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447228,"statfs":{"total":95995035648,"available":95689961472,"internally_reserved":0,"allocated":581632,"data_stored":245844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.40300000000000002,"15min":0.41999999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.441,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.33900000000000002,"5min":0.378,"15min":0.39000000000000001},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.47199999999999998,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.376}]},{"osd":1,"last update":"Fri Sep 13 14:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.41299999999999998,"15min":0.42299999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.51900000000000002,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.39100000000000001,"15min":0.40100000000000002},"min":{"1min":0.27200000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.50700000000000001,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.43099999999999999}]},{"osd":2,"last update":"Fri Sep 13 14:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.436,"15min":0.45100000000000001},"min":{"1min":0.28399999999999997,"5min":0.248,"15min":0.248},"max":{"1min":0.57699999999999996,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.41199999999999998,"15min":0.42799999999999999},"min":{"1min":0.23999999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.53400000000000003,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.40000000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:27:07 2024","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.60299999999999998,"15min":0.57999999999999996},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.3979999999999997,"5min":4.3979999999999997,"15min":4.3979999999999997},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.623,"15min":0.59399999999999997},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.4660000000000002,"5min":4.4660000000000002,"15min":4.4660000000000002},"last":0.77800000000000002}]},{"osd":5,"last update":"Fri Sep 13 14:27:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.70999999999999996,"15min":0.74099999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.71499999999999997,"5min":4.4139999999999997,"15min":4.4139999999999997},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.70399999999999996,"15min":0.73199999999999998},"min":{"1min":0.35399999999999998,"5min":0.312,"15min":0.312},"max":{"1min":0.748,"5min":4.3799999999999999,"15min":4.3799999999999999},"last":0.71999999999999997}]},{"osd":6,"last update":"Fri Sep 13 14:27:58 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.74199999999999999,"15min":0.77000000000000002},"min":{"1min":0.36899999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.77700000000000002,"5min":4.4329999999999998,"15min":4.4329999999999998},"last":0.754},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.55000000000000004,"15min":0.54800000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.65300000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:27:20 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.69199999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411376,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297924,"kb_used_data":568,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447228,"statfs":{"total":95995035648,"available":95689961472,"internally_reserved":0,"allocated":581632,"data_stored":245844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:27:20 2024","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":0.34100000000000003,"15min":0.33700000000000002},"min":{"1min":0.24099999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.53100000000000003,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.436},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.37,"15min":0.36399999999999999},"min":{"1min":0.251,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.30199999999999999}]},{"osd":1,"last update":"Fri Sep 13 14:27:20 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.36699999999999999,"15min":0.36199999999999999},"min":{"1min":0.28199999999999997,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.39300000000000002,"15min":0.38900000000000001},"min":{"1min":0.28699999999999998,"5min":0.249,"15min":0.249},"max":{"1min":0.55900000000000005,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.33900000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.40600000000000003,"15min":0.41299999999999998},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.503,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.38,"5min":0.42499999999999999,"15min":0.42999999999999999},"min":{"1min":0.29999999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.53700000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.38600000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:27:09 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.38200000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.52800000000000002,"15min":0.52400000000000002},"min":{"1min":0.45100000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.67100000000000004}]},{"osd":5,"last update":"Fri Sep 13 14:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.53100000000000003,"15min":0.53800000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.75800000000000001,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.55100000000000005,"15min":0.55400000000000005},"min":{"1min":0.42299999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.74199999999999999,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.49199999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:27:59 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.57799999999999996,"15min":0.58299999999999996},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.56899999999999995,"15min":0.57699999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.70199999999999996,"5min":0.75,"15min":0.75},"last":0.58499999999999996}]},{"osd":7,"last update":"Fri Sep 13 14:27:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.51300000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574901,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":589824,"data_stored":246107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:27:58 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.33400000000000002,"15min":0.29799999999999999},"min":{"1min":0.248,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.47199999999999998,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.36399999999999999,"15min":0.32100000000000001},"min":{"1min":0.219,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.198,"5min":1.198,"15min":1.198},"last":0.441}]},{"osd":2,"last update":"Fri Sep 13 14:27:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.377,"15min":0.36299999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":1.0800000000000001,"5min":1.0800000000000001,"15min":1.0800000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.36499999999999999,"15min":0.35099999999999998},"min":{"1min":0.22900000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.35799999999999998}]},{"osd":3,"last update":"Fri Sep 13 14:27:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.433,"15min":0.441},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.41099999999999998,"15min":0.41199999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.42199999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:27:08 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.53900000000000003,"15min":0.53700000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.71699999999999997,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.59099999999999997}]},{"osd":5,"last update":"Fri Sep 13 14:27:32 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.57899999999999996,"15min":0.58199999999999996},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.71399999999999997,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.44800000000000001,"5min":0.433,"15min":0.433},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.59999999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:27:54 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.58699999999999997,"15min":0.58499999999999996},"min":{"1min":0.36799999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.83399999999999996,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.59599999999999997,"15min":0.59399999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80000000000000004,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.65200000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.66600000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738425,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:27:01 2024","interfaces":[{"interface":"back","average":{"1min":0.34100000000000003,"5min":0.32800000000000001,"15min":0.316},"min":{"1min":0.23799999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.499,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.38,"5min":0.33700000000000002,"15min":0.313},"min":{"1min":0.22500000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.54800000000000004,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.32300000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:27:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.36499999999999999,"15min":0.35499999999999998},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.35599999999999998,"15min":0.34799999999999998},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.54400000000000004,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.34899999999999998}]},{"osd":3,"last update":"Fri Sep 13 14:27:52 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.40400000000000003,"15min":0.40699999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.33100000000000002,"5min":0.38900000000000001,"15min":0.40000000000000002},"min":{"1min":0.26700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.45500000000000002,"5min":0.627,"15min":0.627},"last":0.52400000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:27:08 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.54400000000000004,"15min":0.54300000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55800000000000005,"15min":0.56000000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.73099999999999998,"5min":0.755,"15min":0.755},"last":0.71399999999999997}]},{"osd":5,"last update":"Fri Sep 13 14:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.39400000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.71699999999999997,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.65200000000000002}]},{"osd":6,"last update":"Fri Sep 13 14:27:56 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.54000000000000004,"15min":0.54600000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.72899999999999998,"5min":0.75,"15min":0.75},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.57099999999999995,"15min":0.57799999999999996},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.75600000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.74199999999999999}]},{"osd":7,"last update":"Fri Sep 13 14:27:18 2024","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":1.6539999999999999,"15min":1.6539999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":19.390000000000001,"5min":19.390000000000001,"15min":19.390000000000001},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.53500000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:28:04.056 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:28:04.056 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T14:28:04.056 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T14:28:04.056 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T14:28:04.192 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-mon[29229]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:04.193 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-mon[29229]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:28:04.193 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:04.193 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:04.193 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-mon[39645]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:04.193 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-mon[39645]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:28:04.193 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:04.193 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:03 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:04.301 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:04.399 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:03 smithi204 ceph-mon[32546]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:04.399 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:03 smithi204 ceph-mon[32546]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:28:04.399 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:03 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:04.399 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:03 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:05.253 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:04 smithi096 ceph-mon[29229]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:28:05.253 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:04 smithi096 ceph-mon[29229]: Reconfiguring daemon prometheus.a on smithi204 2024-09-13T14:28:05.253 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:04 smithi096 ceph-mon[39645]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:28:05.254 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:04 smithi096 ceph-mon[39645]: Reconfiguring daemon prometheus.a on smithi204 2024-09-13T14:28:05.319 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:04 smithi204 ceph-mon[32546]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:28:05.319 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:04 smithi204 ceph-mon[32546]: Reconfiguring daemon prometheus.a on smithi204 2024-09-13T14:28:05.818 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:05 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[72700]: ts=2024-09-13T14:28:05.510Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000570477s 2024-09-13T14:28:06.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:05 smithi096 ceph-mon[29229]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:06.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:05 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:06.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:05 smithi096 ceph-mon[39645]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:06.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:05 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:06.195 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:28:06.195 INFO:teuthology.orchestra.run.smithi096.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T14:28:06.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:05 smithi204 ceph-mon[32546]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:06.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:05 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:06.894 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T14:28:06.894 INFO:tasks.cephadm:Setup complete, yielding 2024-09-13T14:28:06.895 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T14:28:06.906 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi096.front.sepia.ceph.com 2024-09-13T14:28:06.906 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-09-13T14:28:06.968 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:06 smithi204 systemd[1]: Stopping Ceph prometheus.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:28:07.153 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:07.266 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:07 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3611620025' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:28:07.266 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:28:06.997Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:28:07.266 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:28:06.997Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:28:07.266 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:28:06.997Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:28:07.266 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:28:06.997Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:28:07.266 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:28:06.997Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:28:07.267 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:28:06.997Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:28:07.267 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:28:06.997Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T14:28:07.267 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:28:06.997Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T14:28:07.267 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:28:06.998Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T14:28:07.267 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:28:06.998Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:28:07.267 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:28:06.998Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T14:28:07.267 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[59449]: ts=2024-09-13T14:28:06.998Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T14:28:07.267 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:07 smithi204 podman[62560]: 2024-09-13 14:28:07.007610051 +0000 UTC m=+0.120238100 container died 9afae272a3c42ce71981b95df456be86761c98096d07c124a253d72d94d3956f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:28:07.273 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:07 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3611620025' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:28:07.274 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:07 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3611620025' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:28:07.674 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:07 smithi204 podman[62560]: 2024-09-13 14:28:07.372307529 +0000 UTC m=+0.484935620 container cleanup 9afae272a3c42ce71981b95df456be86761c98096d07c124a253d72d94d3956f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:28:07.674 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:07 smithi204 bash[62560]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:28:07.968 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:07 smithi204 podman[62573]: 2024-09-13 14:28:07.673727734 +0000 UTC m=+0.664811384 container remove 9afae272a3c42ce71981b95df456be86761c98096d07c124a253d72d94d3956f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:28:07.968 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:07 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:28:07.968 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:07 smithi204 systemd[1]: Stopped Ceph prometheus.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:28:07.969 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:07 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.020s CPU time. 2024-09-13T14:28:07.969 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:07 smithi204 systemd[1]: Starting Ceph prometheus.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:28:08.311 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:08 smithi096 ceph-mon[29229]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:08.311 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:08 smithi096 ceph-mon[39645]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:08.315 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-mon[32546]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:08.316 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 podman[62666]: 2024-09-13 14:28:07.955266141 +0000 UTC m=+0.017032255 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T14:28:08.316 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 podman[62666]: 2024-09-13 14:28:08.083215441 +0000 UTC m=+0.144981555 container create 420af5726f6f10b9e2ab86b8caf914b184ff25a8ae6e18c4ad7f1881243fefcb (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:28:08.572 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:08 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:08] ENGINE Bus STOPPING 2024-09-13T14:28:08.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 podman[62666]: 2024-09-13 14:28:08.315813289 +0000 UTC m=+0.377579403 container init 420af5726f6f10b9e2ab86b8caf914b184ff25a8ae6e18c4ad7f1881243fefcb (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:28:08.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 podman[62666]: 2024-09-13 14:28:08.318772461 +0000 UTC m=+0.380538585 container start 420af5726f6f10b9e2ab86b8caf914b184ff25a8ae6e18c4ad7f1881243fefcb (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:28:08.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.342Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T14:28:08.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.342Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T14:28:08.720 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.342Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi204 (none))" 2024-09-13T14:28:08.720 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.343Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:28:08.720 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.343Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:28:08.720 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.344Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:28:08.720 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.345Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T14:28:08.720 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.346Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:28:08.720 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.346Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:28:08.721 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.348Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:28:08.721 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.348Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.179µs 2024-09-13T14:28:08.721 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.348Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:28:08.721 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.348Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-13T14:28:08.721 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.349Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-13T14:28:08.721 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.349Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=29.005µs wal_replay_duration=603.079µs wbl_replay_duration=109ns total_replay_duration=661.702µs 2024-09-13T14:28:08.721 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.349Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:28:08.721 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.349Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T14:28:08.722 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.349Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:28:08.722 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.365Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.55331ms db_storage=939ns remote_storage=1.214µs web_handler=402ns query_engine=545ns scrape=381.224µs scrape_sd=26.718µs notify=26.379µs notify_sd=6.391µs rules=14.707084ms tracing=6.03µs 2024-09-13T14:28:08.722 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.365Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T14:28:08.722 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:28:08.365Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:28:08.722 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 bash[62666]: 420af5726f6f10b9e2ab86b8caf914b184ff25a8ae6e18c4ad7f1881243fefcb 2024-09-13T14:28:08.722 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:28:08 smithi204 systemd[1]: Started Ceph prometheus.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:28:08.931 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:08 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:28:08.931 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:08 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:08] ENGINE Bus STOPPED 2024-09-13T14:28:08.931 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:08 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:08] ENGINE Bus STARTING 2024-09-13T14:28:09.297 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:09] ENGINE Serving on http://:::9283 2024-09-13T14:28:09.297 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:09] ENGINE Bus STARTED 2024-09-13T14:28:09.297 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:09] ENGINE Bus STOPPING 2024-09-13T14:28:09.572 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:28:09.572 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:09] ENGINE Bus STOPPED 2024-09-13T14:28:09.572 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:09] ENGINE Bus STARTING 2024-09-13T14:28:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:28:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:28:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi096.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:28:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi096.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:28:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:28:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:28:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi204.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:28:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi204.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:28:09.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:09.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:28:09.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:28:09.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi204.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:28:09.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi204.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:28:09.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:09.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:28:09.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:09.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:09.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:28:09.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:09.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:09.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:28:09.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:28:09.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi096.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:28:09.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi096.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:28:09.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:09.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:28:09.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:28:09.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi204.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:28:09.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi204.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:28:09.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:09.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:28:09.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:28:09.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi204.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:28:09.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi204.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:28:09.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:09.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:28:09.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:09.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:09.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:28:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:28:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:28:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi096.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:28:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi096.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:28:09.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:09.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:28:09.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:28:09.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi204.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:28:09.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi204.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:28:09.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:09.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:28:09.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:28:09.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi204.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:28:09.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi204.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:28:09.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:09.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:28:09.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:09.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:09.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:09 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:28:10.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:09] ENGINE Serving on http://:::9283 2024-09-13T14:28:10.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:09] ENGINE Bus STARTED 2024-09-13T14:28:10.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:09] ENGINE Bus STOPPING 2024-09-13T14:28:10.480 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:10 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:10] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:28:10.480 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:10 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:10] ENGINE Bus STOPPED 2024-09-13T14:28:10.480 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:10 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:10] ENGINE Bus STARTING 2024-09-13T14:28:10.480 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:10 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:10] ENGINE Serving on http://:::9283 2024-09-13T14:28:10.480 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:10 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:28:10] ENGINE Bus STARTED 2024-09-13T14:28:10.780 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:10 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-13T14:28:10.780 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:10 smithi204 ceph-mon[32546]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T14:28:10.780 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:10 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:28:10.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:10 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-13T14:28:10.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:10 smithi096 ceph-mon[39645]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T14:28:10.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:10 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:28:10.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:10 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a[29225]: 2024-09-13T14:28:10.489+0000 7f9e1b023640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:28:10.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:10 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-13T14:28:10.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:10 smithi096 ceph-mon[29229]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T14:28:10.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:10 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:28:11.534 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-13T14:28:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:11 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-13T14:28:11.738 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:11 smithi204 ceph-mon[32546]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T14:28:11.738 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:11 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:28:11.738 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:11 smithi204 ceph-mon[32546]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:28:11.739 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:11 smithi204 ceph-mon[32546]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T14:28:11.739 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:11 smithi204 ceph-mon[32546]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T14:28:11.739 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:11 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-13T14:28:11.739 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:11 smithi204 ceph-mon[32546]: fsmap foo:0 2024-09-13T14:28:11.739 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:11 smithi204 ceph-mon[32546]: Saving service mds.foo spec with placement count:2 2024-09-13T14:28:11.739 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:11 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:11.739 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:11 smithi204 ceph-mon[32546]: pgmap v53: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:11.773 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-13T14:28:11.773 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[29229]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T14:28:11.773 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:28:11.773 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[29229]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:28:11.773 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[29229]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T14:28:11.773 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[29229]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T14:28:11.773 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-13T14:28:11.773 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[29229]: fsmap foo:0 2024-09-13T14:28:11.773 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[29229]: Saving service mds.foo spec with placement count:2 2024-09-13T14:28:11.773 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:11.774 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[29229]: pgmap v53: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:11.774 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-13T14:28:11.774 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[39645]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T14:28:11.774 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:28:11.774 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[39645]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:28:11.774 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[39645]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T14:28:11.774 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[39645]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T14:28:11.774 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-13T14:28:11.774 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[39645]: fsmap foo:0 2024-09-13T14:28:11.774 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[39645]: Saving service mds.foo spec with placement count:2 2024-09-13T14:28:11.775 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:11.775 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:11 smithi096 ceph-mon[39645]: pgmap v53: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:12.225 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:12.618 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[29229]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T14:28:12.619 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:12.619 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:12.619 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:12.619 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:12.619 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:12.619 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:28:12.619 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:12.619 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi096.dqiqpz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:28:12.619 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi096.dqiqpz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:28:12.619 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:12.619 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[39645]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T14:28:12.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:12.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:12.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:12.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:12.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:12.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:28:12.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:12.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi096.dqiqpz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:28:12.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi096.dqiqpz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:28:12.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:12 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:12.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:12 smithi204 ceph-mon[32546]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T14:28:12.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:12 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:12.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:12 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:12.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:12 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:12.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:12 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:12.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:12 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:12.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:12 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:28:12.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:12 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:12.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:12 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi096.dqiqpz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:28:12.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:12 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi096.dqiqpz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:28:12.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:12 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:13.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:13 smithi096 ceph-mon[39645]: Deploying daemon mds.foo.smithi096.dqiqpz on smithi096 2024-09-13T14:28:13.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:13 smithi096 ceph-mon[39645]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T14:28:13.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:13 smithi096 ceph-mon[39645]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:13.573 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:28:13 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[72700]: ts=2024-09-13T14:28:13.513Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002962238s 2024-09-13T14:28:13.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:13 smithi096 ceph-mon[29229]: Deploying daemon mds.foo.smithi096.dqiqpz on smithi096 2024-09-13T14:28:13.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:13 smithi096 ceph-mon[29229]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T14:28:13.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:13 smithi096 ceph-mon[29229]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:13.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:13 smithi204 ceph-mon[32546]: Deploying daemon mds.foo.smithi096.dqiqpz on smithi096 2024-09-13T14:28:13.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:13 smithi204 ceph-mon[32546]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T14:28:13.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:13 smithi204 ceph-mon[32546]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:15.662 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:15 smithi096 ceph-mon[29229]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:15.662 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:15 smithi096 ceph-mon[39645]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:15.685 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-13T14:28:15.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:15 smithi204 ceph-mon[32546]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:16.416 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:16.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[29229]: daemon mds.foo.smithi096.dqiqpz assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-13T14:28:16.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[29229]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:28:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[29229]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T14:28:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[29229]: Cluster is now healthy 2024-09-13T14:28:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[29229]: mds.? [v2:172.21.15.96:6834/1494616691,v1:172.21.15.96:6835/1494616691] up:boot 2024-09-13T14:28:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[29229]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:creating} 2024-09-13T14:28:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi096.dqiqpz"}]: dispatch 2024-09-13T14:28:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[29229]: daemon mds.foo.smithi096.dqiqpz is now active in filesystem foo as rank 0 2024-09-13T14:28:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi204.hbwbot", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:28:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi204.hbwbot", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:28:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:16.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[39645]: daemon mds.foo.smithi096.dqiqpz assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-13T14:28:16.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[39645]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:28:16.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[39645]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T14:28:16.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[39645]: Cluster is now healthy 2024-09-13T14:28:16.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[39645]: mds.? [v2:172.21.15.96:6834/1494616691,v1:172.21.15.96:6835/1494616691] up:boot 2024-09-13T14:28:16.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[39645]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:creating} 2024-09-13T14:28:16.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi096.dqiqpz"}]: dispatch 2024-09-13T14:28:16.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[39645]: daemon mds.foo.smithi096.dqiqpz is now active in filesystem foo as rank 0 2024-09-13T14:28:16.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:16.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:16.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:16.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi204.hbwbot", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:28:16.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi204.hbwbot", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:28:16.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:16 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:16.938 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:16 smithi204 ceph-mon[32546]: daemon mds.foo.smithi096.dqiqpz assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-13T14:28:16.938 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:16 smithi204 ceph-mon[32546]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:28:16.938 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:16 smithi204 ceph-mon[32546]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T14:28:16.938 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:16 smithi204 ceph-mon[32546]: Cluster is now healthy 2024-09-13T14:28:16.938 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:16 smithi204 ceph-mon[32546]: mds.? [v2:172.21.15.96:6834/1494616691,v1:172.21.15.96:6835/1494616691] up:boot 2024-09-13T14:28:16.938 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:16 smithi204 ceph-mon[32546]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:creating} 2024-09-13T14:28:16.939 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:16 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi096.dqiqpz"}]: dispatch 2024-09-13T14:28:16.939 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:16 smithi204 ceph-mon[32546]: daemon mds.foo.smithi096.dqiqpz is now active in filesystem foo as rank 0 2024-09-13T14:28:16.939 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:16 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:16.939 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:16 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:16.939 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:16 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:16.939 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:16 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi204.hbwbot", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:28:16.939 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:16 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi204.hbwbot", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:28:16.939 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:16 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:17.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:17 smithi096 ceph-mon[29229]: Deploying daemon mds.foo.smithi204.hbwbot on smithi204 2024-09-13T14:28:17.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:17 smithi096 ceph-mon[29229]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:17.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:17 smithi096 ceph-mon[29229]: mds.? [v2:172.21.15.96:6834/1494616691,v1:172.21.15.96:6835/1494616691] up:active 2024-09-13T14:28:17.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:17 smithi096 ceph-mon[29229]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 2024-09-13T14:28:17.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:17 smithi096 ceph-mon[39645]: Deploying daemon mds.foo.smithi204.hbwbot on smithi204 2024-09-13T14:28:17.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:17 smithi096 ceph-mon[39645]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:17.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:17 smithi096 ceph-mon[39645]: mds.? [v2:172.21.15.96:6834/1494616691,v1:172.21.15.96:6835/1494616691] up:active 2024-09-13T14:28:17.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:17 smithi096 ceph-mon[39645]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 2024-09-13T14:28:17.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:17 smithi204 ceph-mon[32546]: Deploying daemon mds.foo.smithi204.hbwbot on smithi204 2024-09-13T14:28:17.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:17 smithi204 ceph-mon[32546]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:17.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:17 smithi204 ceph-mon[32546]: mds.? [v2:172.21.15.96:6834/1494616691,v1:172.21.15.96:6835/1494616691] up:active 2024-09-13T14:28:17.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:17 smithi204 ceph-mon[32546]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 2024-09-13T14:28:18.248 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:28:17] "GET /metrics HTTP/1.1" 200 203805 "" "Prometheus/2.43.0" 2024-09-13T14:28:18.851 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-13T14:28:18.862 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T14:28:18.862 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T14:28:18.862 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T14:28:19.146 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:20.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[29229]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-13T14:28:20.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:20.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:20.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:20.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:20.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:20.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:28:20.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[29229]: mds.? [v2:172.21.15.204:6832/291805447,v1:172.21.15.204:6833/291805447] up:boot 2024-09-13T14:28:20.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[29229]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:28:20.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi204.hbwbot"}]: dispatch 2024-09-13T14:28:20.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[29229]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:28:20.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[39645]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-13T14:28:20.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:20.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:20.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:20.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:20.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:20.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:28:20.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[39645]: mds.? [v2:172.21.15.204:6832/291805447,v1:172.21.15.204:6833/291805447] up:boot 2024-09-13T14:28:20.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[39645]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:28:20.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi204.hbwbot"}]: dispatch 2024-09-13T14:28:20.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:19 smithi096 ceph-mon[39645]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:28:20.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:19 smithi204 ceph-mon[32546]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-13T14:28:20.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:19 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:20.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:19 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:20.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:19 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:20.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:19 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:20.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:19 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:20.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:19 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:28:20.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:19 smithi204 ceph-mon[32546]: mds.? [v2:172.21.15.204:6832/291805447,v1:172.21.15.204:6833/291805447] up:boot 2024-09-13T14:28:20.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:19 smithi204 ceph-mon[32546]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:28:20.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:19 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi204.hbwbot"}]: dispatch 2024-09-13T14:28:20.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:19 smithi204 ceph-mon[32546]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:28:20.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:20 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:20.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:20 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:21.056 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:28:21.057 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:20 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:21.057 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:20 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:21.058 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:20 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:21.058 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:20 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:22.018 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:21 smithi096 ceph-mon[29229]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-13T14:28:22.018 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:21 smithi096 ceph-mon[29229]: mds.? [v2:172.21.15.96:6834/1494616691,v1:172.21.15.96:6835/1494616691] up:active 2024-09-13T14:28:22.018 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:21 smithi096 ceph-mon[29229]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:28:22.018 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:21 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1900028494' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:28:22.019 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:21 smithi096 ceph-mon[39645]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-13T14:28:22.019 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:21 smithi096 ceph-mon[39645]: mds.? [v2:172.21.15.96:6834/1494616691,v1:172.21.15.96:6835/1494616691] up:active 2024-09-13T14:28:22.019 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:21 smithi096 ceph-mon[39645]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:28:22.019 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:21 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1900028494' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:28:22.068 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:21 smithi204 ceph-mon[32546]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-13T14:28:22.068 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:21 smithi204 ceph-mon[32546]: mds.? [v2:172.21.15.96:6834/1494616691,v1:172.21.15.96:6835/1494616691] up:active 2024-09-13T14:28:22.068 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:21 smithi204 ceph-mon[32546]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:28:22.069 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:21 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1900028494' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:28:22.410 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":20,"flags":0,"active_gid":24335,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6800","nonce":776211071},{"type":"v1","addr":"172.21.15.96:6801","nonce":776211071}]},"active_addr":"172.21.15.96:6801/776211071","active_change":"2024-09-13T14:26:50.514430+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24331,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.96:8443/","prometheus":"http://172.21.15.96:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":52,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.96:0","nonce":902309444}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.96:0","nonce":3782790019}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.96:0","nonce":3050864675}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.96:0","nonce":1652886381}]}]} 2024-09-13T14:28:22.414 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T14:28:22.414 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T14:28:22.414 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:28:22.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:28:22 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:28:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:28:22.687 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:23.809 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:23 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:23.810 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:23 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:23.810 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:23 smithi096 ceph-mon[29229]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-13T14:28:23.810 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:23 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:23.810 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:23 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:23.810 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:23 smithi096 ceph-mon[39645]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-13T14:28:23.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:23 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:23.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:23 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:23.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:23 smithi204 ceph-mon[32546]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-13T14:28:24.816 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:24 smithi096 ceph-mon[29229]: mds.? [v2:172.21.15.204:6832/291805447,v1:172.21.15.204:6833/291805447] up:standby 2024-09-13T14:28:24.817 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:24 smithi096 ceph-mon[29229]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:28:24.817 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:24 smithi096 ceph-mon[39645]: mds.? [v2:172.21.15.204:6832/291805447,v1:172.21.15.204:6833/291805447] up:standby 2024-09-13T14:28:24.817 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:24 smithi096 ceph-mon[39645]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:28:24.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:24 smithi204 ceph-mon[32546]: mds.? [v2:172.21.15.204:6832/291805447,v1:172.21.15.204:6833/291805447] up:standby 2024-09-13T14:28:24.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:24 smithi204 ceph-mon[32546]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:28:25.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:25 smithi096 ceph-mon[29229]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:28:25.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:25 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:25.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:25 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:25.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:25 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:25.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:25 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:28:25.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:25 smithi096 ceph-mon[39645]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:28:25.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:25 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:25.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:25 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:25.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:25 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:25.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:25 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:28:25.928 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:28:25.928 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":64,"fsid":"51e50ed6-71db-11ef-bceb-c7b262605968","created":"2024-09-13T14:21:16.367777+0000","modified":"2024-09-13T14:28:12.500588+0000","last_up_change":"2024-09-13T14:26:18.016817+0000","last_in_change":"2024-09-13T14:26:00.677688+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:24:20.514144+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:26:54.629267+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:28:09.271608+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:28:09.734465+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6802","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6803","nonce":766102065}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6804","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6805","nonce":766102065}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6808","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6809","nonce":766102065}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6806","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6807","nonce":766102065}]},"public_addr":"172.21.15.96:6803/766102065","cluster_addr":"172.21.15.96:6805/766102065","heartbeat_back_addr":"172.21.15.96:6809/766102065","heartbeat_front_addr":"172.21.15.96:6807/766102065","state":["exists","up"]},{"osd":1,"uuid":"1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6810","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6811","nonce":401053665}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6812","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6813","nonce":401053665}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6816","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6817","nonce":401053665}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6814","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6815","nonce":401053665}]},"public_addr":"172.21.15.96:6811/401053665","cluster_addr":"172.21.15.96:6813/401053665","heartbeat_back_addr":"172.21.15.96:6817/401053665","heartbeat_front_addr":"172.21.15.96:6815/401053665","state":["exists","up"]},{"osd":2,"uuid":"6a92a5b3-bdc7-499a-8b0f-2b0936fd9598","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6818","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6819","nonce":2386475672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6820","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6821","nonce":2386475672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6824","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6825","nonce":2386475672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6822","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6823","nonce":2386475672}]},"public_addr":"172.21.15.96:6819/2386475672","cluster_addr":"172.21.15.96:6821/2386475672","heartbeat_back_addr":"172.21.15.96:6825/2386475672","heartbeat_front_addr":"172.21.15.96:6823/2386475672","state":["exists","up"]},{"osd":3,"uuid":"1230646c-934e-4eb4-b4b9-16be52ffba66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6826","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6827","nonce":282100645}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6828","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6829","nonce":282100645}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6832","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6833","nonce":282100645}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6830","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6831","nonce":282100645}]},"public_addr":"172.21.15.96:6827/282100645","cluster_addr":"172.21.15.96:6829/282100645","heartbeat_back_addr":"172.21.15.96:6833/282100645","heartbeat_front_addr":"172.21.15.96:6831/282100645","state":["exists","up"]},{"osd":4,"uuid":"14115269-3d41-4ebf-8613-71ed59ec0f29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6800","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6801","nonce":1188704632}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6802","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6803","nonce":1188704632}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6806","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6807","nonce":1188704632}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6804","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6805","nonce":1188704632}]},"public_addr":"172.21.15.204:6801/1188704632","cluster_addr":"172.21.15.204:6803/1188704632","heartbeat_back_addr":"172.21.15.204:6807/1188704632","heartbeat_front_addr":"172.21.15.204:6805/1188704632","state":["exists","up"]},{"osd":5,"uuid":"15148a0c-df44-419e-b0f0-2b1600c207f4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6808","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6809","nonce":580729752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6810","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6811","nonce":580729752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6814","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6815","nonce":580729752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6812","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6813","nonce":580729752}]},"public_addr":"172.21.15.204:6809/580729752","cluster_addr":"172.21.15.204:6811/580729752","heartbeat_back_addr":"172.21.15.204:6815/580729752","heartbeat_front_addr":"172.21.15.204:6813/580729752","state":["exists","up"]},{"osd":6,"uuid":"8404e088-3523-4c21-b7c2-0a31b22e0279","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6816","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6817","nonce":118365497}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6818","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6819","nonce":118365497}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6822","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6823","nonce":118365497}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6820","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6821","nonce":118365497}]},"public_addr":"172.21.15.204:6817/118365497","cluster_addr":"172.21.15.204:6819/118365497","heartbeat_back_addr":"172.21.15.204:6823/118365497","heartbeat_front_addr":"172.21.15.204:6821/118365497","state":["exists","up"]},{"osd":7,"uuid":"93b23a35-0059-4e03-a2ba-28fee6431892","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6824","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6825","nonce":3529639424}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6826","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6827","nonce":3529639424}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6830","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6831","nonce":3529639424}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6828","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6829","nonce":3529639424}]},"public_addr":"172.21.15.204:6825/3529639424","cluster_addr":"172.21.15.204:6827/3529639424","heartbeat_back_addr":"172.21.15.204:6831/3529639424","heartbeat_front_addr":"172.21.15.204:6829/3529639424","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:28.841233+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:53.021149+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:17.120825+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:41.339674+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:02.571454+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:26.608523+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:51.119331+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:26:16.050687+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.96:0/1790922234":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/260172418":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/2292899363":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/960238088":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/378263291":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/376655558":"2024-09-14T14:21:45.592521+0000","172.21.15.96:6800/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6800/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:6801/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/1644412166":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6801/2749644659":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/19119516":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/752011643":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/1494034928":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6801/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6800/2749644659":"2024-09-14T14:26:50.514276+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:28:25.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:25 smithi204 ceph-mon[32546]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:28:25.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:25 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:25.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:25 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:25.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:25 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:25.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:25 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:28:26.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:26 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:26.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:26 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:28:26.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:26 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1721249370' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:28:26.767 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T14:28:26.767 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:28:26.792 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:26 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:26.792 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:26 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:28:26.792 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:26 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1721249370' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:28:26.793 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:26 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:26.793 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:26 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:28:26.793 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:26 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1721249370' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:28:27.453 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:27.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:27 smithi096 ceph-mon[29229]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:28:27.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:27 smithi096 ceph-mon[39645]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:28:27.838 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:27 smithi204 ceph-mon[32546]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:28:28.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:28:27] "GET /metrics HTTP/1.1" 200 239632 "" "Prometheus/2.43.0" 2024-09-13T14:28:29.291 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:28:29.291 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":64,"fsid":"51e50ed6-71db-11ef-bceb-c7b262605968","created":"2024-09-13T14:21:16.367777+0000","modified":"2024-09-13T14:28:12.500588+0000","last_up_change":"2024-09-13T14:26:18.016817+0000","last_in_change":"2024-09-13T14:26:00.677688+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:24:20.514144+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:26:54.629267+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:28:09.271608+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:28:09.734465+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6802","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6803","nonce":766102065}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6804","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6805","nonce":766102065}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6808","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6809","nonce":766102065}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6806","nonce":766102065},{"type":"v1","addr":"172.21.15.96:6807","nonce":766102065}]},"public_addr":"172.21.15.96:6803/766102065","cluster_addr":"172.21.15.96:6805/766102065","heartbeat_back_addr":"172.21.15.96:6809/766102065","heartbeat_front_addr":"172.21.15.96:6807/766102065","state":["exists","up"]},{"osd":1,"uuid":"1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6810","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6811","nonce":401053665}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6812","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6813","nonce":401053665}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6816","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6817","nonce":401053665}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6814","nonce":401053665},{"type":"v1","addr":"172.21.15.96:6815","nonce":401053665}]},"public_addr":"172.21.15.96:6811/401053665","cluster_addr":"172.21.15.96:6813/401053665","heartbeat_back_addr":"172.21.15.96:6817/401053665","heartbeat_front_addr":"172.21.15.96:6815/401053665","state":["exists","up"]},{"osd":2,"uuid":"6a92a5b3-bdc7-499a-8b0f-2b0936fd9598","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6818","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6819","nonce":2386475672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6820","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6821","nonce":2386475672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6824","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6825","nonce":2386475672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6822","nonce":2386475672},{"type":"v1","addr":"172.21.15.96:6823","nonce":2386475672}]},"public_addr":"172.21.15.96:6819/2386475672","cluster_addr":"172.21.15.96:6821/2386475672","heartbeat_back_addr":"172.21.15.96:6825/2386475672","heartbeat_front_addr":"172.21.15.96:6823/2386475672","state":["exists","up"]},{"osd":3,"uuid":"1230646c-934e-4eb4-b4b9-16be52ffba66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6826","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6827","nonce":282100645}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6828","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6829","nonce":282100645}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6832","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6833","nonce":282100645}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6830","nonce":282100645},{"type":"v1","addr":"172.21.15.96:6831","nonce":282100645}]},"public_addr":"172.21.15.96:6827/282100645","cluster_addr":"172.21.15.96:6829/282100645","heartbeat_back_addr":"172.21.15.96:6833/282100645","heartbeat_front_addr":"172.21.15.96:6831/282100645","state":["exists","up"]},{"osd":4,"uuid":"14115269-3d41-4ebf-8613-71ed59ec0f29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6800","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6801","nonce":1188704632}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6802","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6803","nonce":1188704632}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6806","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6807","nonce":1188704632}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6804","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6805","nonce":1188704632}]},"public_addr":"172.21.15.204:6801/1188704632","cluster_addr":"172.21.15.204:6803/1188704632","heartbeat_back_addr":"172.21.15.204:6807/1188704632","heartbeat_front_addr":"172.21.15.204:6805/1188704632","state":["exists","up"]},{"osd":5,"uuid":"15148a0c-df44-419e-b0f0-2b1600c207f4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6808","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6809","nonce":580729752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6810","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6811","nonce":580729752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6814","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6815","nonce":580729752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6812","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6813","nonce":580729752}]},"public_addr":"172.21.15.204:6809/580729752","cluster_addr":"172.21.15.204:6811/580729752","heartbeat_back_addr":"172.21.15.204:6815/580729752","heartbeat_front_addr":"172.21.15.204:6813/580729752","state":["exists","up"]},{"osd":6,"uuid":"8404e088-3523-4c21-b7c2-0a31b22e0279","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6816","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6817","nonce":118365497}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6818","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6819","nonce":118365497}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6822","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6823","nonce":118365497}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6820","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6821","nonce":118365497}]},"public_addr":"172.21.15.204:6817/118365497","cluster_addr":"172.21.15.204:6819/118365497","heartbeat_back_addr":"172.21.15.204:6823/118365497","heartbeat_front_addr":"172.21.15.204:6821/118365497","state":["exists","up"]},{"osd":7,"uuid":"93b23a35-0059-4e03-a2ba-28fee6431892","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6824","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6825","nonce":3529639424}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6826","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6827","nonce":3529639424}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6830","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6831","nonce":3529639424}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6828","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6829","nonce":3529639424}]},"public_addr":"172.21.15.204:6825/3529639424","cluster_addr":"172.21.15.204:6827/3529639424","heartbeat_back_addr":"172.21.15.204:6831/3529639424","heartbeat_front_addr":"172.21.15.204:6829/3529639424","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:28.841233+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:53.021149+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:17.120825+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:41.339674+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:02.571454+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:26.608523+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:51.119331+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:26:16.050687+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.96:0/1790922234":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/260172418":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/2292899363":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/960238088":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/378263291":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/376655558":"2024-09-14T14:21:45.592521+0000","172.21.15.96:6800/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6800/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:6801/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/1644412166":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6801/2749644659":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/19119516":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/752011643":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/1494034928":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6801/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6800/2749644659":"2024-09-14T14:26:50.514276+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:28:29.931 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:29 smithi096 ceph-mon[29229]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:28:29.932 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:29 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:29.932 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:29 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:29.932 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:29 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3846469595' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:28:29.932 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:29 smithi096 ceph-mon[39645]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:28:29.932 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:29 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:29.932 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:29 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:29.932 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:29 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3846469595' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:28:29.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:29 smithi204 ceph-mon[32546]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:28:29.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:29 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:29.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:29 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:29.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:29 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3846469595' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:28:30.670 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T14:28:30.671 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T14:28:30.671 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T14:28:30.672 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T14:28:30.672 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T14:28:30.672 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T14:28:30.673 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T14:28:30.673 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T14:28:31.508 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:31.508 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:31.510 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:31.512 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:31.516 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:31.517 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:31.538 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:31.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:31 smithi096 ceph-mon[29229]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:28:31.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:31 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:31.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:31 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:31.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:31 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:31.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:31 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:28:31.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:31 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:31.812 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:31 smithi096 ceph-mon[39645]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:28:31.813 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:31 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:31.813 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:31 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:31.813 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:31 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:31.813 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:31 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:28:31.813 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:31 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:31.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:31 smithi204 ceph-mon[32546]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:28:31.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:31 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:31.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:31 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:31.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:31 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:31.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:31 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:28:31.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:31 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:28:32.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:28:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:28:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:28:33.931 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:33 smithi096 ceph-mon[29229]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:28:33.931 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:33 smithi096 ceph-mon[39645]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:28:33.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:33 smithi204 ceph-mon[32546]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:28:35.702 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:35 smithi096 ceph-mon[29229]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:35.703 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:35 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:35.703 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:35 smithi096 ceph-mon[39645]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:35.703 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:35 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:35.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:35 smithi204 ceph-mon[32546]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:35.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:35 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:37.719 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:37 smithi096 ceph-mon[29229]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:37.719 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:37 smithi096 ceph-mon[39645]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:37.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:37 smithi204 ceph-mon[32546]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:38.027 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:28:37] "GET /metrics HTTP/1.1" 200 239620 "" "Prometheus/2.43.0" 2024-09-13T14:28:39.937 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:39 smithi096 ceph-mon[29229]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:39.937 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:39 smithi096 ceph-mon[39645]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:39.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:39 smithi204 ceph-mon[32546]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:41.493 INFO:teuthology.orchestra.run.smithi096.stdout:111669149747 2024-09-13T14:28:41.494 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T14:28:41.718 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:41 smithi096 ceph-mon[29229]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:41.718 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:41 smithi096 ceph-mon[39645]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:41.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:41 smithi204 ceph-mon[32546]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:42.440 INFO:teuthology.orchestra.run.smithi096.stdout:55834574909 2024-09-13T14:28:42.441 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T14:28:42.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:28:42 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:28:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:28:43.159 INFO:teuthology.orchestra.run.smithi096.stdout:34359738434 2024-09-13T14:28:43.160 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T14:28:43.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:43 smithi204 ceph-mon[32546]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:44.033 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:43 smithi096 ceph-mon[29229]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:44.033 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:43 smithi096 ceph-mon[39645]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:44.057 INFO:teuthology.orchestra.run.smithi096.stdout:158913789994 2024-09-13T14:28:44.057 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T14:28:45.114 INFO:teuthology.orchestra.run.smithi096.stdout:133143986222 2024-09-13T14:28:45.114 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T14:28:45.135 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:45.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:45 smithi096 ceph-mon[29229]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:45.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:45 smithi096 ceph-mon[39645]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:45.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:45 smithi204 ceph-mon[32546]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:47.393 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:47.393 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:47.394 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:47.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:47 smithi096 ceph-mon[29229]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:47.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:47 smithi096 ceph-mon[39645]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:47.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:47 smithi204 ceph-mon[32546]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:48.285 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:28:47] "GET /metrics HTTP/1.1" 200 239620 "" "Prometheus/2.43.0" 2024-09-13T14:28:48.309 INFO:teuthology.orchestra.run.smithi096.stdout:210453397537 2024-09-13T14:28:48.309 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T14:28:49.072 INFO:teuthology.orchestra.run.smithi096.stdout:77309411385 2024-09-13T14:28:49.073 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T14:28:49.861 INFO:teuthology.orchestra.run.smithi096.stdout:184683593766 2024-09-13T14:28:49.862 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T14:28:49.884 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:49 smithi096 ceph-mon[29229]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:49.884 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:49 smithi096 ceph-mon[39645]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:49.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:49 smithi204 ceph-mon[32546]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:50.876 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:50 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2024-09-13T14:28:50.876 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:50 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-13T14:28:50.876 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:50 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-09-13T14:28:50.876 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:50 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:50.877 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:50 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2024-09-13T14:28:50.877 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:50 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-13T14:28:50.877 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:50 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-09-13T14:28:50.877 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:50 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:50.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:50 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2024-09-13T14:28:50.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:50 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-13T14:28:50.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:50 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-09-13T14:28:50.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:50 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:51.311 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:51.453 INFO:teuthology.orchestra.run.smithi096.stdout:55834574911 2024-09-13T14:28:51.706 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:51 smithi096 ceph-mon[29229]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:51.706 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-09-13T14:28:51.706 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-09-13T14:28:51.706 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:51 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-13T14:28:51.707 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:51 smithi096 ceph-mon[29229]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T14:28:51.707 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:51 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3795890224' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:28:51.707 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:51 smithi096 ceph-mon[39645]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:51.707 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-09-13T14:28:51.707 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-09-13T14:28:51.707 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:51 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-13T14:28:51.707 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:51 smithi096 ceph-mon[39645]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T14:28:51.707 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:51 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3795890224' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:28:51.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:51 smithi204 ceph-mon[32546]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:51.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-09-13T14:28:51.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-09-13T14:28:51.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:51 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-13T14:28:51.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:51 smithi204 ceph-mon[32546]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T14:28:51.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:51 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3795890224' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:28:52.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:28:52 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:28:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:28:52.933 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:52 smithi096 ceph-mon[29229]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T14:28:52.934 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:52 smithi096 ceph-mon[39645]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T14:28:52.962 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574909 got 55834574911 for osd.1 2024-09-13T14:28:52.962 DEBUG:teuthology.parallel:result is None 2024-09-13T14:28:52.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:52 smithi204 ceph-mon[32546]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T14:28:53.771 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:53 smithi096 ceph-mon[39645]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:53.772 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:53 smithi096 ceph-mon[29229]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:53.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:53 smithi204 ceph-mon[32546]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:54.418 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:54.418 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:54.419 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:54.667 INFO:teuthology.orchestra.run.smithi096.stdout:158913789996 2024-09-13T14:28:55.786 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:55.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:55 smithi096 ceph-mon[29229]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:55.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:55 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2736570684' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:28:55.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:55 smithi096 ceph-mon[39645]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:55.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:55 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2736570684' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:28:55.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:55 smithi204 ceph-mon[32546]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:55.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:55 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2736570684' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:28:56.165 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913789994 got 158913789996 for osd.5 2024-09-13T14:28:56.165 DEBUG:teuthology.parallel:result is None 2024-09-13T14:28:56.999 INFO:teuthology.orchestra.run.smithi096.stdout:34359738437 2024-09-13T14:28:57.872 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:57 smithi096 ceph-mon[29229]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:57.872 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:57 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/213274920' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:28:57.872 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:57 smithi096 ceph-mon[39645]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:57.872 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:57 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/213274920' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:28:57.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:57 smithi204 ceph-mon[32546]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:28:57.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:57 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/213274920' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:28:58.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:28:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:28:57] "GET /metrics HTTP/1.1" 200 239596 "" "Prometheus/2.43.0" 2024-09-13T14:28:58.395 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738434 got 34359738437 for osd.0 2024-09-13T14:28:58.395 DEBUG:teuthology.parallel:result is None 2024-09-13T14:28:58.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:58 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/853945190' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:28:58.969 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:58 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/853945190' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:28:58.969 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:58 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/853945190' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:28:59.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:28:59 smithi204 ceph-mon[32546]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:00.043 INFO:teuthology.orchestra.run.smithi096.stdout:111669149751 2024-09-13T14:29:00.044 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:28:59 smithi096 ceph-mon[29229]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:00.044 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:28:59 smithi096 ceph-mon[39645]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:00.742 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149747 got 111669149751 for osd.3 2024-09-13T14:29:00.743 DEBUG:teuthology.parallel:result is None 2024-09-13T14:29:01.951 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:01 smithi096 ceph-mon[29229]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:01.952 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:01 smithi096 ceph-mon[39645]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:01.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:01 smithi204 ceph-mon[32546]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:02.296 INFO:teuthology.orchestra.run.smithi096.stdout:210453397539 2024-09-13T14:29:02.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:29:02 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:29:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:29:02.491 INFO:teuthology.orchestra.run.smithi096.stdout:133143986226 2024-09-13T14:29:02.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:02 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2363377634' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:29:02.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:02 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/881379431' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:29:02.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:02 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2363377634' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:29:02.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:02 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/881379431' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:29:02.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:02 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2363377634' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:29:02.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:02 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/881379431' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:29:03.460 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986222 got 133143986226 for osd.4 2024-09-13T14:29:03.460 DEBUG:teuthology.parallel:result is None 2024-09-13T14:29:03.594 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397537 got 210453397539 for osd.7 2024-09-13T14:29:03.594 DEBUG:teuthology.parallel:result is None 2024-09-13T14:29:03.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:03 smithi096 ceph-mon[29229]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:03.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:03 smithi096 ceph-mon[39645]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:03.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:03 smithi204 ceph-mon[32546]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:05.357 INFO:teuthology.orchestra.run.smithi096.stdout:77309411389 2024-09-13T14:29:05.545 INFO:teuthology.orchestra.run.smithi096.stdout:184683593769 2024-09-13T14:29:05.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:05 smithi096 ceph-mon[29229]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:05.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:05 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/753877434' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:29:05.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:05 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1573430244' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:29:05.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:05 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:05 smithi096 ceph-mon[39645]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:05 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/753877434' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:29:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:05 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1573430244' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:29:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:05 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:05.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:05 smithi204 ceph-mon[32546]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:05.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:05 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/753877434' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:29:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:05 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1573430244' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:29:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:05 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:06.203 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593766 got 184683593769 for osd.6 2024-09-13T14:29:06.203 DEBUG:teuthology.parallel:result is None 2024-09-13T14:29:06.756 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411385 got 77309411389 for osd.2 2024-09-13T14:29:06.756 DEBUG:teuthology.parallel:result is None 2024-09-13T14:29:06.757 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:29:06.757 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:29:07.000 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:07.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:07 smithi204 ceph-mon[32546]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:08.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:07 smithi096 ceph-mon[29229]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:08.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:29:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:29:07] "GET /metrics HTTP/1.1" 200 239663 "" "Prometheus/2.43.0" 2024-09-13T14:29:08.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:07 smithi096 ceph-mon[39645]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:08.890 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:29:08.891 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:29:09.713 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":86,"stamp":"2024-09-13T14:29:08.556981+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592725,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":160,"num_write_kb":1643,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121,"ondisk_log_size":121,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385960,"kb_used_data":7080,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575256,"statfs":{"total":767960285184,"available":765517062144,"internally_reserved":0,"allocated":7249920,"data_stored":4149184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002608"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649338+0000","last_change":"2024-09-13T14:28:11.512308+0000","last_active":"2024-09-13T14:28:50.649338+0000","last_peered":"2024-09-13T14:28:50.649338+0000","last_clean":"2024-09-13T14:28:50.649338+0000","last_became_active":"2024-09-13T14:28:11.512003+0000","last_became_peered":"2024-09-13T14:28:11.512003+0000","last_unstale":"2024-09-13T14:28:50.649338+0000","last_undegraded":"2024-09-13T14:28:50.649338+0000","last_fullsized":"2024-09-13T14:28:50.649338+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:51:00.952944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587897+0000","last_change":"2024-09-13T14:28:10.508904+0000","last_active":"2024-09-13T14:28:51.587897+0000","last_peered":"2024-09-13T14:28:51.587897+0000","last_clean":"2024-09-13T14:28:51.587897+0000","last_became_active":"2024-09-13T14:28:10.508638+0000","last_became_peered":"2024-09-13T14:28:10.508638+0000","last_unstale":"2024-09-13T14:28:51.587897+0000","last_undegraded":"2024-09-13T14:28:51.587897+0000","last_fullsized":"2024-09-13T14:28:51.587897+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:22.870537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646422+0000","last_change":"2024-09-13T14:28:11.512154+0000","last_active":"2024-09-13T14:28:50.646422+0000","last_peered":"2024-09-13T14:28:50.646422+0000","last_clean":"2024-09-13T14:28:50.646422+0000","last_became_active":"2024-09-13T14:28:11.511903+0000","last_became_peered":"2024-09-13T14:28:11.511903+0000","last_unstale":"2024-09-13T14:28:50.646422+0000","last_undegraded":"2024-09-13T14:28:50.646422+0000","last_fullsized":"2024-09-13T14:28:50.646422+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:34.544812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587446+0000","last_change":"2024-09-13T14:28:10.496527+0000","last_active":"2024-09-13T14:28:51.587446+0000","last_peered":"2024-09-13T14:28:51.587446+0000","last_clean":"2024-09-13T14:28:51.587446+0000","last_became_active":"2024-09-13T14:28:10.496273+0000","last_became_peered":"2024-09-13T14:28:10.496273+0000","last_unstale":"2024-09-13T14:28:51.587446+0000","last_undegraded":"2024-09-13T14:28:51.587446+0000","last_fullsized":"2024-09-13T14:28:51.587446+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:30:48.755762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587354+0000","last_change":"2024-09-13T14:28:11.512769+0000","last_active":"2024-09-13T14:28:51.587354+0000","last_peered":"2024-09-13T14:28:51.587354+0000","last_clean":"2024-09-13T14:28:51.587354+0000","last_became_active":"2024-09-13T14:28:11.512391+0000","last_became_peered":"2024-09-13T14:28:11.512391+0000","last_unstale":"2024-09-13T14:28:51.587354+0000","last_undegraded":"2024-09-13T14:28:51.587354+0000","last_fullsized":"2024-09-13T14:28:51.587354+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:06:17.869350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587782+0000","last_change":"2024-09-13T14:28:10.508894+0000","last_active":"2024-09-13T14:28:51.587782+0000","last_peered":"2024-09-13T14:28:51.587782+0000","last_clean":"2024-09-13T14:28:51.587782+0000","last_became_active":"2024-09-13T14:28:10.508612+0000","last_became_peered":"2024-09-13T14:28:10.508612+0000","last_unstale":"2024-09-13T14:28:51.587782+0000","last_undegraded":"2024-09-13T14:28:51.587782+0000","last_fullsized":"2024-09-13T14:28:51.587782+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:01.774646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649251+0000","last_change":"2024-09-13T14:28:11.511942+0000","last_active":"2024-09-13T14:28:50.649251+0000","last_peered":"2024-09-13T14:28:50.649251+0000","last_clean":"2024-09-13T14:28:50.649251+0000","last_became_active":"2024-09-13T14:28:11.511494+0000","last_became_peered":"2024-09-13T14:28:11.511494+0000","last_unstale":"2024-09-13T14:28:50.649251+0000","last_undegraded":"2024-09-13T14:28:50.649251+0000","last_fullsized":"2024-09-13T14:28:50.649251+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:55.285495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":22,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646708+0000","last_change":"2024-09-13T14:28:10.507948+0000","last_active":"2024-09-13T14:28:50.646708+0000","last_peered":"2024-09-13T14:28:50.646708+0000","last_clean":"2024-09-13T14:28:50.646708+0000","last_became_active":"2024-09-13T14:28:10.507641+0000","last_became_peered":"2024-09-13T14:28:10.507641+0000","last_unstale":"2024-09-13T14:28:50.646708+0000","last_undegraded":"2024-09-13T14:28:50.646708+0000","last_fullsized":"2024-09-13T14:28:50.646708+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:37:05.617992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.588098+0000","last_change":"2024-09-13T14:28:10.510694+0000","last_active":"2024-09-13T14:28:51.588098+0000","last_peered":"2024-09-13T14:28:51.588098+0000","last_clean":"2024-09-13T14:28:51.588098+0000","last_became_active":"2024-09-13T14:28:10.510425+0000","last_became_peered":"2024-09-13T14:28:10.510425+0000","last_unstale":"2024-09-13T14:28:51.588098+0000","last_undegraded":"2024-09-13T14:28:51.588098+0000","last_fullsized":"2024-09-13T14:28:51.588098+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:55:23.655536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646012+0000","last_change":"2024-09-13T14:28:11.507252+0000","last_active":"2024-09-13T14:28:50.646012+0000","last_peered":"2024-09-13T14:28:50.646012+0000","last_clean":"2024-09-13T14:28:50.646012+0000","last_became_active":"2024-09-13T14:28:11.506972+0000","last_became_peered":"2024-09-13T14:28:11.506972+0000","last_unstale":"2024-09-13T14:28:50.646012+0000","last_undegraded":"2024-09-13T14:28:50.646012+0000","last_fullsized":"2024-09-13T14:28:50.646012+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:01.583699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.588126+0000","last_change":"2024-09-13T14:28:10.507693+0000","last_active":"2024-09-13T14:28:51.588126+0000","last_peered":"2024-09-13T14:28:51.588126+0000","last_clean":"2024-09-13T14:28:51.588126+0000","last_became_active":"2024-09-13T14:28:10.507437+0000","last_became_peered":"2024-09-13T14:28:10.507437+0000","last_unstale":"2024-09-13T14:28:51.588126+0000","last_undegraded":"2024-09-13T14:28:51.588126+0000","last_fullsized":"2024-09-13T14:28:51.588126+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:20:30.075633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:28:52.113107+0000","last_change":"2024-09-13T14:28:52.113107+0000","last_active":"2024-09-13T14:28:52.113107+0000","last_peered":"2024-09-13T14:28:52.113107+0000","last_clean":"2024-09-13T14:28:52.113107+0000","last_became_active":"2024-09-13T14:28:52.112842+0000","last_became_peered":"2024-09-13T14:28:52.112842+0000","last_unstale":"2024-09-13T14:28:52.113107+0000","last_undegraded":"2024-09-13T14:28:52.113107+0000","last_fullsized":"2024-09-13T14:28:52.113107+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:59:51.295506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.588004+0000","last_change":"2024-09-13T14:28:10.504588+0000","last_active":"2024-09-13T14:28:51.588004+0000","last_peered":"2024-09-13T14:28:51.588004+0000","last_clean":"2024-09-13T14:28:51.588004+0000","last_became_active":"2024-09-13T14:28:10.504338+0000","last_became_peered":"2024-09-13T14:28:10.504338+0000","last_unstale":"2024-09-13T14:28:51.588004+0000","last_undegraded":"2024-09-13T14:28:51.588004+0000","last_fullsized":"2024-09-13T14:28:51.588004+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:06:49.774993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646171+0000","last_change":"2024-09-13T14:28:11.508578+0000","last_active":"2024-09-13T14:28:50.646171+0000","last_peered":"2024-09-13T14:28:50.646171+0000","last_clean":"2024-09-13T14:28:50.646171+0000","last_became_active":"2024-09-13T14:28:11.508311+0000","last_became_peered":"2024-09-13T14:28:11.508311+0000","last_unstale":"2024-09-13T14:28:50.646171+0000","last_undegraded":"2024-09-13T14:28:50.646171+0000","last_fullsized":"2024-09-13T14:28:50.646171+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:47.732109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646276+0000","last_change":"2024-09-13T14:28:10.496693+0000","last_active":"2024-09-13T14:28:50.646276+0000","last_peered":"2024-09-13T14:28:50.646276+0000","last_clean":"2024-09-13T14:28:50.646276+0000","last_became_active":"2024-09-13T14:28:10.496424+0000","last_became_peered":"2024-09-13T14:28:10.496424+0000","last_unstale":"2024-09-13T14:28:50.646276+0000","last_undegraded":"2024-09-13T14:28:50.646276+0000","last_fullsized":"2024-09-13T14:28:50.646276+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:11.554436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587308+0000","last_change":"2024-09-13T14:28:11.531836+0000","last_active":"2024-09-13T14:28:51.587308+0000","last_peered":"2024-09-13T14:28:51.587308+0000","last_clean":"2024-09-13T14:28:51.587308+0000","last_became_active":"2024-09-13T14:28:11.531539+0000","last_became_peered":"2024-09-13T14:28:11.531539+0000","last_unstale":"2024-09-13T14:28:51.587308+0000","last_undegraded":"2024-09-13T14:28:51.587308+0000","last_fullsized":"2024-09-13T14:28:51.587308+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:46:34.698598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.648894+0000","last_change":"2024-09-13T14:28:10.500498+0000","last_active":"2024-09-13T14:28:50.648894+0000","last_peered":"2024-09-13T14:28:50.648894+0000","last_clean":"2024-09-13T14:28:50.648894+0000","last_became_active":"2024-09-13T14:28:10.500213+0000","last_became_peered":"2024-09-13T14:28:10.500213+0000","last_unstale":"2024-09-13T14:28:50.648894+0000","last_undegraded":"2024-09-13T14:28:50.648894+0000","last_fullsized":"2024-09-13T14:28:50.648894+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:08.352943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644060+0000","last_change":"2024-09-13T14:28:10.584320+0000","last_active":"2024-09-13T14:28:50.644060+0000","last_peered":"2024-09-13T14:28:50.644060+0000","last_clean":"2024-09-13T14:28:50.644060+0000","last_became_active":"2024-09-13T14:28:10.582950+0000","last_became_peered":"2024-09-13T14:28:10.582950+0000","last_unstale":"2024-09-13T14:28:50.644060+0000","last_undegraded":"2024-09-13T14:28:50.644060+0000","last_fullsized":"2024-09-13T14:28:50.644060+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:58:35.955340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646231+0000","last_change":"2024-09-13T14:28:10.508505+0000","last_active":"2024-09-13T14:28:50.646231+0000","last_peered":"2024-09-13T14:28:50.646231+0000","last_clean":"2024-09-13T14:28:50.646231+0000","last_became_active":"2024-09-13T14:28:10.508248+0000","last_became_peered":"2024-09-13T14:28:10.508248+0000","last_unstale":"2024-09-13T14:28:50.646231+0000","last_undegraded":"2024-09-13T14:28:50.646231+0000","last_fullsized":"2024-09-13T14:28:50.646231+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:51:45.033653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644718+0000","last_change":"2024-09-13T14:28:10.582336+0000","last_active":"2024-09-13T14:28:50.644718+0000","last_peered":"2024-09-13T14:28:50.644718+0000","last_clean":"2024-09-13T14:28:50.644718+0000","last_became_active":"2024-09-13T14:28:10.581923+0000","last_became_peered":"2024-09-13T14:28:10.581923+0000","last_unstale":"2024-09-13T14:28:50.644718+0000","last_undegraded":"2024-09-13T14:28:50.644718+0000","last_fullsized":"2024-09-13T14:28:50.644718+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:58.533266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644022+0000","last_change":"2024-09-13T14:28:10.504481+0000","last_active":"2024-09-13T14:28:50.644022+0000","last_peered":"2024-09-13T14:28:50.644022+0000","last_clean":"2024-09-13T14:28:50.644022+0000","last_became_active":"2024-09-13T14:28:10.504215+0000","last_became_peered":"2024-09-13T14:28:10.504215+0000","last_unstale":"2024-09-13T14:28:50.644022+0000","last_undegraded":"2024-09-13T14:28:50.644022+0000","last_fullsized":"2024-09-13T14:28:50.644022+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:27:01.645695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644246+0000","last_change":"2024-09-13T14:28:10.582606+0000","last_active":"2024-09-13T14:28:50.644246+0000","last_peered":"2024-09-13T14:28:50.644246+0000","last_clean":"2024-09-13T14:28:50.644246+0000","last_became_active":"2024-09-13T14:28:10.582316+0000","last_became_peered":"2024-09-13T14:28:10.582316+0000","last_unstale":"2024-09-13T14:28:50.644246+0000","last_undegraded":"2024-09-13T14:28:50.644246+0000","last_fullsized":"2024-09-13T14:28:50.644246+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:29.466745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.648972+0000","last_change":"2024-09-13T14:28:10.498813+0000","last_active":"2024-09-13T14:28:50.648972+0000","last_peered":"2024-09-13T14:28:50.648972+0000","last_clean":"2024-09-13T14:28:50.648972+0000","last_became_active":"2024-09-13T14:28:10.498529+0000","last_became_peered":"2024-09-13T14:28:10.498529+0000","last_unstale":"2024-09-13T14:28:50.648972+0000","last_undegraded":"2024-09-13T14:28:50.648972+0000","last_fullsized":"2024-09-13T14:28:50.648972+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:22:58.735920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.650800+0000","last_change":"2024-09-13T14:28:10.516027+0000","last_active":"2024-09-13T14:28:50.650800+0000","last_peered":"2024-09-13T14:28:50.650800+0000","last_clean":"2024-09-13T14:28:50.650800+0000","last_became_active":"2024-09-13T14:28:10.515758+0000","last_became_peered":"2024-09-13T14:28:10.515758+0000","last_unstale":"2024-09-13T14:28:50.650800+0000","last_undegraded":"2024-09-13T14:28:50.650800+0000","last_fullsized":"2024-09-13T14:28:50.650800+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:43.506170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"52'79","reported_seq":56,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587732+0000","last_change":"2024-09-13T14:26:20.041272+0000","last_active":"2024-09-13T14:28:51.587732+0000","last_peered":"2024-09-13T14:28:51.587732+0000","last_clean":"2024-09-13T14:28:51.587732+0000","last_became_active":"2024-09-13T14:26:20.035983+0000","last_became_peered":"2024-09-13T14:26:20.035983+0000","last_unstale":"2024-09-13T14:28:51.587732+0000","last_undegraded":"2024-09-13T14:28:51.587732+0000","last_fullsized":"2024-09-13T14:28:51.587732+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:24:20.875473+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:32:28.938482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":27,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646135+0000","last_change":"2024-09-13T14:27:01.130820+0000","last_active":"2024-09-13T14:28:50.646135+0000","last_peered":"2024-09-13T14:28:50.646135+0000","last_clean":"2024-09-13T14:28:50.646135+0000","last_became_active":"2024-09-13T14:26:56.113569+0000","last_became_peered":"2024-09-13T14:26:56.113569+0000","last_unstale":"2024-09-13T14:28:50.646135+0000","last_undegraded":"2024-09-13T14:28:50.646135+0000","last_fullsized":"2024-09-13T14:28:50.646135+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:47.705568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034224499999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649155+0000","last_change":"2024-09-13T14:28:10.507315+0000","last_active":"2024-09-13T14:28:50.649155+0000","last_peered":"2024-09-13T14:28:50.649155+0000","last_clean":"2024-09-13T14:28:50.649155+0000","last_became_active":"2024-09-13T14:28:10.507014+0000","last_became_peered":"2024-09-13T14:28:10.507014+0000","last_unstale":"2024-09-13T14:28:50.649155+0000","last_undegraded":"2024-09-13T14:28:50.649155+0000","last_fullsized":"2024-09-13T14:28:50.649155+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:12:03.810763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646533+0000","last_change":"2024-09-13T14:28:11.532509+0000","last_active":"2024-09-13T14:28:50.646533+0000","last_peered":"2024-09-13T14:28:50.646533+0000","last_clean":"2024-09-13T14:28:50.646533+0000","last_became_active":"2024-09-13T14:28:11.532207+0000","last_became_peered":"2024-09-13T14:28:11.532207+0000","last_unstale":"2024-09-13T14:28:50.646533+0000","last_undegraded":"2024-09-13T14:28:50.646533+0000","last_fullsized":"2024-09-13T14:28:50.646533+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:33.385182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587711+0000","last_change":"2024-09-13T14:27:01.908723+0000","last_active":"2024-09-13T14:28:51.587711+0000","last_peered":"2024-09-13T14:28:51.587711+0000","last_clean":"2024-09-13T14:28:51.587711+0000","last_became_active":"2024-09-13T14:26:56.115228+0000","last_became_peered":"2024-09-13T14:26:56.115228+0000","last_unstale":"2024-09-13T14:28:51.587711+0000","last_undegraded":"2024-09-13T14:28:51.587711+0000","last_fullsized":"2024-09-13T14:28:51.587711+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:48.775507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027767100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587419+0000","last_change":"2024-09-13T14:28:10.499750+0000","last_active":"2024-09-13T14:28:51.587419+0000","last_peered":"2024-09-13T14:28:51.587419+0000","last_clean":"2024-09-13T14:28:51.587419+0000","last_became_active":"2024-09-13T14:28:10.499453+0000","last_became_peered":"2024-09-13T14:28:10.499453+0000","last_unstale":"2024-09-13T14:28:51.587419+0000","last_undegraded":"2024-09-13T14:28:51.587419+0000","last_fullsized":"2024-09-13T14:28:51.587419+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:07:07.147783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587469+0000","last_change":"2024-09-13T14:28:11.508247+0000","last_active":"2024-09-13T14:28:51.587469+0000","last_peered":"2024-09-13T14:28:51.587469+0000","last_clean":"2024-09-13T14:28:51.587469+0000","last_became_active":"2024-09-13T14:28:11.507871+0000","last_became_peered":"2024-09-13T14:28:11.507871+0000","last_unstale":"2024-09-13T14:28:51.587469+0000","last_undegraded":"2024-09-13T14:28:51.587469+0000","last_fullsized":"2024-09-13T14:28:51.587469+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:17:56.989487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.645935+0000","last_change":"2024-09-13T14:27:01.127122+0000","last_active":"2024-09-13T14:28:50.645935+0000","last_peered":"2024-09-13T14:28:50.645935+0000","last_clean":"2024-09-13T14:28:50.645935+0000","last_became_active":"2024-09-13T14:26:56.178609+0000","last_became_peered":"2024-09-13T14:26:56.178609+0000","last_unstale":"2024-09-13T14:28:50.645935+0000","last_undegraded":"2024-09-13T14:28:50.645935+0000","last_fullsized":"2024-09-13T14:28:50.645935+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:23:16.906453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036252799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.648847+0000","last_change":"2024-09-13T14:28:10.495859+0000","last_active":"2024-09-13T14:28:50.648847+0000","last_peered":"2024-09-13T14:28:50.648847+0000","last_clean":"2024-09-13T14:28:50.648847+0000","last_became_active":"2024-09-13T14:28:10.495596+0000","last_became_peered":"2024-09-13T14:28:10.495596+0000","last_unstale":"2024-09-13T14:28:50.648847+0000","last_undegraded":"2024-09-13T14:28:50.648847+0000","last_fullsized":"2024-09-13T14:28:50.648847+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:00.760166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.650854+0000","last_change":"2024-09-13T14:28:10.511496+0000","last_active":"2024-09-13T14:28:50.650854+0000","last_peered":"2024-09-13T14:28:50.650854+0000","last_clean":"2024-09-13T14:28:50.650854+0000","last_became_active":"2024-09-13T14:28:10.511178+0000","last_became_peered":"2024-09-13T14:28:10.511178+0000","last_unstale":"2024-09-13T14:28:50.650854+0000","last_undegraded":"2024-09-13T14:28:50.650854+0000","last_fullsized":"2024-09-13T14:28:50.650854+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:17.860038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":28,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646833+0000","last_change":"2024-09-13T14:27:01.131429+0000","last_active":"2024-09-13T14:28:50.646833+0000","last_peered":"2024-09-13T14:28:50.646833+0000","last_clean":"2024-09-13T14:28:50.646833+0000","last_became_active":"2024-09-13T14:26:56.114214+0000","last_became_peered":"2024-09-13T14:26:56.114214+0000","last_unstale":"2024-09-13T14:28:50.646833+0000","last_undegraded":"2024-09-13T14:28:50.646833+0000","last_fullsized":"2024-09-13T14:28:50.646833+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:43.625300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00095382599999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644478+0000","last_change":"2024-09-13T14:28:10.506278+0000","last_active":"2024-09-13T14:28:50.644478+0000","last_peered":"2024-09-13T14:28:50.644478+0000","last_clean":"2024-09-13T14:28:50.644478+0000","last_became_active":"2024-09-13T14:28:10.506016+0000","last_became_peered":"2024-09-13T14:28:10.506016+0000","last_unstale":"2024-09-13T14:28:50.644478+0000","last_undegraded":"2024-09-13T14:28:50.644478+0000","last_fullsized":"2024-09-13T14:28:50.644478+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:32:02.776894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649699+0000","last_change":"2024-09-13T14:28:10.582166+0000","last_active":"2024-09-13T14:28:50.649699+0000","last_peered":"2024-09-13T14:28:50.649699+0000","last_clean":"2024-09-13T14:28:50.649699+0000","last_became_active":"2024-09-13T14:28:10.581870+0000","last_became_peered":"2024-09-13T14:28:10.581870+0000","last_unstale":"2024-09-13T14:28:50.649699+0000","last_undegraded":"2024-09-13T14:28:50.649699+0000","last_fullsized":"2024-09-13T14:28:50.649699+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:21.983145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646587+0000","last_change":"2024-09-13T14:28:10.496940+0000","last_active":"2024-09-13T14:28:50.646587+0000","last_peered":"2024-09-13T14:28:50.646587+0000","last_clean":"2024-09-13T14:28:50.646587+0000","last_became_active":"2024-09-13T14:28:10.495827+0000","last_became_peered":"2024-09-13T14:28:10.495827+0000","last_unstale":"2024-09-13T14:28:50.646587+0000","last_undegraded":"2024-09-13T14:28:50.646587+0000","last_fullsized":"2024-09-13T14:28:50.646587+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:29.695045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.650562+0000","last_change":"2024-09-13T14:27:01.127754+0000","last_active":"2024-09-13T14:28:50.650562+0000","last_peered":"2024-09-13T14:28:50.650562+0000","last_clean":"2024-09-13T14:28:50.650562+0000","last_became_active":"2024-09-13T14:26:56.112967+0000","last_became_peered":"2024-09-13T14:26:56.112967+0000","last_unstale":"2024-09-13T14:28:50.650562+0000","last_undegraded":"2024-09-13T14:28:50.650562+0000","last_fullsized":"2024-09-13T14:28:50.650562+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:40:20.095122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000367501,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649264+0000","last_change":"2024-09-13T14:28:10.516645+0000","last_active":"2024-09-13T14:28:50.649264+0000","last_peered":"2024-09-13T14:28:50.649264+0000","last_clean":"2024-09-13T14:28:50.649264+0000","last_became_active":"2024-09-13T14:28:10.516332+0000","last_became_peered":"2024-09-13T14:28:10.516332+0000","last_unstale":"2024-09-13T14:28:50.649264+0000","last_undegraded":"2024-09-13T14:28:50.649264+0000","last_fullsized":"2024-09-13T14:28:50.649264+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:51.371216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.650257+0000","last_change":"2024-09-13T14:28:10.491972+0000","last_active":"2024-09-13T14:28:50.650257+0000","last_peered":"2024-09-13T14:28:50.650257+0000","last_clean":"2024-09-13T14:28:50.650257+0000","last_became_active":"2024-09-13T14:28:10.491458+0000","last_became_peered":"2024-09-13T14:28:10.491458+0000","last_unstale":"2024-09-13T14:28:50.650257+0000","last_undegraded":"2024-09-13T14:28:50.650257+0000","last_fullsized":"2024-09-13T14:28:50.650257+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:54.166762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.588049+0000","last_change":"2024-09-13T14:27:01.908723+0000","last_active":"2024-09-13T14:28:51.588049+0000","last_peered":"2024-09-13T14:28:51.588049+0000","last_clean":"2024-09-13T14:28:51.588049+0000","last_became_active":"2024-09-13T14:26:56.124401+0000","last_became_peered":"2024-09-13T14:26:56.124401+0000","last_unstale":"2024-09-13T14:28:51.588049+0000","last_undegraded":"2024-09-13T14:28:51.588049+0000","last_fullsized":"2024-09-13T14:28:51.588049+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:46.437020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027405700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587304+0000","last_change":"2024-09-13T14:28:11.509145+0000","last_active":"2024-09-13T14:28:51.587304+0000","last_peered":"2024-09-13T14:28:51.587304+0000","last_clean":"2024-09-13T14:28:51.587304+0000","last_became_active":"2024-09-13T14:28:11.508833+0000","last_became_peered":"2024-09-13T14:28:11.508833+0000","last_unstale":"2024-09-13T14:28:51.587304+0000","last_undegraded":"2024-09-13T14:28:51.587304+0000","last_fullsized":"2024-09-13T14:28:51.587304+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:17:08.969691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649372+0000","last_change":"2024-09-13T14:28:10.500545+0000","last_active":"2024-09-13T14:28:50.649372+0000","last_peered":"2024-09-13T14:28:50.649372+0000","last_clean":"2024-09-13T14:28:50.649372+0000","last_became_active":"2024-09-13T14:28:10.500220+0000","last_became_peered":"2024-09-13T14:28:10.500220+0000","last_unstale":"2024-09-13T14:28:50.649372+0000","last_undegraded":"2024-09-13T14:28:50.649372+0000","last_fullsized":"2024-09-13T14:28:50.649372+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:33.274960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.650457+0000","last_change":"2024-09-13T14:27:01.127728+0000","last_active":"2024-09-13T14:28:50.650457+0000","last_peered":"2024-09-13T14:28:50.650457+0000","last_clean":"2024-09-13T14:28:50.650457+0000","last_became_active":"2024-09-13T14:26:56.124603+0000","last_became_peered":"2024-09-13T14:26:56.124603+0000","last_unstale":"2024-09-13T14:28:50.650457+0000","last_undegraded":"2024-09-13T14:28:50.650457+0000","last_fullsized":"2024-09-13T14:28:50.650457+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:46:33.019632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000373223,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.648804+0000","last_change":"2024-09-13T14:28:11.513073+0000","last_active":"2024-09-13T14:28:50.648804+0000","last_peered":"2024-09-13T14:28:50.648804+0000","last_clean":"2024-09-13T14:28:50.648804+0000","last_became_active":"2024-09-13T14:28:11.512821+0000","last_became_peered":"2024-09-13T14:28:11.512821+0000","last_unstale":"2024-09-13T14:28:50.648804+0000","last_undegraded":"2024-09-13T14:28:50.648804+0000","last_fullsized":"2024-09-13T14:28:50.648804+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:56.692654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587888+0000","last_change":"2024-09-13T14:28:10.497814+0000","last_active":"2024-09-13T14:28:51.587888+0000","last_peered":"2024-09-13T14:28:51.587888+0000","last_clean":"2024-09-13T14:28:51.587888+0000","last_became_active":"2024-09-13T14:28:10.497541+0000","last_became_peered":"2024-09-13T14:28:10.497541+0000","last_unstale":"2024-09-13T14:28:51.587888+0000","last_undegraded":"2024-09-13T14:28:51.587888+0000","last_fullsized":"2024-09-13T14:28:51.587888+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:14:56.511439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646454+0000","last_change":"2024-09-13T14:27:01.132344+0000","last_active":"2024-09-13T14:28:50.646454+0000","last_peered":"2024-09-13T14:28:50.646454+0000","last_clean":"2024-09-13T14:28:50.646454+0000","last_became_active":"2024-09-13T14:26:56.113467+0000","last_became_peered":"2024-09-13T14:26:56.113467+0000","last_unstale":"2024-09-13T14:28:50.646454+0000","last_undegraded":"2024-09-13T14:28:50.646454+0000","last_fullsized":"2024-09-13T14:28:50.646454+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:57:56.604631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023893800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.643876+0000","last_change":"2024-09-13T14:28:10.583355+0000","last_active":"2024-09-13T14:28:50.643876+0000","last_peered":"2024-09-13T14:28:50.643876+0000","last_clean":"2024-09-13T14:28:50.643876+0000","last_became_active":"2024-09-13T14:28:10.583032+0000","last_became_peered":"2024-09-13T14:28:10.583032+0000","last_unstale":"2024-09-13T14:28:50.643876+0000","last_undegraded":"2024-09-13T14:28:50.643876+0000","last_fullsized":"2024-09-13T14:28:50.643876+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:05:15.659274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644523+0000","last_change":"2024-09-13T14:28:10.583688+0000","last_active":"2024-09-13T14:28:50.644523+0000","last_peered":"2024-09-13T14:28:50.644523+0000","last_clean":"2024-09-13T14:28:50.644523+0000","last_became_active":"2024-09-13T14:28:10.583409+0000","last_became_peered":"2024-09-13T14:28:10.583409+0000","last_unstale":"2024-09-13T14:28:50.644523+0000","last_undegraded":"2024-09-13T14:28:50.644523+0000","last_fullsized":"2024-09-13T14:28:50.644523+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:33.587275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"64'6","reported_seq":20,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644168+0000","last_change":"2024-09-13T14:28:10.505107+0000","last_active":"2024-09-13T14:28:50.644168+0000","last_peered":"2024-09-13T14:28:50.644168+0000","last_clean":"2024-09-13T14:28:50.644168+0000","last_became_active":"2024-09-13T14:28:10.504857+0000","last_became_peered":"2024-09-13T14:28:10.504857+0000","last_unstale":"2024-09-13T14:28:50.644168+0000","last_undegraded":"2024-09-13T14:28:50.644168+0000","last_fullsized":"2024-09-13T14:28:50.644168+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:50:15.966787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649399+0000","last_change":"2024-09-13T14:28:10.582866+0000","last_active":"2024-09-13T14:28:50.649399+0000","last_peered":"2024-09-13T14:28:50.649399+0000","last_clean":"2024-09-13T14:28:50.649399+0000","last_became_active":"2024-09-13T14:28:10.582189+0000","last_became_peered":"2024-09-13T14:28:10.582189+0000","last_unstale":"2024-09-13T14:28:50.649399+0000","last_undegraded":"2024-09-13T14:28:50.649399+0000","last_fullsized":"2024-09-13T14:28:50.649399+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:20:53.470398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":17,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587961+0000","last_change":"2024-09-13T14:28:10.509946+0000","last_active":"2024-09-13T14:28:51.587961+0000","last_peered":"2024-09-13T14:28:51.587961+0000","last_clean":"2024-09-13T14:28:51.587961+0000","last_became_active":"2024-09-13T14:28:10.509689+0000","last_became_peered":"2024-09-13T14:28:10.509689+0000","last_unstale":"2024-09-13T14:28:51.587961+0000","last_undegraded":"2024-09-13T14:28:51.587961+0000","last_fullsized":"2024-09-13T14:28:51.587961+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:13.955928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646243+0000","last_change":"2024-09-13T14:28:10.498279+0000","last_active":"2024-09-13T14:28:50.646243+0000","last_peered":"2024-09-13T14:28:50.646243+0000","last_clean":"2024-09-13T14:28:50.646243+0000","last_became_active":"2024-09-13T14:28:10.498016+0000","last_became_peered":"2024-09-13T14:28:10.498016+0000","last_unstale":"2024-09-13T14:28:50.646243+0000","last_undegraded":"2024-09-13T14:28:50.646243+0000","last_fullsized":"2024-09-13T14:28:50.646243+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:04.417632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.648897+0000","last_change":"2024-09-13T14:28:11.512646+0000","last_active":"2024-09-13T14:28:50.648897+0000","last_peered":"2024-09-13T14:28:50.648897+0000","last_clean":"2024-09-13T14:28:50.648897+0000","last_became_active":"2024-09-13T14:28:11.512392+0000","last_became_peered":"2024-09-13T14:28:11.512392+0000","last_unstale":"2024-09-13T14:28:50.648897+0000","last_undegraded":"2024-09-13T14:28:50.648897+0000","last_fullsized":"2024-09-13T14:28:50.648897+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:09.381308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587975+0000","last_change":"2024-09-13T14:28:10.495999+0000","last_active":"2024-09-13T14:28:51.587975+0000","last_peered":"2024-09-13T14:28:51.587975+0000","last_clean":"2024-09-13T14:28:51.587975+0000","last_became_active":"2024-09-13T14:28:10.495736+0000","last_became_peered":"2024-09-13T14:28:10.495736+0000","last_unstale":"2024-09-13T14:28:51.587975+0000","last_undegraded":"2024-09-13T14:28:51.587975+0000","last_fullsized":"2024-09-13T14:28:51.587975+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:39.379161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.648699+0000","last_change":"2024-09-13T14:28:11.512381+0000","last_active":"2024-09-13T14:28:50.648699+0000","last_peered":"2024-09-13T14:28:50.648699+0000","last_clean":"2024-09-13T14:28:50.648699+0000","last_became_active":"2024-09-13T14:28:11.512031+0000","last_became_peered":"2024-09-13T14:28:11.512031+0000","last_unstale":"2024-09-13T14:28:50.648699+0000","last_undegraded":"2024-09-13T14:28:50.648699+0000","last_fullsized":"2024-09-13T14:28:50.648699+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:56.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646379+0000","last_change":"2024-09-13T14:28:11.507361+0000","last_active":"2024-09-13T14:28:50.646379+0000","last_peered":"2024-09-13T14:28:50.646379+0000","last_clean":"2024-09-13T14:28:50.646379+0000","last_became_active":"2024-09-13T14:28:11.507077+0000","last_became_peered":"2024-09-13T14:28:11.507077+0000","last_unstale":"2024-09-13T14:28:50.646379+0000","last_undegraded":"2024-09-13T14:28:50.646379+0000","last_fullsized":"2024-09-13T14:28:50.646379+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:33.602421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.648897+0000","last_change":"2024-09-13T14:28:10.501264+0000","last_active":"2024-09-13T14:28:50.648897+0000","last_peered":"2024-09-13T14:28:50.648897+0000","last_clean":"2024-09-13T14:28:50.648897+0000","last_became_active":"2024-09-13T14:28:10.500188+0000","last_became_peered":"2024-09-13T14:28:10.500188+0000","last_unstale":"2024-09-13T14:28:50.648897+0000","last_undegraded":"2024-09-13T14:28:50.648897+0000","last_fullsized":"2024-09-13T14:28:50.648897+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:46:53.209853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587575+0000","last_change":"2024-09-13T14:28:11.513490+0000","last_active":"2024-09-13T14:28:51.587575+0000","last_peered":"2024-09-13T14:28:51.587575+0000","last_clean":"2024-09-13T14:28:51.587575+0000","last_became_active":"2024-09-13T14:28:11.513180+0000","last_became_peered":"2024-09-13T14:28:11.513180+0000","last_unstale":"2024-09-13T14:28:51.587575+0000","last_undegraded":"2024-09-13T14:28:51.587575+0000","last_fullsized":"2024-09-13T14:28:51.587575+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:06:31.355641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649424+0000","last_change":"2024-09-13T14:28:10.499527+0000","last_active":"2024-09-13T14:28:50.649424+0000","last_peered":"2024-09-13T14:28:50.649424+0000","last_clean":"2024-09-13T14:28:50.649424+0000","last_became_active":"2024-09-13T14:28:10.499202+0000","last_became_peered":"2024-09-13T14:28:10.499202+0000","last_unstale":"2024-09-13T14:28:50.649424+0000","last_undegraded":"2024-09-13T14:28:50.649424+0000","last_fullsized":"2024-09-13T14:28:50.649424+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:21.301397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:28:52.112821+0000","last_change":"2024-09-13T14:28:52.112821+0000","last_active":"2024-09-13T14:28:52.112821+0000","last_peered":"2024-09-13T14:28:52.112821+0000","last_clean":"2024-09-13T14:28:52.112821+0000","last_became_active":"2024-09-13T14:28:52.112591+0000","last_became_peered":"2024-09-13T14:28:52.112591+0000","last_unstale":"2024-09-13T14:28:52.112821+0000","last_undegraded":"2024-09-13T14:28:52.112821+0000","last_fullsized":"2024-09-13T14:28:52.112821+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:05:06.365986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644028+0000","last_change":"2024-09-13T14:28:10.497427+0000","last_active":"2024-09-13T14:28:50.644028+0000","last_peered":"2024-09-13T14:28:50.644028+0000","last_clean":"2024-09-13T14:28:50.644028+0000","last_became_active":"2024-09-13T14:28:10.497149+0000","last_became_peered":"2024-09-13T14:28:10.497149+0000","last_unstale":"2024-09-13T14:28:50.644028+0000","last_undegraded":"2024-09-13T14:28:50.644028+0000","last_fullsized":"2024-09-13T14:28:50.644028+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:00:25.254819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646024+0000","last_change":"2024-09-13T14:28:11.514186+0000","last_active":"2024-09-13T14:28:50.646024+0000","last_peered":"2024-09-13T14:28:50.646024+0000","last_clean":"2024-09-13T14:28:50.646024+0000","last_became_active":"2024-09-13T14:28:11.513858+0000","last_became_peered":"2024-09-13T14:28:11.513858+0000","last_unstale":"2024-09-13T14:28:50.646024+0000","last_undegraded":"2024-09-13T14:28:50.646024+0000","last_fullsized":"2024-09-13T14:28:50.646024+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:53:10.642897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587595+0000","last_change":"2024-09-13T14:28:10.498999+0000","last_active":"2024-09-13T14:28:51.587595+0000","last_peered":"2024-09-13T14:28:51.587595+0000","last_clean":"2024-09-13T14:28:51.587595+0000","last_became_active":"2024-09-13T14:28:10.498736+0000","last_became_peered":"2024-09-13T14:28:10.498736+0000","last_unstale":"2024-09-13T14:28:51.587595+0000","last_undegraded":"2024-09-13T14:28:51.587595+0000","last_fullsized":"2024-09-13T14:28:51.587595+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:33.684548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644324+0000","last_change":"2024-09-13T14:28:10.517704+0000","last_active":"2024-09-13T14:28:50.644324+0000","last_peered":"2024-09-13T14:28:50.644324+0000","last_clean":"2024-09-13T14:28:50.644324+0000","last_became_active":"2024-09-13T14:28:10.517400+0000","last_became_peered":"2024-09-13T14:28:10.517400+0000","last_unstale":"2024-09-13T14:28:50.644324+0000","last_undegraded":"2024-09-13T14:28:50.644324+0000","last_fullsized":"2024-09-13T14:28:50.644324+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:01.199610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646194+0000","last_change":"2024-09-13T14:28:10.494883+0000","last_active":"2024-09-13T14:28:50.646194+0000","last_peered":"2024-09-13T14:28:50.646194+0000","last_clean":"2024-09-13T14:28:50.646194+0000","last_became_active":"2024-09-13T14:28:10.494623+0000","last_became_peered":"2024-09-13T14:28:10.494623+0000","last_unstale":"2024-09-13T14:28:50.646194+0000","last_undegraded":"2024-09-13T14:28:50.646194+0000","last_fullsized":"2024-09-13T14:28:50.646194+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:17.878113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.643925+0000","last_change":"2024-09-13T14:28:10.515419+0000","last_active":"2024-09-13T14:28:50.643925+0000","last_peered":"2024-09-13T14:28:50.643925+0000","last_clean":"2024-09-13T14:28:50.643925+0000","last_became_active":"2024-09-13T14:28:10.515214+0000","last_became_peered":"2024-09-13T14:28:10.515214+0000","last_unstale":"2024-09-13T14:28:50.643925+0000","last_undegraded":"2024-09-13T14:28:50.643925+0000","last_fullsized":"2024-09-13T14:28:50.643925+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:36:45.422984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646513+0000","last_change":"2024-09-13T14:28:10.500736+0000","last_active":"2024-09-13T14:28:50.646513+0000","last_peered":"2024-09-13T14:28:50.646513+0000","last_clean":"2024-09-13T14:28:50.646513+0000","last_became_active":"2024-09-13T14:28:10.500467+0000","last_became_peered":"2024-09-13T14:28:10.500467+0000","last_unstale":"2024-09-13T14:28:50.646513+0000","last_undegraded":"2024-09-13T14:28:50.646513+0000","last_fullsized":"2024-09-13T14:28:50.646513+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:19.688685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649092+0000","last_change":"2024-09-13T14:28:11.513626+0000","last_active":"2024-09-13T14:28:50.649092+0000","last_peered":"2024-09-13T14:28:50.649092+0000","last_clean":"2024-09-13T14:28:50.649092+0000","last_became_active":"2024-09-13T14:28:11.513267+0000","last_became_peered":"2024-09-13T14:28:11.513267+0000","last_unstale":"2024-09-13T14:28:50.649092+0000","last_undegraded":"2024-09-13T14:28:50.649092+0000","last_fullsized":"2024-09-13T14:28:50.649092+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:02:52.487064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649089+0000","last_change":"2024-09-13T14:28:10.504329+0000","last_active":"2024-09-13T14:28:50.649089+0000","last_peered":"2024-09-13T14:28:50.649089+0000","last_clean":"2024-09-13T14:28:50.649089+0000","last_became_active":"2024-09-13T14:28:10.504074+0000","last_became_peered":"2024-09-13T14:28:10.504074+0000","last_unstale":"2024-09-13T14:28:50.649089+0000","last_undegraded":"2024-09-13T14:28:50.649089+0000","last_fullsized":"2024-09-13T14:28:50.649089+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:03:47.723724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:28:52.557448+0000","last_change":"2024-09-13T14:28:52.557448+0000","last_active":"2024-09-13T14:28:52.557448+0000","last_peered":"2024-09-13T14:28:52.557448+0000","last_clean":"2024-09-13T14:28:52.557448+0000","last_became_active":"2024-09-13T14:28:52.557090+0000","last_became_peered":"2024-09-13T14:28:52.557090+0000","last_unstale":"2024-09-13T14:28:52.557448+0000","last_undegraded":"2024-09-13T14:28:52.557448+0000","last_fullsized":"2024-09-13T14:28:52.557448+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:59.236430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587950+0000","last_change":"2024-09-13T14:28:10.496380+0000","last_active":"2024-09-13T14:28:51.587950+0000","last_peered":"2024-09-13T14:28:51.587950+0000","last_clean":"2024-09-13T14:28:51.587950+0000","last_became_active":"2024-09-13T14:28:10.494751+0000","last_became_peered":"2024-09-13T14:28:10.494751+0000","last_unstale":"2024-09-13T14:28:51.587950+0000","last_undegraded":"2024-09-13T14:28:51.587950+0000","last_fullsized":"2024-09-13T14:28:51.587950+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:34:56.137599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397541,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298624,"kb_used_data":1264,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446528,"statfs":{"total":95995035648,"available":95689244672,"internally_reserved":0,"allocated":1294336,"data_stored":888643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57099999999999995,"15min":0.57599999999999996},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.67700000000000005,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.55200000000000005,"15min":0.55800000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.72199999999999998,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.51300000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.50800000000000001,"15min":0.51000000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.495},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.53500000000000003,"15min":0.53700000000000003},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.71499999999999997,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.63500000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.56299999999999994,"15min":0.56999999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.69999999999999996,"5min":1.0880000000000001,"15min":1.0880000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.38300000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.81799999999999995,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.55000000000000004}]},{"osd":3,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56899999999999995,"15min":0.57599999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.76900000000000002,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.44},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.56499999999999995,"15min":0.57099999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.69299999999999995,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.64900000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.49199999999999999,"15min":0.49099999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.80100000000000005,"5min":1.45,"15min":1.45},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.32800000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.79100000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.621}]},{"osd":5,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.47399999999999998,"15min":0.46800000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.68799999999999994,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.35799999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.73199999999999998,"5min":1.421,"15min":1.421},"last":0.66000000000000003}]},{"osd":6,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.53500000000000003,"15min":0.53300000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.79700000000000004,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.61799999999999999,"15min":0.622},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.82699999999999996,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.69799999999999995}]}]},{"osd":6,"up_from":43,"seq":184683593770,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298612,"kb_used_data":1252,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446540,"statfs":{"total":95995035648,"available":95689256960,"internally_reserved":0,"allocated":1282048,"data_stored":888201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.52500000000000002,"15min":0.51700000000000002},"min":{"1min":0.34799999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.73599999999999999,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.69699999999999995,"15min":0.58699999999999997},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.77300000000000002,"5min":17.888000000000002,"15min":17.888000000000002},"last":0.65600000000000003}]},{"osd":1,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.52900000000000003,"15min":0.53300000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39200000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.69899999999999995,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.50800000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51200000000000001,"15min":0.51300000000000001},"min":{"1min":0.40400000000000003,"5min":0.379,"15min":0.379},"max":{"1min":0.71099999999999997,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.55900000000000005,"15min":0.56599999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.83699999999999997,"5min":0.874,"15min":0.874},"last":0.52000000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.626,"15min":0.56000000000000005},"min":{"1min":0.376,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.412000000000001,"5min":10.412000000000001,"15min":10.412000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.70799999999999996,"15min":0.60299999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.753,"5min":17.952999999999999,"15min":17.952999999999999},"last":0.497}]},{"osd":4,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.45400000000000001,"15min":0.45500000000000002},"min":{"1min":0.35099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.45300000000000001,"15min":0.44800000000000001},"min":{"1min":0.29999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.76400000000000001,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.34699999999999998}]},{"osd":5,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.44900000000000001,"15min":0.443},"min":{"1min":0.32700000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.61899999999999999,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.504,"15min":0.503},"min":{"1min":0.33300000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.373}]},{"osd":7,"last update":"Fri Sep 13 14:28:18 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54500000000000004,"15min":0.54700000000000004},"min":{"1min":0.36299999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.80300000000000005,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.51300000000000001,"15min":0.51000000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.79100000000000004,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.39600000000000002}]}]},{"osd":5,"up_from":37,"seq":158913789999,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298024,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447128,"statfs":{"total":95995035648,"available":95689859072,"internally_reserved":0,"allocated":679936,"data_stored":296922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:28:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.53600000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.52200000000000002,"15min":0.52000000000000002},"min":{"1min":0.371,"5min":0.311,"15min":0.311},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.64400000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:28:34 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51500000000000001,"15min":0.51800000000000002},"min":{"1min":0.34100000000000003,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.68600000000000005,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52800000000000002,"15min":0.53500000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.69099999999999995,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.55300000000000005}]},{"osd":2,"last update":"Fri Sep 13 14:28:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52500000000000002,"15min":0.53100000000000003},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.64300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.52500000000000002,"15min":0.52600000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.69899999999999995,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.67200000000000004}]},{"osd":3,"last update":"Fri Sep 13 14:28:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52800000000000002,"15min":0.53300000000000003},"min":{"1min":0.35399999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.67900000000000005,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.53200000000000003,"15min":0.53400000000000003},"min":{"1min":0.35899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.63}]},{"osd":4,"last update":"Fri Sep 13 14:28:34 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.376,"15min":0.35999999999999999},"min":{"1min":0.26900000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.65700000000000003,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.496},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.39000000000000001,"15min":0.372},"min":{"1min":0.28599999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.438}]},{"osd":6,"last update":"Fri Sep 13 14:28:57 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.44,"15min":0.42599999999999999},"min":{"1min":0.313,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.67300000000000004,"5min":0.747,"15min":0.747},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.41999999999999998,"15min":0.40899999999999997},"min":{"1min":0.30399999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.70099999999999996,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.47699999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:28:24 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.44900000000000001,"15min":0.45100000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.70599999999999996,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.47999999999999998,"15min":0.47899999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.58199999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986228,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298040,"kb_used_data":680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447112,"statfs":{"total":95995035648,"available":95689842688,"internally_reserved":0,"allocated":696320,"data_stored":298205,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:28:11 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.53200000000000003,"15min":0.55600000000000005},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":0.66800000000000004,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.436},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.62,"15min":0.56299999999999994},"min":{"1min":0.35799999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.6240000000000006,"5min":9.6240000000000006,"15min":9.6240000000000006},"last":0.497}]},{"osd":1,"last update":"Fri Sep 13 14:28:11 2024","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":0.66500000000000004,"15min":0.57499999999999996},"min":{"1min":0.32600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.5850000000000009,"5min":9.5850000000000009,"15min":9.5850000000000009},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.53900000000000003,"15min":0.55700000000000005},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.65300000000000002,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.51200000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:28:11 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.52900000000000003,"15min":0.54000000000000004},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.69799999999999995,"5min":0.745,"15min":0.745},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.53200000000000003,"15min":0.53800000000000003},"min":{"1min":0.34200000000000003,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.68600000000000005,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.46600000000000003}]},{"osd":3,"last update":"Fri Sep 13 14:28:11 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.52300000000000002,"15min":0.53000000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.68600000000000005,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.498,"5min":0.53900000000000003,"15min":0.55100000000000005},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.71899999999999997,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.54700000000000004}]},{"osd":5,"last update":"Fri Sep 13 14:28:36 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.39000000000000001,"15min":0.38200000000000001},"min":{"1min":0.255,"5min":0.219,"15min":0.219},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.38500000000000001,"15min":0.377},"min":{"1min":0.28799999999999998,"5min":0.217,"15min":0.217},"max":{"1min":0.54400000000000004,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.35499999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:28:58 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.46200000000000002,"15min":0.44900000000000001},"min":{"1min":0.30399999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.45600000000000002,"15min":0.442},"min":{"1min":0.35399999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.57299999999999995}]},{"osd":7,"last update":"Fri Sep 13 14:28:19 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.46700000000000003,"15min":0.46999999999999997},"min":{"1min":0.33200000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.626,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.502,"5min":0.51300000000000001,"15min":0.51500000000000001},"min":{"1min":0.35499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.67600000000000005,"5min":0.745,"15min":0.745},"last":0.41999999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149753,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298016,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447136,"statfs":{"total":95995035648,"available":95689867264,"internally_reserved":0,"allocated":671744,"data_stored":296353,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:28:52 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.38900000000000001,"15min":0.41499999999999998},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.50600000000000001,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.247},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.371,"15min":0.38700000000000001},"min":{"1min":0.23000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.51900000000000002,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.23200000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:28:52 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.40300000000000002,"15min":0.41999999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.55200000000000005,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.376,"5min":0.38500000000000001,"15min":0.39900000000000002},"min":{"1min":0.24299999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.52100000000000002,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.34899999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:28:52 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.42499999999999999,"15min":0.44700000000000001},"min":{"1min":0.26600000000000001,"5min":0.248,"15min":0.248},"max":{"1min":0.54000000000000004,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.38,"5min":0.40100000000000002,"15min":0.42499999999999999},"min":{"1min":0.25,"5min":0.22,"15min":0.22},"max":{"1min":0.51800000000000002,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.30199999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:28:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.59399999999999997,"15min":0.57699999999999996},"min":{"1min":0.33300000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.80800000000000005,"5min":4.3979999999999997,"15min":4.3979999999999997},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.624,"15min":0.59399999999999997},"min":{"1min":0.38700000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81100000000000005,"5min":4.4660000000000002,"15min":4.4660000000000002},"last":0.59799999999999998}]},{"osd":5,"last update":"Fri Sep 13 14:28:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.67800000000000005,"15min":0.72999999999999998},"min":{"1min":0.37,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.78900000000000003,"5min":4.4139999999999997,"15min":4.4139999999999997},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.67300000000000004,"15min":0.72199999999999998},"min":{"1min":0.38400000000000001,"5min":0.312,"15min":0.312},"max":{"1min":0.81000000000000005,"5min":4.3799999999999999,"15min":4.3799999999999999},"last":0.58099999999999996}]},{"osd":6,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.70999999999999996,"15min":0.76000000000000001},"min":{"1min":0.42899999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.22,"5min":4.4329999999999998,"15min":4.4329999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.55700000000000005,"15min":0.55100000000000005},"min":{"1min":0.42099999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.78900000000000003,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.51500000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:28:23 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.52000000000000002,"15min":0.50900000000000001},"min":{"1min":0.442,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.55600000000000005,"15min":0.55000000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.56000000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411390,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298020,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447132,"statfs":{"total":95995035648,"available":95689863168,"internally_reserved":0,"allocated":675840,"data_stored":296867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.34200000000000003,"15min":0.33700000000000002},"min":{"1min":0.20200000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.53100000000000003,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.32700000000000001,"5min":0.36399999999999999,"15min":0.36199999999999999},"min":{"1min":0.151,"5min":0.151,"15min":0.151},"max":{"1min":0.51100000000000001,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.379}]},{"osd":1,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.36299999999999999,"15min":0.36099999999999999},"min":{"1min":0.24099999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.52600000000000002,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.505},{"interface":"front","average":{"1min":0.33400000000000002,"5min":0.38300000000000001,"15min":0.38600000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.55000000000000004,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.31}]},{"osd":3,"last update":"Fri Sep 13 14:28:49 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.40500000000000003,"15min":0.41299999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.41999999999999998,"15min":0.42799999999999999},"min":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.45700000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:28:10 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.52600000000000002,"15min":0.52300000000000002},"min":{"1min":0.38900000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.77300000000000002,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.59299999999999997}]},{"osd":5,"last update":"Fri Sep 13 14:28:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.53100000000000003,"15min":0.53800000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.73699999999999999,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.755,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.76600000000000001}]},{"osd":6,"last update":"Fri Sep 13 14:29:00 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.59099999999999997,"15min":0.58699999999999997},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.204,"5min":1.204,"15min":1.204},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.57399999999999995,"15min":0.57799999999999996},"min":{"1min":0.39500000000000002,"5min":0.374,"15min":0.374},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.63800000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.54300000000000004,"15min":0.54100000000000004},"min":{"1min":0.41799999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.57899999999999996,"15min":0.58499999999999996},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.73199999999999998,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.72199999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574915,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298020,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447132,"statfs":{"total":95995035648,"available":95689863168,"internally_reserved":0,"allocated":675840,"data_stored":296526,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.34599999999999997,"15min":0.30199999999999999},"min":{"1min":0.23899999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.46899999999999997,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.33400000000000002,"5min":0.371,"15min":0.32300000000000001},"min":{"1min":0.21299999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.53700000000000003,"5min":1.198,"15min":1.198},"last":0.309}]},{"osd":2,"last update":"Fri Sep 13 14:28:28 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.379,"15min":0.36399999999999999},"min":{"1min":0.24399999999999999,"5min":0.223,"15min":0.223},"max":{"1min":0.53100000000000003,"5min":1.0800000000000001,"15min":1.0800000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.36699999999999999,"15min":0.35099999999999998},"min":{"1min":0.254,"5min":0.216,"15min":0.216},"max":{"1min":0.51300000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.27600000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:28:54 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.41999999999999998,"15min":0.436},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.63400000000000001,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.40100000000000002,"15min":0.40899999999999997},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.51800000000000002,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.36899999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:28:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.54700000000000004,"15min":0.54000000000000004},"min":{"1min":0.38900000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.55000000000000004,"15min":0.54300000000000004},"min":{"1min":0.38400000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.80200000000000005,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.66600000000000004}]},{"osd":5,"last update":"Fri Sep 13 14:28:33 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.57899999999999996,"15min":0.58199999999999996},"min":{"1min":0.40000000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79200000000000004,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.59099999999999997,"15min":0.58699999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.49099999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:28:57 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.59299999999999997,"15min":0.58699999999999997},"min":{"1min":0.371,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76900000000000002,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.51600000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:28:22 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.59099999999999997,"15min":0.58599999999999997},"min":{"1min":0.41999999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.78400000000000003,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.59399999999999997,"15min":0.59499999999999997},"min":{"1min":0.39300000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80200000000000005,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.61499999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298604,"kb_used_data":1244,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446548,"statfs":{"total":95995035648,"available":95689265152,"internally_reserved":0,"allocated":1273856,"data_stored":887467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:29:03 2024","interfaces":[{"interface":"back","average":{"1min":0.32500000000000001,"5min":0.33200000000000002,"15min":0.317},"min":{"1min":0.223,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.41899999999999998,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.371},{"interface":"front","average":{"1min":0.32800000000000001,"5min":0.34300000000000003,"15min":0.315},"min":{"1min":0.23400000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.44,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.28199999999999997}]},{"osd":2,"last update":"Fri Sep 13 14:28:28 2024","interfaces":[{"interface":"back","average":{"1min":0.34100000000000003,"5min":0.36399999999999999,"15min":0.35399999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.48599999999999999,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.32900000000000001,"5min":0.35199999999999998,"15min":0.34699999999999998},"min":{"1min":0.23899999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.42899999999999999,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.33700000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:28:53 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.39500000000000002,"15min":0.40400000000000003},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.46999999999999997,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.33700000000000002,"5min":0.376,"15min":0.39600000000000002},"min":{"1min":0.252,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.52400000000000002,"5min":0.627,"15min":0.627},"last":0.42699999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:28:10 2024","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.58499999999999996,"15min":0.55700000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.5510000000000002,"5min":5.5510000000000002,"15min":5.5510000000000002},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.54900000000000004,"15min":0.55700000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.71499999999999997,"5min":0.755,"15min":0.755},"last":0.57699999999999996}]},{"osd":5,"last update":"Fri Sep 13 14:28:31 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.55100000000000005,"15min":0.55300000000000005},"min":{"1min":0.40600000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.76700000000000002,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.38100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.56699999999999995}]},{"osd":6,"last update":"Fri Sep 13 14:28:58 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.38900000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.83099999999999996},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.56299999999999994,"15min":0.57499999999999996},"min":{"1min":0.35999999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.75700000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.60199999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:28:19 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.4350000000000001,"15min":1.581},"min":{"1min":0.45700000000000002,"5min":0.378,"15min":0.378},"max":{"1min":0.70499999999999996,"5min":19.390000000000001,"15min":19.390000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.56000000000000005,"15min":0.56499999999999995},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.747,"5min":0.748,"15min":0.748},"last":0.63900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:29:09.718 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:29:09.963 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:09.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:09 smithi204 ceph-mon[32546]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:09.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:09 smithi204 ceph-mon[32546]: from='client.24668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:29:10.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:09 smithi096 ceph-mon[29229]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:10.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:09 smithi096 ceph-mon[29229]: from='client.24668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:29:10.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:09 smithi096 ceph-mon[39645]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:10.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:09 smithi096 ceph-mon[39645]: from='client.24668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:29:11.763 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:29:11.764 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:29:11.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:11 smithi204 ceph-mon[32546]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:12.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:11 smithi096 ceph-mon[39645]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:12.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:11 smithi096 ceph-mon[29229]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:12.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:29:12 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:29:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:29:12.531 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":87,"stamp":"2024-09-13T14:29:10.557408+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592725,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":160,"num_write_kb":1643,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121,"ondisk_log_size":121,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385960,"kb_used_data":7080,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575256,"statfs":{"total":767960285184,"available":765517062144,"internally_reserved":0,"allocated":7249920,"data_stored":4149184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649338+0000","last_change":"2024-09-13T14:28:11.512308+0000","last_active":"2024-09-13T14:28:50.649338+0000","last_peered":"2024-09-13T14:28:50.649338+0000","last_clean":"2024-09-13T14:28:50.649338+0000","last_became_active":"2024-09-13T14:28:11.512003+0000","last_became_peered":"2024-09-13T14:28:11.512003+0000","last_unstale":"2024-09-13T14:28:50.649338+0000","last_undegraded":"2024-09-13T14:28:50.649338+0000","last_fullsized":"2024-09-13T14:28:50.649338+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:51:00.952944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587897+0000","last_change":"2024-09-13T14:28:10.508904+0000","last_active":"2024-09-13T14:28:51.587897+0000","last_peered":"2024-09-13T14:28:51.587897+0000","last_clean":"2024-09-13T14:28:51.587897+0000","last_became_active":"2024-09-13T14:28:10.508638+0000","last_became_peered":"2024-09-13T14:28:10.508638+0000","last_unstale":"2024-09-13T14:28:51.587897+0000","last_undegraded":"2024-09-13T14:28:51.587897+0000","last_fullsized":"2024-09-13T14:28:51.587897+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:22.870537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646422+0000","last_change":"2024-09-13T14:28:11.512154+0000","last_active":"2024-09-13T14:28:50.646422+0000","last_peered":"2024-09-13T14:28:50.646422+0000","last_clean":"2024-09-13T14:28:50.646422+0000","last_became_active":"2024-09-13T14:28:11.511903+0000","last_became_peered":"2024-09-13T14:28:11.511903+0000","last_unstale":"2024-09-13T14:28:50.646422+0000","last_undegraded":"2024-09-13T14:28:50.646422+0000","last_fullsized":"2024-09-13T14:28:50.646422+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:34.544812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587446+0000","last_change":"2024-09-13T14:28:10.496527+0000","last_active":"2024-09-13T14:28:51.587446+0000","last_peered":"2024-09-13T14:28:51.587446+0000","last_clean":"2024-09-13T14:28:51.587446+0000","last_became_active":"2024-09-13T14:28:10.496273+0000","last_became_peered":"2024-09-13T14:28:10.496273+0000","last_unstale":"2024-09-13T14:28:51.587446+0000","last_undegraded":"2024-09-13T14:28:51.587446+0000","last_fullsized":"2024-09-13T14:28:51.587446+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:30:48.755762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587354+0000","last_change":"2024-09-13T14:28:11.512769+0000","last_active":"2024-09-13T14:28:51.587354+0000","last_peered":"2024-09-13T14:28:51.587354+0000","last_clean":"2024-09-13T14:28:51.587354+0000","last_became_active":"2024-09-13T14:28:11.512391+0000","last_became_peered":"2024-09-13T14:28:11.512391+0000","last_unstale":"2024-09-13T14:28:51.587354+0000","last_undegraded":"2024-09-13T14:28:51.587354+0000","last_fullsized":"2024-09-13T14:28:51.587354+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:06:17.869350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587782+0000","last_change":"2024-09-13T14:28:10.508894+0000","last_active":"2024-09-13T14:28:51.587782+0000","last_peered":"2024-09-13T14:28:51.587782+0000","last_clean":"2024-09-13T14:28:51.587782+0000","last_became_active":"2024-09-13T14:28:10.508612+0000","last_became_peered":"2024-09-13T14:28:10.508612+0000","last_unstale":"2024-09-13T14:28:51.587782+0000","last_undegraded":"2024-09-13T14:28:51.587782+0000","last_fullsized":"2024-09-13T14:28:51.587782+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:01.774646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649251+0000","last_change":"2024-09-13T14:28:11.511942+0000","last_active":"2024-09-13T14:28:50.649251+0000","last_peered":"2024-09-13T14:28:50.649251+0000","last_clean":"2024-09-13T14:28:50.649251+0000","last_became_active":"2024-09-13T14:28:11.511494+0000","last_became_peered":"2024-09-13T14:28:11.511494+0000","last_unstale":"2024-09-13T14:28:50.649251+0000","last_undegraded":"2024-09-13T14:28:50.649251+0000","last_fullsized":"2024-09-13T14:28:50.649251+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:55.285495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":22,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646708+0000","last_change":"2024-09-13T14:28:10.507948+0000","last_active":"2024-09-13T14:28:50.646708+0000","last_peered":"2024-09-13T14:28:50.646708+0000","last_clean":"2024-09-13T14:28:50.646708+0000","last_became_active":"2024-09-13T14:28:10.507641+0000","last_became_peered":"2024-09-13T14:28:10.507641+0000","last_unstale":"2024-09-13T14:28:50.646708+0000","last_undegraded":"2024-09-13T14:28:50.646708+0000","last_fullsized":"2024-09-13T14:28:50.646708+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:37:05.617992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.588098+0000","last_change":"2024-09-13T14:28:10.510694+0000","last_active":"2024-09-13T14:28:51.588098+0000","last_peered":"2024-09-13T14:28:51.588098+0000","last_clean":"2024-09-13T14:28:51.588098+0000","last_became_active":"2024-09-13T14:28:10.510425+0000","last_became_peered":"2024-09-13T14:28:10.510425+0000","last_unstale":"2024-09-13T14:28:51.588098+0000","last_undegraded":"2024-09-13T14:28:51.588098+0000","last_fullsized":"2024-09-13T14:28:51.588098+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:55:23.655536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646012+0000","last_change":"2024-09-13T14:28:11.507252+0000","last_active":"2024-09-13T14:28:50.646012+0000","last_peered":"2024-09-13T14:28:50.646012+0000","last_clean":"2024-09-13T14:28:50.646012+0000","last_became_active":"2024-09-13T14:28:11.506972+0000","last_became_peered":"2024-09-13T14:28:11.506972+0000","last_unstale":"2024-09-13T14:28:50.646012+0000","last_undegraded":"2024-09-13T14:28:50.646012+0000","last_fullsized":"2024-09-13T14:28:50.646012+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:01.583699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.588126+0000","last_change":"2024-09-13T14:28:10.507693+0000","last_active":"2024-09-13T14:28:51.588126+0000","last_peered":"2024-09-13T14:28:51.588126+0000","last_clean":"2024-09-13T14:28:51.588126+0000","last_became_active":"2024-09-13T14:28:10.507437+0000","last_became_peered":"2024-09-13T14:28:10.507437+0000","last_unstale":"2024-09-13T14:28:51.588126+0000","last_undegraded":"2024-09-13T14:28:51.588126+0000","last_fullsized":"2024-09-13T14:28:51.588126+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:20:30.075633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:28:52.113107+0000","last_change":"2024-09-13T14:28:52.113107+0000","last_active":"2024-09-13T14:28:52.113107+0000","last_peered":"2024-09-13T14:28:52.113107+0000","last_clean":"2024-09-13T14:28:52.113107+0000","last_became_active":"2024-09-13T14:28:52.112842+0000","last_became_peered":"2024-09-13T14:28:52.112842+0000","last_unstale":"2024-09-13T14:28:52.113107+0000","last_undegraded":"2024-09-13T14:28:52.113107+0000","last_fullsized":"2024-09-13T14:28:52.113107+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:59:51.295506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.588004+0000","last_change":"2024-09-13T14:28:10.504588+0000","last_active":"2024-09-13T14:28:51.588004+0000","last_peered":"2024-09-13T14:28:51.588004+0000","last_clean":"2024-09-13T14:28:51.588004+0000","last_became_active":"2024-09-13T14:28:10.504338+0000","last_became_peered":"2024-09-13T14:28:10.504338+0000","last_unstale":"2024-09-13T14:28:51.588004+0000","last_undegraded":"2024-09-13T14:28:51.588004+0000","last_fullsized":"2024-09-13T14:28:51.588004+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:06:49.774993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646171+0000","last_change":"2024-09-13T14:28:11.508578+0000","last_active":"2024-09-13T14:28:50.646171+0000","last_peered":"2024-09-13T14:28:50.646171+0000","last_clean":"2024-09-13T14:28:50.646171+0000","last_became_active":"2024-09-13T14:28:11.508311+0000","last_became_peered":"2024-09-13T14:28:11.508311+0000","last_unstale":"2024-09-13T14:28:50.646171+0000","last_undegraded":"2024-09-13T14:28:50.646171+0000","last_fullsized":"2024-09-13T14:28:50.646171+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:47.732109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646276+0000","last_change":"2024-09-13T14:28:10.496693+0000","last_active":"2024-09-13T14:28:50.646276+0000","last_peered":"2024-09-13T14:28:50.646276+0000","last_clean":"2024-09-13T14:28:50.646276+0000","last_became_active":"2024-09-13T14:28:10.496424+0000","last_became_peered":"2024-09-13T14:28:10.496424+0000","last_unstale":"2024-09-13T14:28:50.646276+0000","last_undegraded":"2024-09-13T14:28:50.646276+0000","last_fullsized":"2024-09-13T14:28:50.646276+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:11.554436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587308+0000","last_change":"2024-09-13T14:28:11.531836+0000","last_active":"2024-09-13T14:28:51.587308+0000","last_peered":"2024-09-13T14:28:51.587308+0000","last_clean":"2024-09-13T14:28:51.587308+0000","last_became_active":"2024-09-13T14:28:11.531539+0000","last_became_peered":"2024-09-13T14:28:11.531539+0000","last_unstale":"2024-09-13T14:28:51.587308+0000","last_undegraded":"2024-09-13T14:28:51.587308+0000","last_fullsized":"2024-09-13T14:28:51.587308+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:46:34.698598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.648894+0000","last_change":"2024-09-13T14:28:10.500498+0000","last_active":"2024-09-13T14:28:50.648894+0000","last_peered":"2024-09-13T14:28:50.648894+0000","last_clean":"2024-09-13T14:28:50.648894+0000","last_became_active":"2024-09-13T14:28:10.500213+0000","last_became_peered":"2024-09-13T14:28:10.500213+0000","last_unstale":"2024-09-13T14:28:50.648894+0000","last_undegraded":"2024-09-13T14:28:50.648894+0000","last_fullsized":"2024-09-13T14:28:50.648894+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:08.352943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644060+0000","last_change":"2024-09-13T14:28:10.584320+0000","last_active":"2024-09-13T14:28:50.644060+0000","last_peered":"2024-09-13T14:28:50.644060+0000","last_clean":"2024-09-13T14:28:50.644060+0000","last_became_active":"2024-09-13T14:28:10.582950+0000","last_became_peered":"2024-09-13T14:28:10.582950+0000","last_unstale":"2024-09-13T14:28:50.644060+0000","last_undegraded":"2024-09-13T14:28:50.644060+0000","last_fullsized":"2024-09-13T14:28:50.644060+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:58:35.955340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646231+0000","last_change":"2024-09-13T14:28:10.508505+0000","last_active":"2024-09-13T14:28:50.646231+0000","last_peered":"2024-09-13T14:28:50.646231+0000","last_clean":"2024-09-13T14:28:50.646231+0000","last_became_active":"2024-09-13T14:28:10.508248+0000","last_became_peered":"2024-09-13T14:28:10.508248+0000","last_unstale":"2024-09-13T14:28:50.646231+0000","last_undegraded":"2024-09-13T14:28:50.646231+0000","last_fullsized":"2024-09-13T14:28:50.646231+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:51:45.033653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644718+0000","last_change":"2024-09-13T14:28:10.582336+0000","last_active":"2024-09-13T14:28:50.644718+0000","last_peered":"2024-09-13T14:28:50.644718+0000","last_clean":"2024-09-13T14:28:50.644718+0000","last_became_active":"2024-09-13T14:28:10.581923+0000","last_became_peered":"2024-09-13T14:28:10.581923+0000","last_unstale":"2024-09-13T14:28:50.644718+0000","last_undegraded":"2024-09-13T14:28:50.644718+0000","last_fullsized":"2024-09-13T14:28:50.644718+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:58.533266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644022+0000","last_change":"2024-09-13T14:28:10.504481+0000","last_active":"2024-09-13T14:28:50.644022+0000","last_peered":"2024-09-13T14:28:50.644022+0000","last_clean":"2024-09-13T14:28:50.644022+0000","last_became_active":"2024-09-13T14:28:10.504215+0000","last_became_peered":"2024-09-13T14:28:10.504215+0000","last_unstale":"2024-09-13T14:28:50.644022+0000","last_undegraded":"2024-09-13T14:28:50.644022+0000","last_fullsized":"2024-09-13T14:28:50.644022+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:27:01.645695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644246+0000","last_change":"2024-09-13T14:28:10.582606+0000","last_active":"2024-09-13T14:28:50.644246+0000","last_peered":"2024-09-13T14:28:50.644246+0000","last_clean":"2024-09-13T14:28:50.644246+0000","last_became_active":"2024-09-13T14:28:10.582316+0000","last_became_peered":"2024-09-13T14:28:10.582316+0000","last_unstale":"2024-09-13T14:28:50.644246+0000","last_undegraded":"2024-09-13T14:28:50.644246+0000","last_fullsized":"2024-09-13T14:28:50.644246+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:29.466745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.648972+0000","last_change":"2024-09-13T14:28:10.498813+0000","last_active":"2024-09-13T14:28:50.648972+0000","last_peered":"2024-09-13T14:28:50.648972+0000","last_clean":"2024-09-13T14:28:50.648972+0000","last_became_active":"2024-09-13T14:28:10.498529+0000","last_became_peered":"2024-09-13T14:28:10.498529+0000","last_unstale":"2024-09-13T14:28:50.648972+0000","last_undegraded":"2024-09-13T14:28:50.648972+0000","last_fullsized":"2024-09-13T14:28:50.648972+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:22:58.735920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.650800+0000","last_change":"2024-09-13T14:28:10.516027+0000","last_active":"2024-09-13T14:28:50.650800+0000","last_peered":"2024-09-13T14:28:50.650800+0000","last_clean":"2024-09-13T14:28:50.650800+0000","last_became_active":"2024-09-13T14:28:10.515758+0000","last_became_peered":"2024-09-13T14:28:10.515758+0000","last_unstale":"2024-09-13T14:28:50.650800+0000","last_undegraded":"2024-09-13T14:28:50.650800+0000","last_fullsized":"2024-09-13T14:28:50.650800+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:43.506170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"52'79","reported_seq":56,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587732+0000","last_change":"2024-09-13T14:26:20.041272+0000","last_active":"2024-09-13T14:28:51.587732+0000","last_peered":"2024-09-13T14:28:51.587732+0000","last_clean":"2024-09-13T14:28:51.587732+0000","last_became_active":"2024-09-13T14:26:20.035983+0000","last_became_peered":"2024-09-13T14:26:20.035983+0000","last_unstale":"2024-09-13T14:28:51.587732+0000","last_undegraded":"2024-09-13T14:28:51.587732+0000","last_fullsized":"2024-09-13T14:28:51.587732+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:24:20.875473+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:32:28.938482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":27,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646135+0000","last_change":"2024-09-13T14:27:01.130820+0000","last_active":"2024-09-13T14:28:50.646135+0000","last_peered":"2024-09-13T14:28:50.646135+0000","last_clean":"2024-09-13T14:28:50.646135+0000","last_became_active":"2024-09-13T14:26:56.113569+0000","last_became_peered":"2024-09-13T14:26:56.113569+0000","last_unstale":"2024-09-13T14:28:50.646135+0000","last_undegraded":"2024-09-13T14:28:50.646135+0000","last_fullsized":"2024-09-13T14:28:50.646135+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:47.705568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034224499999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649155+0000","last_change":"2024-09-13T14:28:10.507315+0000","last_active":"2024-09-13T14:28:50.649155+0000","last_peered":"2024-09-13T14:28:50.649155+0000","last_clean":"2024-09-13T14:28:50.649155+0000","last_became_active":"2024-09-13T14:28:10.507014+0000","last_became_peered":"2024-09-13T14:28:10.507014+0000","last_unstale":"2024-09-13T14:28:50.649155+0000","last_undegraded":"2024-09-13T14:28:50.649155+0000","last_fullsized":"2024-09-13T14:28:50.649155+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:12:03.810763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646533+0000","last_change":"2024-09-13T14:28:11.532509+0000","last_active":"2024-09-13T14:28:50.646533+0000","last_peered":"2024-09-13T14:28:50.646533+0000","last_clean":"2024-09-13T14:28:50.646533+0000","last_became_active":"2024-09-13T14:28:11.532207+0000","last_became_peered":"2024-09-13T14:28:11.532207+0000","last_unstale":"2024-09-13T14:28:50.646533+0000","last_undegraded":"2024-09-13T14:28:50.646533+0000","last_fullsized":"2024-09-13T14:28:50.646533+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:33.385182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587711+0000","last_change":"2024-09-13T14:27:01.908723+0000","last_active":"2024-09-13T14:28:51.587711+0000","last_peered":"2024-09-13T14:28:51.587711+0000","last_clean":"2024-09-13T14:28:51.587711+0000","last_became_active":"2024-09-13T14:26:56.115228+0000","last_became_peered":"2024-09-13T14:26:56.115228+0000","last_unstale":"2024-09-13T14:28:51.587711+0000","last_undegraded":"2024-09-13T14:28:51.587711+0000","last_fullsized":"2024-09-13T14:28:51.587711+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:48.775507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027767100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587419+0000","last_change":"2024-09-13T14:28:10.499750+0000","last_active":"2024-09-13T14:28:51.587419+0000","last_peered":"2024-09-13T14:28:51.587419+0000","last_clean":"2024-09-13T14:28:51.587419+0000","last_became_active":"2024-09-13T14:28:10.499453+0000","last_became_peered":"2024-09-13T14:28:10.499453+0000","last_unstale":"2024-09-13T14:28:51.587419+0000","last_undegraded":"2024-09-13T14:28:51.587419+0000","last_fullsized":"2024-09-13T14:28:51.587419+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:07:07.147783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587469+0000","last_change":"2024-09-13T14:28:11.508247+0000","last_active":"2024-09-13T14:28:51.587469+0000","last_peered":"2024-09-13T14:28:51.587469+0000","last_clean":"2024-09-13T14:28:51.587469+0000","last_became_active":"2024-09-13T14:28:11.507871+0000","last_became_peered":"2024-09-13T14:28:11.507871+0000","last_unstale":"2024-09-13T14:28:51.587469+0000","last_undegraded":"2024-09-13T14:28:51.587469+0000","last_fullsized":"2024-09-13T14:28:51.587469+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:17:56.989487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.645935+0000","last_change":"2024-09-13T14:27:01.127122+0000","last_active":"2024-09-13T14:28:50.645935+0000","last_peered":"2024-09-13T14:28:50.645935+0000","last_clean":"2024-09-13T14:28:50.645935+0000","last_became_active":"2024-09-13T14:26:56.178609+0000","last_became_peered":"2024-09-13T14:26:56.178609+0000","last_unstale":"2024-09-13T14:28:50.645935+0000","last_undegraded":"2024-09-13T14:28:50.645935+0000","last_fullsized":"2024-09-13T14:28:50.645935+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:23:16.906453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036252799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.648847+0000","last_change":"2024-09-13T14:28:10.495859+0000","last_active":"2024-09-13T14:28:50.648847+0000","last_peered":"2024-09-13T14:28:50.648847+0000","last_clean":"2024-09-13T14:28:50.648847+0000","last_became_active":"2024-09-13T14:28:10.495596+0000","last_became_peered":"2024-09-13T14:28:10.495596+0000","last_unstale":"2024-09-13T14:28:50.648847+0000","last_undegraded":"2024-09-13T14:28:50.648847+0000","last_fullsized":"2024-09-13T14:28:50.648847+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:00.760166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.650854+0000","last_change":"2024-09-13T14:28:10.511496+0000","last_active":"2024-09-13T14:28:50.650854+0000","last_peered":"2024-09-13T14:28:50.650854+0000","last_clean":"2024-09-13T14:28:50.650854+0000","last_became_active":"2024-09-13T14:28:10.511178+0000","last_became_peered":"2024-09-13T14:28:10.511178+0000","last_unstale":"2024-09-13T14:28:50.650854+0000","last_undegraded":"2024-09-13T14:28:50.650854+0000","last_fullsized":"2024-09-13T14:28:50.650854+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:17.860038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":28,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646833+0000","last_change":"2024-09-13T14:27:01.131429+0000","last_active":"2024-09-13T14:28:50.646833+0000","last_peered":"2024-09-13T14:28:50.646833+0000","last_clean":"2024-09-13T14:28:50.646833+0000","last_became_active":"2024-09-13T14:26:56.114214+0000","last_became_peered":"2024-09-13T14:26:56.114214+0000","last_unstale":"2024-09-13T14:28:50.646833+0000","last_undegraded":"2024-09-13T14:28:50.646833+0000","last_fullsized":"2024-09-13T14:28:50.646833+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:43.625300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00095382599999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644478+0000","last_change":"2024-09-13T14:28:10.506278+0000","last_active":"2024-09-13T14:28:50.644478+0000","last_peered":"2024-09-13T14:28:50.644478+0000","last_clean":"2024-09-13T14:28:50.644478+0000","last_became_active":"2024-09-13T14:28:10.506016+0000","last_became_peered":"2024-09-13T14:28:10.506016+0000","last_unstale":"2024-09-13T14:28:50.644478+0000","last_undegraded":"2024-09-13T14:28:50.644478+0000","last_fullsized":"2024-09-13T14:28:50.644478+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:32:02.776894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649699+0000","last_change":"2024-09-13T14:28:10.582166+0000","last_active":"2024-09-13T14:28:50.649699+0000","last_peered":"2024-09-13T14:28:50.649699+0000","last_clean":"2024-09-13T14:28:50.649699+0000","last_became_active":"2024-09-13T14:28:10.581870+0000","last_became_peered":"2024-09-13T14:28:10.581870+0000","last_unstale":"2024-09-13T14:28:50.649699+0000","last_undegraded":"2024-09-13T14:28:50.649699+0000","last_fullsized":"2024-09-13T14:28:50.649699+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:21.983145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646587+0000","last_change":"2024-09-13T14:28:10.496940+0000","last_active":"2024-09-13T14:28:50.646587+0000","last_peered":"2024-09-13T14:28:50.646587+0000","last_clean":"2024-09-13T14:28:50.646587+0000","last_became_active":"2024-09-13T14:28:10.495827+0000","last_became_peered":"2024-09-13T14:28:10.495827+0000","last_unstale":"2024-09-13T14:28:50.646587+0000","last_undegraded":"2024-09-13T14:28:50.646587+0000","last_fullsized":"2024-09-13T14:28:50.646587+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:29.695045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.650562+0000","last_change":"2024-09-13T14:27:01.127754+0000","last_active":"2024-09-13T14:28:50.650562+0000","last_peered":"2024-09-13T14:28:50.650562+0000","last_clean":"2024-09-13T14:28:50.650562+0000","last_became_active":"2024-09-13T14:26:56.112967+0000","last_became_peered":"2024-09-13T14:26:56.112967+0000","last_unstale":"2024-09-13T14:28:50.650562+0000","last_undegraded":"2024-09-13T14:28:50.650562+0000","last_fullsized":"2024-09-13T14:28:50.650562+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:40:20.095122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000367501,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649264+0000","last_change":"2024-09-13T14:28:10.516645+0000","last_active":"2024-09-13T14:28:50.649264+0000","last_peered":"2024-09-13T14:28:50.649264+0000","last_clean":"2024-09-13T14:28:50.649264+0000","last_became_active":"2024-09-13T14:28:10.516332+0000","last_became_peered":"2024-09-13T14:28:10.516332+0000","last_unstale":"2024-09-13T14:28:50.649264+0000","last_undegraded":"2024-09-13T14:28:50.649264+0000","last_fullsized":"2024-09-13T14:28:50.649264+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:51.371216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.650257+0000","last_change":"2024-09-13T14:28:10.491972+0000","last_active":"2024-09-13T14:28:50.650257+0000","last_peered":"2024-09-13T14:28:50.650257+0000","last_clean":"2024-09-13T14:28:50.650257+0000","last_became_active":"2024-09-13T14:28:10.491458+0000","last_became_peered":"2024-09-13T14:28:10.491458+0000","last_unstale":"2024-09-13T14:28:50.650257+0000","last_undegraded":"2024-09-13T14:28:50.650257+0000","last_fullsized":"2024-09-13T14:28:50.650257+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:54.166762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.588049+0000","last_change":"2024-09-13T14:27:01.908723+0000","last_active":"2024-09-13T14:28:51.588049+0000","last_peered":"2024-09-13T14:28:51.588049+0000","last_clean":"2024-09-13T14:28:51.588049+0000","last_became_active":"2024-09-13T14:26:56.124401+0000","last_became_peered":"2024-09-13T14:26:56.124401+0000","last_unstale":"2024-09-13T14:28:51.588049+0000","last_undegraded":"2024-09-13T14:28:51.588049+0000","last_fullsized":"2024-09-13T14:28:51.588049+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:46.437020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027405700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587304+0000","last_change":"2024-09-13T14:28:11.509145+0000","last_active":"2024-09-13T14:28:51.587304+0000","last_peered":"2024-09-13T14:28:51.587304+0000","last_clean":"2024-09-13T14:28:51.587304+0000","last_became_active":"2024-09-13T14:28:11.508833+0000","last_became_peered":"2024-09-13T14:28:11.508833+0000","last_unstale":"2024-09-13T14:28:51.587304+0000","last_undegraded":"2024-09-13T14:28:51.587304+0000","last_fullsized":"2024-09-13T14:28:51.587304+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:17:08.969691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649372+0000","last_change":"2024-09-13T14:28:10.500545+0000","last_active":"2024-09-13T14:28:50.649372+0000","last_peered":"2024-09-13T14:28:50.649372+0000","last_clean":"2024-09-13T14:28:50.649372+0000","last_became_active":"2024-09-13T14:28:10.500220+0000","last_became_peered":"2024-09-13T14:28:10.500220+0000","last_unstale":"2024-09-13T14:28:50.649372+0000","last_undegraded":"2024-09-13T14:28:50.649372+0000","last_fullsized":"2024-09-13T14:28:50.649372+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:33.274960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.650457+0000","last_change":"2024-09-13T14:27:01.127728+0000","last_active":"2024-09-13T14:28:50.650457+0000","last_peered":"2024-09-13T14:28:50.650457+0000","last_clean":"2024-09-13T14:28:50.650457+0000","last_became_active":"2024-09-13T14:26:56.124603+0000","last_became_peered":"2024-09-13T14:26:56.124603+0000","last_unstale":"2024-09-13T14:28:50.650457+0000","last_undegraded":"2024-09-13T14:28:50.650457+0000","last_fullsized":"2024-09-13T14:28:50.650457+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:46:33.019632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000373223,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.648804+0000","last_change":"2024-09-13T14:28:11.513073+0000","last_active":"2024-09-13T14:28:50.648804+0000","last_peered":"2024-09-13T14:28:50.648804+0000","last_clean":"2024-09-13T14:28:50.648804+0000","last_became_active":"2024-09-13T14:28:11.512821+0000","last_became_peered":"2024-09-13T14:28:11.512821+0000","last_unstale":"2024-09-13T14:28:50.648804+0000","last_undegraded":"2024-09-13T14:28:50.648804+0000","last_fullsized":"2024-09-13T14:28:50.648804+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:56.692654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587888+0000","last_change":"2024-09-13T14:28:10.497814+0000","last_active":"2024-09-13T14:28:51.587888+0000","last_peered":"2024-09-13T14:28:51.587888+0000","last_clean":"2024-09-13T14:28:51.587888+0000","last_became_active":"2024-09-13T14:28:10.497541+0000","last_became_peered":"2024-09-13T14:28:10.497541+0000","last_unstale":"2024-09-13T14:28:51.587888+0000","last_undegraded":"2024-09-13T14:28:51.587888+0000","last_fullsized":"2024-09-13T14:28:51.587888+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:14:56.511439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646454+0000","last_change":"2024-09-13T14:27:01.132344+0000","last_active":"2024-09-13T14:28:50.646454+0000","last_peered":"2024-09-13T14:28:50.646454+0000","last_clean":"2024-09-13T14:28:50.646454+0000","last_became_active":"2024-09-13T14:26:56.113467+0000","last_became_peered":"2024-09-13T14:26:56.113467+0000","last_unstale":"2024-09-13T14:28:50.646454+0000","last_undegraded":"2024-09-13T14:28:50.646454+0000","last_fullsized":"2024-09-13T14:28:50.646454+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:57:56.604631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023893800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.643876+0000","last_change":"2024-09-13T14:28:10.583355+0000","last_active":"2024-09-13T14:28:50.643876+0000","last_peered":"2024-09-13T14:28:50.643876+0000","last_clean":"2024-09-13T14:28:50.643876+0000","last_became_active":"2024-09-13T14:28:10.583032+0000","last_became_peered":"2024-09-13T14:28:10.583032+0000","last_unstale":"2024-09-13T14:28:50.643876+0000","last_undegraded":"2024-09-13T14:28:50.643876+0000","last_fullsized":"2024-09-13T14:28:50.643876+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:05:15.659274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644523+0000","last_change":"2024-09-13T14:28:10.583688+0000","last_active":"2024-09-13T14:28:50.644523+0000","last_peered":"2024-09-13T14:28:50.644523+0000","last_clean":"2024-09-13T14:28:50.644523+0000","last_became_active":"2024-09-13T14:28:10.583409+0000","last_became_peered":"2024-09-13T14:28:10.583409+0000","last_unstale":"2024-09-13T14:28:50.644523+0000","last_undegraded":"2024-09-13T14:28:50.644523+0000","last_fullsized":"2024-09-13T14:28:50.644523+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:33.587275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"64'6","reported_seq":20,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644168+0000","last_change":"2024-09-13T14:28:10.505107+0000","last_active":"2024-09-13T14:28:50.644168+0000","last_peered":"2024-09-13T14:28:50.644168+0000","last_clean":"2024-09-13T14:28:50.644168+0000","last_became_active":"2024-09-13T14:28:10.504857+0000","last_became_peered":"2024-09-13T14:28:10.504857+0000","last_unstale":"2024-09-13T14:28:50.644168+0000","last_undegraded":"2024-09-13T14:28:50.644168+0000","last_fullsized":"2024-09-13T14:28:50.644168+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:50:15.966787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649399+0000","last_change":"2024-09-13T14:28:10.582866+0000","last_active":"2024-09-13T14:28:50.649399+0000","last_peered":"2024-09-13T14:28:50.649399+0000","last_clean":"2024-09-13T14:28:50.649399+0000","last_became_active":"2024-09-13T14:28:10.582189+0000","last_became_peered":"2024-09-13T14:28:10.582189+0000","last_unstale":"2024-09-13T14:28:50.649399+0000","last_undegraded":"2024-09-13T14:28:50.649399+0000","last_fullsized":"2024-09-13T14:28:50.649399+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:20:53.470398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":17,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587961+0000","last_change":"2024-09-13T14:28:10.509946+0000","last_active":"2024-09-13T14:28:51.587961+0000","last_peered":"2024-09-13T14:28:51.587961+0000","last_clean":"2024-09-13T14:28:51.587961+0000","last_became_active":"2024-09-13T14:28:10.509689+0000","last_became_peered":"2024-09-13T14:28:10.509689+0000","last_unstale":"2024-09-13T14:28:51.587961+0000","last_undegraded":"2024-09-13T14:28:51.587961+0000","last_fullsized":"2024-09-13T14:28:51.587961+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:13.955928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646243+0000","last_change":"2024-09-13T14:28:10.498279+0000","last_active":"2024-09-13T14:28:50.646243+0000","last_peered":"2024-09-13T14:28:50.646243+0000","last_clean":"2024-09-13T14:28:50.646243+0000","last_became_active":"2024-09-13T14:28:10.498016+0000","last_became_peered":"2024-09-13T14:28:10.498016+0000","last_unstale":"2024-09-13T14:28:50.646243+0000","last_undegraded":"2024-09-13T14:28:50.646243+0000","last_fullsized":"2024-09-13T14:28:50.646243+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:04.417632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.648897+0000","last_change":"2024-09-13T14:28:11.512646+0000","last_active":"2024-09-13T14:28:50.648897+0000","last_peered":"2024-09-13T14:28:50.648897+0000","last_clean":"2024-09-13T14:28:50.648897+0000","last_became_active":"2024-09-13T14:28:11.512392+0000","last_became_peered":"2024-09-13T14:28:11.512392+0000","last_unstale":"2024-09-13T14:28:50.648897+0000","last_undegraded":"2024-09-13T14:28:50.648897+0000","last_fullsized":"2024-09-13T14:28:50.648897+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:09.381308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587975+0000","last_change":"2024-09-13T14:28:10.495999+0000","last_active":"2024-09-13T14:28:51.587975+0000","last_peered":"2024-09-13T14:28:51.587975+0000","last_clean":"2024-09-13T14:28:51.587975+0000","last_became_active":"2024-09-13T14:28:10.495736+0000","last_became_peered":"2024-09-13T14:28:10.495736+0000","last_unstale":"2024-09-13T14:28:51.587975+0000","last_undegraded":"2024-09-13T14:28:51.587975+0000","last_fullsized":"2024-09-13T14:28:51.587975+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:39.379161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.648699+0000","last_change":"2024-09-13T14:28:11.512381+0000","last_active":"2024-09-13T14:28:50.648699+0000","last_peered":"2024-09-13T14:28:50.648699+0000","last_clean":"2024-09-13T14:28:50.648699+0000","last_became_active":"2024-09-13T14:28:11.512031+0000","last_became_peered":"2024-09-13T14:28:11.512031+0000","last_unstale":"2024-09-13T14:28:50.648699+0000","last_undegraded":"2024-09-13T14:28:50.648699+0000","last_fullsized":"2024-09-13T14:28:50.648699+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:56.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646379+0000","last_change":"2024-09-13T14:28:11.507361+0000","last_active":"2024-09-13T14:28:50.646379+0000","last_peered":"2024-09-13T14:28:50.646379+0000","last_clean":"2024-09-13T14:28:50.646379+0000","last_became_active":"2024-09-13T14:28:11.507077+0000","last_became_peered":"2024-09-13T14:28:11.507077+0000","last_unstale":"2024-09-13T14:28:50.646379+0000","last_undegraded":"2024-09-13T14:28:50.646379+0000","last_fullsized":"2024-09-13T14:28:50.646379+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:33.602421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.648897+0000","last_change":"2024-09-13T14:28:10.501264+0000","last_active":"2024-09-13T14:28:50.648897+0000","last_peered":"2024-09-13T14:28:50.648897+0000","last_clean":"2024-09-13T14:28:50.648897+0000","last_became_active":"2024-09-13T14:28:10.500188+0000","last_became_peered":"2024-09-13T14:28:10.500188+0000","last_unstale":"2024-09-13T14:28:50.648897+0000","last_undegraded":"2024-09-13T14:28:50.648897+0000","last_fullsized":"2024-09-13T14:28:50.648897+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:46:53.209853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587575+0000","last_change":"2024-09-13T14:28:11.513490+0000","last_active":"2024-09-13T14:28:51.587575+0000","last_peered":"2024-09-13T14:28:51.587575+0000","last_clean":"2024-09-13T14:28:51.587575+0000","last_became_active":"2024-09-13T14:28:11.513180+0000","last_became_peered":"2024-09-13T14:28:11.513180+0000","last_unstale":"2024-09-13T14:28:51.587575+0000","last_undegraded":"2024-09-13T14:28:51.587575+0000","last_fullsized":"2024-09-13T14:28:51.587575+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:06:31.355641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649424+0000","last_change":"2024-09-13T14:28:10.499527+0000","last_active":"2024-09-13T14:28:50.649424+0000","last_peered":"2024-09-13T14:28:50.649424+0000","last_clean":"2024-09-13T14:28:50.649424+0000","last_became_active":"2024-09-13T14:28:10.499202+0000","last_became_peered":"2024-09-13T14:28:10.499202+0000","last_unstale":"2024-09-13T14:28:50.649424+0000","last_undegraded":"2024-09-13T14:28:50.649424+0000","last_fullsized":"2024-09-13T14:28:50.649424+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:21.301397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:28:52.112821+0000","last_change":"2024-09-13T14:28:52.112821+0000","last_active":"2024-09-13T14:28:52.112821+0000","last_peered":"2024-09-13T14:28:52.112821+0000","last_clean":"2024-09-13T14:28:52.112821+0000","last_became_active":"2024-09-13T14:28:52.112591+0000","last_became_peered":"2024-09-13T14:28:52.112591+0000","last_unstale":"2024-09-13T14:28:52.112821+0000","last_undegraded":"2024-09-13T14:28:52.112821+0000","last_fullsized":"2024-09-13T14:28:52.112821+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:05:06.365986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644028+0000","last_change":"2024-09-13T14:28:10.497427+0000","last_active":"2024-09-13T14:28:50.644028+0000","last_peered":"2024-09-13T14:28:50.644028+0000","last_clean":"2024-09-13T14:28:50.644028+0000","last_became_active":"2024-09-13T14:28:10.497149+0000","last_became_peered":"2024-09-13T14:28:10.497149+0000","last_unstale":"2024-09-13T14:28:50.644028+0000","last_undegraded":"2024-09-13T14:28:50.644028+0000","last_fullsized":"2024-09-13T14:28:50.644028+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:00:25.254819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646024+0000","last_change":"2024-09-13T14:28:11.514186+0000","last_active":"2024-09-13T14:28:50.646024+0000","last_peered":"2024-09-13T14:28:50.646024+0000","last_clean":"2024-09-13T14:28:50.646024+0000","last_became_active":"2024-09-13T14:28:11.513858+0000","last_became_peered":"2024-09-13T14:28:11.513858+0000","last_unstale":"2024-09-13T14:28:50.646024+0000","last_undegraded":"2024-09-13T14:28:50.646024+0000","last_fullsized":"2024-09-13T14:28:50.646024+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:53:10.642897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587595+0000","last_change":"2024-09-13T14:28:10.498999+0000","last_active":"2024-09-13T14:28:51.587595+0000","last_peered":"2024-09-13T14:28:51.587595+0000","last_clean":"2024-09-13T14:28:51.587595+0000","last_became_active":"2024-09-13T14:28:10.498736+0000","last_became_peered":"2024-09-13T14:28:10.498736+0000","last_unstale":"2024-09-13T14:28:51.587595+0000","last_undegraded":"2024-09-13T14:28:51.587595+0000","last_fullsized":"2024-09-13T14:28:51.587595+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:33.684548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.644324+0000","last_change":"2024-09-13T14:28:10.517704+0000","last_active":"2024-09-13T14:28:50.644324+0000","last_peered":"2024-09-13T14:28:50.644324+0000","last_clean":"2024-09-13T14:28:50.644324+0000","last_became_active":"2024-09-13T14:28:10.517400+0000","last_became_peered":"2024-09-13T14:28:10.517400+0000","last_unstale":"2024-09-13T14:28:50.644324+0000","last_undegraded":"2024-09-13T14:28:50.644324+0000","last_fullsized":"2024-09-13T14:28:50.644324+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:01.199610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646194+0000","last_change":"2024-09-13T14:28:10.494883+0000","last_active":"2024-09-13T14:28:50.646194+0000","last_peered":"2024-09-13T14:28:50.646194+0000","last_clean":"2024-09-13T14:28:50.646194+0000","last_became_active":"2024-09-13T14:28:10.494623+0000","last_became_peered":"2024-09-13T14:28:10.494623+0000","last_unstale":"2024-09-13T14:28:50.646194+0000","last_undegraded":"2024-09-13T14:28:50.646194+0000","last_fullsized":"2024-09-13T14:28:50.646194+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:17.878113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.643925+0000","last_change":"2024-09-13T14:28:10.515419+0000","last_active":"2024-09-13T14:28:50.643925+0000","last_peered":"2024-09-13T14:28:50.643925+0000","last_clean":"2024-09-13T14:28:50.643925+0000","last_became_active":"2024-09-13T14:28:10.515214+0000","last_became_peered":"2024-09-13T14:28:10.515214+0000","last_unstale":"2024-09-13T14:28:50.643925+0000","last_undegraded":"2024-09-13T14:28:50.643925+0000","last_fullsized":"2024-09-13T14:28:50.643925+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:36:45.422984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.646513+0000","last_change":"2024-09-13T14:28:10.500736+0000","last_active":"2024-09-13T14:28:50.646513+0000","last_peered":"2024-09-13T14:28:50.646513+0000","last_clean":"2024-09-13T14:28:50.646513+0000","last_became_active":"2024-09-13T14:28:10.500467+0000","last_became_peered":"2024-09-13T14:28:10.500467+0000","last_unstale":"2024-09-13T14:28:50.646513+0000","last_undegraded":"2024-09-13T14:28:50.646513+0000","last_fullsized":"2024-09-13T14:28:50.646513+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:19.688685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649092+0000","last_change":"2024-09-13T14:28:11.513626+0000","last_active":"2024-09-13T14:28:50.649092+0000","last_peered":"2024-09-13T14:28:50.649092+0000","last_clean":"2024-09-13T14:28:50.649092+0000","last_became_active":"2024-09-13T14:28:11.513267+0000","last_became_peered":"2024-09-13T14:28:11.513267+0000","last_unstale":"2024-09-13T14:28:50.649092+0000","last_undegraded":"2024-09-13T14:28:50.649092+0000","last_fullsized":"2024-09-13T14:28:50.649092+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:02:52.487064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:50.649089+0000","last_change":"2024-09-13T14:28:10.504329+0000","last_active":"2024-09-13T14:28:50.649089+0000","last_peered":"2024-09-13T14:28:50.649089+0000","last_clean":"2024-09-13T14:28:50.649089+0000","last_became_active":"2024-09-13T14:28:10.504074+0000","last_became_peered":"2024-09-13T14:28:10.504074+0000","last_unstale":"2024-09-13T14:28:50.649089+0000","last_undegraded":"2024-09-13T14:28:50.649089+0000","last_fullsized":"2024-09-13T14:28:50.649089+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:03:47.723724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:28:52.557448+0000","last_change":"2024-09-13T14:28:52.557448+0000","last_active":"2024-09-13T14:28:52.557448+0000","last_peered":"2024-09-13T14:28:52.557448+0000","last_clean":"2024-09-13T14:28:52.557448+0000","last_became_active":"2024-09-13T14:28:52.557090+0000","last_became_peered":"2024-09-13T14:28:52.557090+0000","last_unstale":"2024-09-13T14:28:52.557448+0000","last_undegraded":"2024-09-13T14:28:52.557448+0000","last_fullsized":"2024-09-13T14:28:52.557448+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:59.236430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:28:51.587950+0000","last_change":"2024-09-13T14:28:10.496380+0000","last_active":"2024-09-13T14:28:51.587950+0000","last_peered":"2024-09-13T14:28:51.587950+0000","last_clean":"2024-09-13T14:28:51.587950+0000","last_became_active":"2024-09-13T14:28:10.494751+0000","last_became_peered":"2024-09-13T14:28:10.494751+0000","last_unstale":"2024-09-13T14:28:51.587950+0000","last_undegraded":"2024-09-13T14:28:51.587950+0000","last_fullsized":"2024-09-13T14:28:51.587950+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:34:56.137599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397541,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298624,"kb_used_data":1264,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446528,"statfs":{"total":95995035648,"available":95689244672,"internally_reserved":0,"allocated":1294336,"data_stored":888643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57099999999999995,"15min":0.57599999999999996},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.67700000000000005,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.55200000000000005,"15min":0.55800000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.72199999999999998,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.51300000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.50800000000000001,"15min":0.51000000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.495},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.53500000000000003,"15min":0.53700000000000003},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.71499999999999997,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.63500000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.56299999999999994,"15min":0.56999999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.69999999999999996,"5min":1.0880000000000001,"15min":1.0880000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.38300000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.81799999999999995,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.55000000000000004}]},{"osd":3,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56899999999999995,"15min":0.57599999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.76900000000000002,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.44},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.56499999999999995,"15min":0.57099999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.69299999999999995,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.64900000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.49199999999999999,"15min":0.49099999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.80100000000000005,"5min":1.45,"15min":1.45},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.32800000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.79100000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.621}]},{"osd":5,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.47399999999999998,"15min":0.46800000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.68799999999999994,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.35799999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.73199999999999998,"5min":1.421,"15min":1.421},"last":0.66000000000000003}]},{"osd":6,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.53500000000000003,"15min":0.53300000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.79700000000000004,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.61799999999999999,"15min":0.622},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.82699999999999996,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.69799999999999995}]}]},{"osd":6,"up_from":43,"seq":184683593770,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298612,"kb_used_data":1252,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446540,"statfs":{"total":95995035648,"available":95689256960,"internally_reserved":0,"allocated":1282048,"data_stored":888201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.52500000000000002,"15min":0.51700000000000002},"min":{"1min":0.34799999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.73599999999999999,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.69699999999999995,"15min":0.58699999999999997},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.77300000000000002,"5min":17.888000000000002,"15min":17.888000000000002},"last":0.65600000000000003}]},{"osd":1,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.52900000000000003,"15min":0.53300000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39200000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.69899999999999995,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.50800000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51200000000000001,"15min":0.51300000000000001},"min":{"1min":0.40400000000000003,"5min":0.379,"15min":0.379},"max":{"1min":0.71099999999999997,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.55900000000000005,"15min":0.56599999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.83699999999999997,"5min":0.874,"15min":0.874},"last":0.52000000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.626,"15min":0.56000000000000005},"min":{"1min":0.376,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.412000000000001,"5min":10.412000000000001,"15min":10.412000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.70799999999999996,"15min":0.60299999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.753,"5min":17.952999999999999,"15min":17.952999999999999},"last":0.497}]},{"osd":4,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.45400000000000001,"15min":0.45500000000000002},"min":{"1min":0.35099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.45300000000000001,"15min":0.44800000000000001},"min":{"1min":0.29999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.76400000000000001,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.34699999999999998}]},{"osd":5,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.44900000000000001,"15min":0.443},"min":{"1min":0.32700000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.61899999999999999,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.504,"15min":0.503},"min":{"1min":0.33300000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.373}]},{"osd":7,"last update":"Fri Sep 13 14:28:18 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54500000000000004,"15min":0.54700000000000004},"min":{"1min":0.36299999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.80300000000000005,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.51300000000000001,"15min":0.51000000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.79100000000000004,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.39600000000000002}]}]},{"osd":5,"up_from":37,"seq":158913789999,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298024,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447128,"statfs":{"total":95995035648,"available":95689859072,"internally_reserved":0,"allocated":679936,"data_stored":296922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:28:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.53600000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.52200000000000002,"15min":0.52000000000000002},"min":{"1min":0.371,"5min":0.311,"15min":0.311},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.64400000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:28:34 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51500000000000001,"15min":0.51800000000000002},"min":{"1min":0.34100000000000003,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.68600000000000005,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52800000000000002,"15min":0.53500000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.69099999999999995,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.55300000000000005}]},{"osd":2,"last update":"Fri Sep 13 14:28:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52500000000000002,"15min":0.53100000000000003},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.64300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.52500000000000002,"15min":0.52600000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.69899999999999995,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.67200000000000004}]},{"osd":3,"last update":"Fri Sep 13 14:28:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52800000000000002,"15min":0.53300000000000003},"min":{"1min":0.35399999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.67900000000000005,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.53200000000000003,"15min":0.53400000000000003},"min":{"1min":0.35899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.63}]},{"osd":4,"last update":"Fri Sep 13 14:28:34 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.376,"15min":0.35999999999999999},"min":{"1min":0.26900000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.65700000000000003,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.496},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.39000000000000001,"15min":0.372},"min":{"1min":0.28599999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.438}]},{"osd":6,"last update":"Fri Sep 13 14:28:57 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.44,"15min":0.42599999999999999},"min":{"1min":0.313,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.67300000000000004,"5min":0.747,"15min":0.747},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.41999999999999998,"15min":0.40899999999999997},"min":{"1min":0.30399999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.70099999999999996,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.47699999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:28:24 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.44900000000000001,"15min":0.45100000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.70599999999999996,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.47999999999999998,"15min":0.47899999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.58199999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986228,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298040,"kb_used_data":680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447112,"statfs":{"total":95995035648,"available":95689842688,"internally_reserved":0,"allocated":696320,"data_stored":298205,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:28:11 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.53200000000000003,"15min":0.55600000000000005},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":0.66800000000000004,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.436},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.62,"15min":0.56299999999999994},"min":{"1min":0.35799999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.6240000000000006,"5min":9.6240000000000006,"15min":9.6240000000000006},"last":0.497}]},{"osd":1,"last update":"Fri Sep 13 14:28:11 2024","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":0.66500000000000004,"15min":0.57499999999999996},"min":{"1min":0.32600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.5850000000000009,"5min":9.5850000000000009,"15min":9.5850000000000009},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.53900000000000003,"15min":0.55700000000000005},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.65300000000000002,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.51200000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:28:11 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.52900000000000003,"15min":0.54000000000000004},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.69799999999999995,"5min":0.745,"15min":0.745},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.53200000000000003,"15min":0.53800000000000003},"min":{"1min":0.34200000000000003,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.68600000000000005,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.46600000000000003}]},{"osd":3,"last update":"Fri Sep 13 14:28:11 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.52300000000000002,"15min":0.53000000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.68600000000000005,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.498,"5min":0.53900000000000003,"15min":0.55100000000000005},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.71899999999999997,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.54700000000000004}]},{"osd":5,"last update":"Fri Sep 13 14:28:36 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.39000000000000001,"15min":0.38200000000000001},"min":{"1min":0.255,"5min":0.219,"15min":0.219},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.38500000000000001,"15min":0.377},"min":{"1min":0.28799999999999998,"5min":0.217,"15min":0.217},"max":{"1min":0.54400000000000004,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.35499999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:28:58 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.46200000000000002,"15min":0.44900000000000001},"min":{"1min":0.30399999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.45600000000000002,"15min":0.442},"min":{"1min":0.35399999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.57299999999999995}]},{"osd":7,"last update":"Fri Sep 13 14:28:19 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.46700000000000003,"15min":0.46999999999999997},"min":{"1min":0.33200000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.626,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.502,"5min":0.51300000000000001,"15min":0.51500000000000001},"min":{"1min":0.35499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.67600000000000005,"5min":0.745,"15min":0.745},"last":0.41999999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149753,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298016,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447136,"statfs":{"total":95995035648,"available":95689867264,"internally_reserved":0,"allocated":671744,"data_stored":296353,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:28:52 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.38900000000000001,"15min":0.41499999999999998},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.50600000000000001,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.247},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.371,"15min":0.38700000000000001},"min":{"1min":0.23000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.51900000000000002,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.23200000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:28:52 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.40300000000000002,"15min":0.41999999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.55200000000000005,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.376,"5min":0.38500000000000001,"15min":0.39900000000000002},"min":{"1min":0.24299999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.52100000000000002,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.34899999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:28:52 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.42499999999999999,"15min":0.44700000000000001},"min":{"1min":0.26600000000000001,"5min":0.248,"15min":0.248},"max":{"1min":0.54000000000000004,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.38,"5min":0.40100000000000002,"15min":0.42499999999999999},"min":{"1min":0.25,"5min":0.22,"15min":0.22},"max":{"1min":0.51800000000000002,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.30199999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:28:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.59399999999999997,"15min":0.57699999999999996},"min":{"1min":0.33300000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.80800000000000005,"5min":4.3979999999999997,"15min":4.3979999999999997},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.624,"15min":0.59399999999999997},"min":{"1min":0.38700000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81100000000000005,"5min":4.4660000000000002,"15min":4.4660000000000002},"last":0.59799999999999998}]},{"osd":5,"last update":"Fri Sep 13 14:28:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.67800000000000005,"15min":0.72999999999999998},"min":{"1min":0.37,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.78900000000000003,"5min":4.4139999999999997,"15min":4.4139999999999997},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.67300000000000004,"15min":0.72199999999999998},"min":{"1min":0.38400000000000001,"5min":0.312,"15min":0.312},"max":{"1min":0.81000000000000005,"5min":4.3799999999999999,"15min":4.3799999999999999},"last":0.58099999999999996}]},{"osd":6,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.70999999999999996,"15min":0.76000000000000001},"min":{"1min":0.42899999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.22,"5min":4.4329999999999998,"15min":4.4329999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.55700000000000005,"15min":0.55100000000000005},"min":{"1min":0.42099999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.78900000000000003,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.51500000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:28:23 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.52000000000000002,"15min":0.50900000000000001},"min":{"1min":0.442,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.55600000000000005,"15min":0.55000000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.56000000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411390,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298020,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447132,"statfs":{"total":95995035648,"available":95689863168,"internally_reserved":0,"allocated":675840,"data_stored":296867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.34200000000000003,"15min":0.33700000000000002},"min":{"1min":0.20200000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.53100000000000003,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.32700000000000001,"5min":0.36399999999999999,"15min":0.36199999999999999},"min":{"1min":0.151,"5min":0.151,"15min":0.151},"max":{"1min":0.51100000000000001,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.379}]},{"osd":1,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.36299999999999999,"15min":0.36099999999999999},"min":{"1min":0.24099999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.52600000000000002,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.505},{"interface":"front","average":{"1min":0.33400000000000002,"5min":0.38300000000000001,"15min":0.38600000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.55000000000000004,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.31}]},{"osd":3,"last update":"Fri Sep 13 14:28:49 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.40500000000000003,"15min":0.41299999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.41999999999999998,"15min":0.42799999999999999},"min":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.45700000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:28:10 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.52600000000000002,"15min":0.52300000000000002},"min":{"1min":0.38900000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.77300000000000002,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.59299999999999997}]},{"osd":5,"last update":"Fri Sep 13 14:28:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.53100000000000003,"15min":0.53800000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.73699999999999999,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.755,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.76600000000000001}]},{"osd":6,"last update":"Fri Sep 13 14:29:00 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.59099999999999997,"15min":0.58699999999999997},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.204,"5min":1.204,"15min":1.204},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.57399999999999995,"15min":0.57799999999999996},"min":{"1min":0.39500000000000002,"5min":0.374,"15min":0.374},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.63800000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:28:20 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.54300000000000004,"15min":0.54100000000000004},"min":{"1min":0.41799999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.57899999999999996,"15min":0.58499999999999996},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.73199999999999998,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.72199999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574915,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298020,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447132,"statfs":{"total":95995035648,"available":95689863168,"internally_reserved":0,"allocated":675840,"data_stored":296526,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.34599999999999997,"15min":0.30199999999999999},"min":{"1min":0.23899999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.46899999999999997,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.33400000000000002,"5min":0.371,"15min":0.32300000000000001},"min":{"1min":0.21299999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.53700000000000003,"5min":1.198,"15min":1.198},"last":0.309}]},{"osd":2,"last update":"Fri Sep 13 14:28:28 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.379,"15min":0.36399999999999999},"min":{"1min":0.24399999999999999,"5min":0.223,"15min":0.223},"max":{"1min":0.53100000000000003,"5min":1.0800000000000001,"15min":1.0800000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.36699999999999999,"15min":0.35099999999999998},"min":{"1min":0.254,"5min":0.216,"15min":0.216},"max":{"1min":0.51300000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.27600000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:28:54 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.41999999999999998,"15min":0.436},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.63400000000000001,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.40100000000000002,"15min":0.40899999999999997},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.51800000000000002,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.36899999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:28:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.54700000000000004,"15min":0.54000000000000004},"min":{"1min":0.38900000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.55000000000000004,"15min":0.54300000000000004},"min":{"1min":0.38400000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.80200000000000005,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.66600000000000004}]},{"osd":5,"last update":"Fri Sep 13 14:28:33 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.57899999999999996,"15min":0.58199999999999996},"min":{"1min":0.40000000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79200000000000004,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.59099999999999997,"15min":0.58699999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.49099999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:28:57 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.59299999999999997,"15min":0.58699999999999997},"min":{"1min":0.371,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76900000000000002,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.51600000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:28:22 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.59099999999999997,"15min":0.58599999999999997},"min":{"1min":0.41999999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.78400000000000003,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.59399999999999997,"15min":0.59499999999999997},"min":{"1min":0.39300000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80200000000000005,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.61499999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298604,"kb_used_data":1244,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446548,"statfs":{"total":95995035648,"available":95689265152,"internally_reserved":0,"allocated":1273856,"data_stored":887467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:29:03 2024","interfaces":[{"interface":"back","average":{"1min":0.32500000000000001,"5min":0.33200000000000002,"15min":0.317},"min":{"1min":0.223,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.41899999999999998,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.371},{"interface":"front","average":{"1min":0.32800000000000001,"5min":0.34300000000000003,"15min":0.315},"min":{"1min":0.23400000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.44,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.28199999999999997}]},{"osd":2,"last update":"Fri Sep 13 14:28:28 2024","interfaces":[{"interface":"back","average":{"1min":0.34100000000000003,"5min":0.36399999999999999,"15min":0.35399999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.48599999999999999,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.32900000000000001,"5min":0.35199999999999998,"15min":0.34699999999999998},"min":{"1min":0.23899999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.42899999999999999,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.33700000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:28:53 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.39500000000000002,"15min":0.40400000000000003},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.46999999999999997,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.33700000000000002,"5min":0.376,"15min":0.39600000000000002},"min":{"1min":0.252,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.52400000000000002,"5min":0.627,"15min":0.627},"last":0.42699999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:28:10 2024","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.58499999999999996,"15min":0.55700000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.5510000000000002,"5min":5.5510000000000002,"15min":5.5510000000000002},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.54900000000000004,"15min":0.55700000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.71499999999999997,"5min":0.755,"15min":0.755},"last":0.57699999999999996}]},{"osd":5,"last update":"Fri Sep 13 14:28:31 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.55100000000000005,"15min":0.55300000000000005},"min":{"1min":0.40600000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.76700000000000002,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.38100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.56699999999999995}]},{"osd":6,"last update":"Fri Sep 13 14:28:58 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.38900000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.83099999999999996},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.56299999999999994,"15min":0.57499999999999996},"min":{"1min":0.35999999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.75700000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.60199999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:28:19 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.4350000000000001,"15min":1.581},"min":{"1min":0.45700000000000002,"5min":0.378,"15min":0.378},"max":{"1min":0.70499999999999996,"5min":19.390000000000001,"15min":19.390000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.56000000000000005,"15min":0.56499999999999995},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.747,"5min":0.748,"15min":0.748},"last":0.63900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:29:12.535 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:29:12.536 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T14:29:12.536 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T14:29:12.536 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T14:29:12.788 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:12.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:12 smithi096 ceph-mon[29229]: from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:29:12.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:12 smithi096 ceph-mon[39645]: from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:29:12.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:12 smithi204 ceph-mon[32546]: from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:29:13.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:13 smithi204 ceph-mon[32546]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:13.994 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:13 smithi096 ceph-mon[29229]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:13.995 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:13 smithi096 ceph-mon[39645]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:14.721 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:29:14.722 INFO:teuthology.orchestra.run.smithi096.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T14:29:15.443 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T14:29:15.444 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:29:15.455 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-09-13T14:29:15.455 INFO:teuthology.run_tasks:Running task parallel... 2024-09-13T14:29:15.470 INFO:teuthology.task.parallel:starting parallel... 2024-09-13T14:29:15.471 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T14:29:15.472 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-09-13T14:29:15.482 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-13T14:29:15.483 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-13T14:29:15.484 DEBUG:teuthology.orchestra.run.smithi096:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:29:15.484 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T14:29:15.485 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-13T14:29:15.485 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi096.front.sepia.ceph.com 2024-09-13T14:29:15.485 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-09-13T14:29:15.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:15.501 INFO:teuthology.orchestra.run.smithi096.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-13T14:29:15.502 DEBUG:teuthology.orchestra.run.smithi096:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:29:15.556 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-13T14:29:15.557 DEBUG:teuthology.orchestra.run.smithi096:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-13T14:29:15.612 INFO:tasks.workunit:timeout=3h 2024-09-13T14:29:15.612 INFO:tasks.workunit:cleanup=True 2024-09-13T14:29:15.613 DEBUG:teuthology.orchestra.run.smithi096:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-13T14:29:15.669 INFO:tasks.workunit.client.0.smithi096.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-13T14:29:15.736 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:15.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:15 smithi096 ceph-mon[29229]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:15.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:15 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2326648192' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:29:15.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:15 smithi096 ceph-mon[39645]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:15.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:15 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2326648192' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:29:15.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:15 smithi204 ceph-mon[32546]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:15.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:15 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2326648192' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:29:17.822 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:29:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:29:17] ENGINE Bus STOPPING 2024-09-13T14:29:17.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:17 smithi096 ceph-mon[29229]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:17.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:17 smithi096 ceph-mon[29229]: from='client.? ' entity='client.admin' 2024-09-13T14:29:17.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:17 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:29:17.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:17 smithi096 ceph-mon[39645]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:17.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:17 smithi096 ceph-mon[39645]: from='client.? ' entity='client.admin' 2024-09-13T14:29:17.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:17 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:29:17.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:17 smithi204 ceph-mon[32546]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:17.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:17 smithi204 ceph-mon[32546]: from='client.? ' entity='client.admin' 2024-09-13T14:29:17.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:17 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:29:18.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:29:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:29:17] "GET /metrics HTTP/1.1" 200 239663 "" "Prometheus/2.43.0" 2024-09-13T14:29:18.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:29:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:29:17] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:29:18.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:29:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:29:17] ENGINE Bus STOPPED 2024-09-13T14:29:18.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:29:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:29:17] ENGINE Bus STARTING 2024-09-13T14:29:18.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:29:18 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:29:18] ENGINE Serving on http://:::9283 2024-09-13T14:29:18.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:29:18 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:29:18] ENGINE Bus STARTED 2024-09-13T14:29:18.495 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-13T14:29:18.796 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:18.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:18 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:29:18.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:18 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:29:18.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:18 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:29:18.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:18 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:29:18.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:18 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:29:18.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:18 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:29:18.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:18 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:29:18.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:18 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:29:18.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:18 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:29:20.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:20 smithi096 ceph-mon[29229]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:20.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:20 smithi096 ceph-mon[39645]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:20.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:20 smithi204 ceph-mon[32546]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:20.873 DEBUG:teuthology.orchestra.run.smithi096:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-13T14:29:20.927 INFO:tasks.workunit.client.0.smithi096.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-13T14:29:20.930 INFO:tasks.workunit.client.0.smithi096.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T14:29:20.930 INFO:tasks.workunit.client.0.smithi096.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-13T14:29:21.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:21 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:21.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:21 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:21.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:21 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:22.198 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-13T14:29:22.452 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:22.467 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:22 smithi096 ceph-mon[29229]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:22.467 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:22 smithi096 ceph-mon[39645]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:22.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:22 smithi204 ceph-mon[32546]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:22.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:29:22 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:29:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:29:22.477 INFO:tasks.workunit.client.0.smithi096.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-13T14:29:22.603 INFO:tasks.workunit.client.0.smithi096.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-13T14:29:22.645 INFO:tasks.workunit.client.0.smithi096.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T14:29:22.646 INFO:tasks.workunit.client.0.smithi096.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T14:29:22.646 INFO:tasks.workunit.client.0.smithi096.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-13T14:29:22.685 INFO:tasks.workunit.client.0.smithi096.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T14:29:22.688 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T14:29:22.688 DEBUG:teuthology.orchestra.run.smithi096:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-13T14:29:22.704 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2024-09-13T14:29:22.705 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2024-09-13T14:29:22.705 DEBUG:teuthology.orchestra.run.smithi096:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2024-09-13T14:29:22.793 INFO:tasks.workunit.client.0.smithi096.stderr:+ export RBD_FORCE_ALLOW_V1=1 2024-09-13T14:29:22.793 INFO:tasks.workunit.client.0.smithi096.stderr:+ RBD_FORCE_ALLOW_V1=1 2024-09-13T14:29:22.793 INFO:tasks.workunit.client.0.smithi096.stderr:+ TMPDIR=/tmp/rbd_import_export_79210 2024-09-13T14:29:22.793 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm -rf /tmp/rbd_import_export_79210 2024-09-13T14:29:22.793 INFO:tasks.workunit.client.0.smithi096.stderr:+ mkdir /tmp/rbd_import_export_79210 2024-09-13T14:29:22.795 INFO:tasks.workunit.client.0.smithi096.stderr:+ trap 'rm -rf /tmp/rbd_import_export_79210' INT TERM EXIT 2024-09-13T14:29:22.795 INFO:tasks.workunit.client.0.smithi096.stderr:+ mkdir foo.79210 2024-09-13T14:29:22.795 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import foo.79210 foo.dir 2024-09-13T14:29:22.871 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: cannot import a directory 2024-09-13T14:29:22.871 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 0% complete...failed. 2024-09-13T14:29:22.871 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: import failed: (21) Is a directory 2024-09-13T14:29:22.876 INFO:tasks.workunit.client.0.smithi096.stderr:+ true 2024-09-13T14:29:22.877 INFO:tasks.workunit.client.0.smithi096.stderr:+ rmdir foo.79210 2024-09-13T14:29:22.909 INFO:tasks.workunit.client.0.smithi096.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_79210/img bs=1k count=1 seek=10 2024-09-13T14:29:22.910 INFO:tasks.workunit.client.0.smithi096.stderr:1+0 records in 2024-09-13T14:29:22.910 INFO:tasks.workunit.client.0.smithi096.stderr:1+0 records out 2024-09-13T14:29:22.910 INFO:tasks.workunit.client.0.smithi096.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.6135e-05 s, 22.2 MB/s 2024-09-13T14:29:22.910 INFO:tasks.workunit.client.0.smithi096.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_79210/img bs=1k count=10 seek=100 2024-09-13T14:29:22.911 INFO:tasks.workunit.client.0.smithi096.stderr:10+0 records in 2024-09-13T14:29:22.911 INFO:tasks.workunit.client.0.smithi096.stderr:10+0 records out 2024-09-13T14:29:22.911 INFO:tasks.workunit.client.0.smithi096.stderr:10240 bytes (10 kB, 10 KiB) copied, 4.9349e-05 s, 208 MB/s 2024-09-13T14:29:22.911 INFO:tasks.workunit.client.0.smithi096.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_79210/img bs=1k count=100 seek=1000 2024-09-13T14:29:22.911 INFO:tasks.workunit.client.0.smithi096.stderr:60+1 records in 2024-09-13T14:29:22.912 INFO:tasks.workunit.client.0.smithi096.stderr:60+1 records out 2024-09-13T14:29:22.912 INFO:tasks.workunit.client.0.smithi096.stderr:61640 bytes (62 kB, 60 KiB) copied, 0.000141642 s, 435 MB/s 2024-09-13T14:29:22.912 INFO:tasks.workunit.client.0.smithi096.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_79210/img bs=1k seek=10000 2024-09-13T14:29:22.913 INFO:tasks.workunit.client.0.smithi096.stderr:137+1 records in 2024-09-13T14:29:22.913 INFO:tasks.workunit.client.0.smithi096.stderr:137+1 records out 2024-09-13T14:29:22.913 INFO:tasks.workunit.client.0.smithi096.stderr:140928 bytes (141 kB, 138 KiB) copied, 0.000344173 s, 409 MB/s 2024-09-13T14:29:22.913 INFO:tasks.workunit.client.0.smithi096.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_79210/img bs=1k seek=100000 2024-09-13T14:29:22.914 INFO:tasks.workunit.client.0.smithi096.stderr:60+1 records in 2024-09-13T14:29:22.914 INFO:tasks.workunit.client.0.smithi096.stderr:60+1 records out 2024-09-13T14:29:22.914 INFO:tasks.workunit.client.0.smithi096.stderr:61592 bytes (62 kB, 60 KiB) copied, 0.000148914 s, 414 MB/s 2024-09-13T14:29:22.914 INFO:tasks.workunit.client.0.smithi096.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_79210/img bs=1k seek=1000000 2024-09-13T14:29:22.915 INFO:tasks.workunit.client.0.smithi096.stderr:154+1 records in 2024-09-13T14:29:22.915 INFO:tasks.workunit.client.0.smithi096.stderr:154+1 records out 2024-09-13T14:29:22.915 INFO:tasks.workunit.client.0.smithi096.stderr:158176 bytes (158 kB, 154 KiB) copied, 0.000361689 s, 437 MB/s 2024-09-13T14:29:22.915 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd rm testimg 2024-09-13T14:29:22.952 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 0% complete...failed. 2024-09-13T14:29:22.952 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: delete error: (2) No such file or directory 2024-09-13T14:29:22.954 INFO:tasks.workunit.client.0.smithi096.stderr:+ true 2024-09-13T14:29:22.954 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --new-format /tmp/rbd_import_export_79210/img testimg 2024-09-13T14:29:23.481 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2024-09-13T14:29:23.678 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:29:23.678 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export testimg /tmp/rbd_import_export_79210/img2 2024-09-13T14:29:23.730 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-13T14:29:23.735 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export testimg - 2024-09-13T14:29:24.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:24 smithi096 ceph-mon[29229]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:24.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:24 smithi096 ceph-mon[39645]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:24.448 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-13T14:29:24.449 INFO:teuthology.orchestra.run.smithi096.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:29:24.452 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd rm testimg 2024-09-13T14:29:24.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:24 smithi204 ceph-mon[32546]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:24.543 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-13T14:29:24.546 INFO:tasks.workunit.client.0.smithi096.stderr:+ cmp /tmp/rbd_import_export_79210/img /tmp/rbd_import_export_79210/img2 2024-09-13T14:29:25.105 INFO:tasks.workunit.client.0.smithi096.stderr:+ cmp /tmp/rbd_import_export_79210/img /tmp/rbd_import_export_79210/img3 2024-09-13T14:29:25.244 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:29:25.526 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:25.533 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/img2 /tmp/rbd_import_export_79210/img3 2024-09-13T14:29:25.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[29229]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:25.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[29229]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:29:25.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:29:25.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:29:25.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:29:25.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:29:25.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:29:25.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[29229]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:29:25.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[29229]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:25.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[39645]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:25.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[39645]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:29:25.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:29:25.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:29:25.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:29:25.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:29:25.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:29:25.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[39645]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:29:25.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:25 smithi096 ceph-mon[39645]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:25.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:25 smithi204 ceph-mon[32546]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:25.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:25 smithi204 ceph-mon[32546]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:29:25.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:25 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:29:25.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:25 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:29:25.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:25 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:29:25.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:25 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:29:25.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:25 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:29:25.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:25 smithi204 ceph-mon[32546]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:29:25.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:25 smithi204 ceph-mon[32546]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:25.728 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --new-format - testimg 2024-09-13T14:29:26.132 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 100% complete...done. 2024-09-13T14:29:26.133 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:29:26.136 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export testimg /tmp/rbd_import_export_79210/img2 2024-09-13T14:29:26.426 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-13T14:29:26.429 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export testimg - 2024-09-13T14:29:27.178 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-13T14:29:27.181 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd rm testimg 2024-09-13T14:29:27.273 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-13T14:29:27.276 INFO:tasks.workunit.client.0.smithi096.stderr:+ cmp /tmp/rbd_import_export_79210/img /tmp/rbd_import_export_79210/img2 2024-09-13T14:29:27.538 INFO:teuthology.orchestra.run.smithi096.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:29:27.538 INFO:teuthology.orchestra.run.smithi096.stdout:alertmanager.a smithi096 *:9093,9094 running (84s) 55s ago 2m 16.9M - 0.25.0 c8568f914cd2 c2fb084ac3b6 2024-09-13T14:29:27.538 INFO:teuthology.orchestra.run.smithi096.stdout:grafana.a smithi204 *:3000 running (2m) 58s ago 2m 84.9M - 9.4.7 954c08fa6188 a693d43fc847 2024-09-13T14:29:27.538 INFO:teuthology.orchestra.run.smithi096.stdout:mds.foo.smithi096.dqiqpz smithi096 running (71s) 55s ago 71s 16.1M - 17.2.7-1627-gbbdadfa3 a994b00deb3a ab8d33048bb7 2024-09-13T14:29:27.538 INFO:teuthology.orchestra.run.smithi096.stdout:mds.foo.smithi204.hbwbot smithi204 running (69s) 58s ago 68s 14.3M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 97a3ff635de2 2024-09-13T14:29:27.538 INFO:teuthology.orchestra.run.smithi096.stdout:mgr.x smithi204 *:8443 running (6m) 58s ago 6m 401M - 17.2.7-1627-gbbdadfa3 a994b00deb3a f7bdd6b72e4d 2024-09-13T14:29:27.539 INFO:teuthology.orchestra.run.smithi096.stdout:mgr.y smithi096 *:9283,8765 running (7m) 55s ago 7m 443M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 7acb3784c92c 2024-09-13T14:29:27.539 INFO:teuthology.orchestra.run.smithi096.stdout:mon.a smithi096 running (8m) 55s ago 8m 44.0M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 96e95ed67f92 2024-09-13T14:29:27.539 INFO:teuthology.orchestra.run.smithi096.stdout:mon.b smithi204 running (6m) 58s ago 6m 36.6M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a a37b0845979d 2024-09-13T14:29:27.539 INFO:teuthology.orchestra.run.smithi096.stdout:mon.c smithi096 running (6m) 55s ago 6m 35.9M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 44a18794779c 2024-09-13T14:29:27.539 INFO:teuthology.orchestra.run.smithi096.stdout:node-exporter.a smithi096 *:9100 running (2m) 55s ago 119s 16.1M - 1.5.0 0da6a335fe13 4e1385024205 2024-09-13T14:29:27.539 INFO:teuthology.orchestra.run.smithi096.stdout:node-exporter.b smithi204 *:9100 running (116s) 58s ago 116s 8158k - 1.5.0 0da6a335fe13 0fc850b0a1c6 2024-09-13T14:29:27.539 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0 smithi096 running (6m) 55s ago 6m 42.0M 4016M 17.2.7-1627-gbbdadfa3 a994b00deb3a a35fe012b65d 2024-09-13T14:29:27.540 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1 smithi096 running (5m) 55s ago 5m 42.4M 4016M 17.2.7-1627-gbbdadfa3 a994b00deb3a edbcf0f2ed44 2024-09-13T14:29:27.540 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2 smithi096 running (5m) 55s ago 5m 40.0M 4016M 17.2.7-1627-gbbdadfa3 a994b00deb3a 565505fd2e48 2024-09-13T14:29:27.540 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3 smithi096 running (4m) 55s ago 4m 41.0M 4016M 17.2.7-1627-gbbdadfa3 a994b00deb3a afd4dcb32d2d 2024-09-13T14:29:27.540 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4 smithi204 running (4m) 58s ago 4m 41.2M 4016M 17.2.7-1627-gbbdadfa3 a994b00deb3a b48920b2b9e7 2024-09-13T14:29:27.540 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5 smithi204 running (4m) 58s ago 4m 41.3M 4016M 17.2.7-1627-gbbdadfa3 a994b00deb3a 0ed6e334d2ea 2024-09-13T14:29:27.540 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6 smithi204 running (3m) 58s ago 3m 40.9M 4016M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8f12f56d751c 2024-09-13T14:29:27.541 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7 smithi204 running (3m) 58s ago 3m 42.1M 4016M 17.2.7-1627-gbbdadfa3 a994b00deb3a ffd75c8be38a 2024-09-13T14:29:27.541 INFO:teuthology.orchestra.run.smithi096.stdout:prometheus.a smithi204 *:9095 running (79s) 58s ago 2m 32.7M - 2.43.0 a07b618ecd1d 420af5726f6f 2024-09-13T14:29:27.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:27 smithi096 ceph-mon[29229]: pgmap v95: 73 pgs: 73 active+clean; 5.9 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 77 KiB/s wr, 54 op/s 2024-09-13T14:29:27.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:27 smithi096 ceph-mon[39645]: pgmap v95: 73 pgs: 73 active+clean; 5.9 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 77 KiB/s wr, 54 op/s 2024-09-13T14:29:27.875 INFO:tasks.workunit.client.0.smithi096.stderr:+ cmp /tmp/rbd_import_export_79210/img /tmp/rbd_import_export_79210/img3 2024-09-13T14:29:27.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:27 smithi204 ceph-mon[32546]: pgmap v95: 73 pgs: 73 active+clean; 5.9 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 77 KiB/s wr, 54 op/s 2024-09-13T14:29:28.226 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:29:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:29:27] "GET /metrics HTTP/1.1" 200 239635 "" "Prometheus/2.43.0" 2024-09-13T14:29:28.256 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for minority of mons to upgrade' 2024-09-13T14:29:28.309 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/img /tmp/rbd_import_export_79210/img2 /tmp/rbd_import_export_79210/img3 2024-09-13T14:29:28.536 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:28.568 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd help export 2024-09-13T14:29:28.568 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep -q export-format 2024-09-13T14:29:28.589 INFO:tasks.workunit.client.0.smithi096.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_79210/img bs=1k count=10 seek=100 2024-09-13T14:29:28.589 INFO:tasks.workunit.client.0.smithi096.stderr:10+0 records in 2024-09-13T14:29:28.589 INFO:tasks.workunit.client.0.smithi096.stderr:10+0 records out 2024-09-13T14:29:28.589 INFO:tasks.workunit.client.0.smithi096.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.9701e-05 s, 172 MB/s 2024-09-13T14:29:28.590 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --new-format /tmp/rbd_import_export_79210/img testimg 2024-09-13T14:29:28.647 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 100% complete...done. 2024-09-13T14:29:28.648 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:29:28.650 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd snap create testimg@snap 2024-09-13T14:29:28.913 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:28 smithi096 ceph-mon[29229]: from='client.14859 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:28.913 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:28 smithi096 ceph-mon[39645]: from='client.14859 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:28.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:28 smithi204 ceph-mon[32546]: from='client.14859 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:29.635 INFO:tasks.workunit.client.0.smithi096.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-09-13T14:29:29.641 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd image-meta set testimg key1 value1 2024-09-13T14:29:29.694 INFO:tasks.workunit.client.0.smithi096.stderr:++ rbd image-meta list testimg 2024-09-13T14:29:29.735 INFO:tasks.workunit.client.0.smithi096.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2024-09-13T14:29:29.736 INFO:tasks.workunit.client.0.smithi096.stderr: 2024-09-13T14:29:29.736 INFO:tasks.workunit.client.0.smithi096.stderr:Key Value 2024-09-13T14:29:29.736 INFO:tasks.workunit.client.0.smithi096.stderr:key1 value1' 2024-09-13T14:29:29.736 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_79210/img_v2 2024-09-13T14:29:29.778 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-13T14:29:29.781 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_79210/img_v2 testimg_import 2024-09-13T14:29:29.948 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:29 smithi096 ceph-mon[29229]: pgmap v96: 73 pgs: 73 active+clean; 5.9 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 77 KiB/s wr, 54 op/s 2024-09-13T14:29:29.948 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:29 smithi096 ceph-mon[39645]: pgmap v96: 73 pgs: 73 active+clean; 5.9 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 77 KiB/s wr, 54 op/s 2024-09-13T14:29:29.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:29 smithi204 ceph-mon[32546]: pgmap v96: 73 pgs: 73 active+clean; 5.9 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 77 KiB/s wr, 54 op/s 2024-09-13T14:29:30.364 INFO:teuthology.orchestra.run.smithi096.stdout:wait for minority of mons to upgrade 2024-09-13T14:29:30.647 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-13T14:29:30.651 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd info testimg_import 2024-09-13T14:29:30.686 INFO:tasks.workunit.client.0.smithi096.stdout:rbd image 'testimg_import': 2024-09-13T14:29:30.686 INFO:tasks.workunit.client.0.smithi096.stdout: size 110 KiB in 1 objects 2024-09-13T14:29:30.686 INFO:tasks.workunit.client.0.smithi096.stdout: order 22 (4 MiB objects) 2024-09-13T14:29:30.686 INFO:tasks.workunit.client.0.smithi096.stdout: snapshot_count: 1 2024-09-13T14:29:30.686 INFO:tasks.workunit.client.0.smithi096.stdout: id: 60e0a45739aa 2024-09-13T14:29:30.687 INFO:tasks.workunit.client.0.smithi096.stdout: block_name_prefix: rbd_data.60e0a45739aa 2024-09-13T14:29:30.687 INFO:tasks.workunit.client.0.smithi096.stdout: format: 2 2024-09-13T14:29:30.687 INFO:tasks.workunit.client.0.smithi096.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-09-13T14:29:30.687 INFO:tasks.workunit.client.0.smithi096.stdout: op_features: 2024-09-13T14:29:30.687 INFO:tasks.workunit.client.0.smithi096.stdout: flags: 2024-09-13T14:29:30.687 INFO:tasks.workunit.client.0.smithi096.stdout: create_timestamp: Fri Sep 13 14:29:29 2024 2024-09-13T14:29:30.687 INFO:tasks.workunit.client.0.smithi096.stdout: access_timestamp: Fri Sep 13 14:29:29 2024 2024-09-13T14:29:30.687 INFO:tasks.workunit.client.0.smithi096.stdout: modify_timestamp: Fri Sep 13 14:29:29 2024 2024-09-13T14:29:30.689 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd info testimg_import@snap 2024-09-13T14:29:30.724 INFO:tasks.workunit.client.0.smithi096.stdout:rbd image 'testimg_import': 2024-09-13T14:29:30.724 INFO:tasks.workunit.client.0.smithi096.stdout: size 110 KiB in 1 objects 2024-09-13T14:29:30.724 INFO:tasks.workunit.client.0.smithi096.stdout: order 22 (4 MiB objects) 2024-09-13T14:29:30.724 INFO:tasks.workunit.client.0.smithi096.stdout: snapshot_count: 1 2024-09-13T14:29:30.724 INFO:tasks.workunit.client.0.smithi096.stdout: id: 60e0a45739aa 2024-09-13T14:29:30.724 INFO:tasks.workunit.client.0.smithi096.stdout: block_name_prefix: rbd_data.60e0a45739aa 2024-09-13T14:29:30.725 INFO:tasks.workunit.client.0.smithi096.stdout: format: 2 2024-09-13T14:29:30.725 INFO:tasks.workunit.client.0.smithi096.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-09-13T14:29:30.725 INFO:tasks.workunit.client.0.smithi096.stdout: op_features: 2024-09-13T14:29:30.725 INFO:tasks.workunit.client.0.smithi096.stdout: flags: 2024-09-13T14:29:30.725 INFO:tasks.workunit.client.0.smithi096.stdout: create_timestamp: Fri Sep 13 14:29:29 2024 2024-09-13T14:29:30.725 INFO:tasks.workunit.client.0.smithi096.stdout: access_timestamp: Fri Sep 13 14:29:29 2024 2024-09-13T14:29:30.725 INFO:tasks.workunit.client.0.smithi096.stdout: modify_timestamp: Fri Sep 13 14:29:29 2024 2024-09-13T14:29:30.725 INFO:tasks.workunit.client.0.smithi096.stdout: protected: False 2024-09-13T14:29:30.727 INFO:tasks.workunit.client.0.smithi096.stderr:++ rbd image-meta list testimg_import 2024-09-13T14:29:30.767 INFO:tasks.workunit.client.0.smithi096.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2024-09-13T14:29:30.767 INFO:tasks.workunit.client.0.smithi096.stderr: 2024-09-13T14:29:30.767 INFO:tasks.workunit.client.0.smithi096.stderr:Key Value 2024-09-13T14:29:30.767 INFO:tasks.workunit.client.0.smithi096.stderr:key1 value1' 2024-09-13T14:29:30.768 INFO:tasks.workunit.client.0.smithi096.stderr:+ '[' 'There is 1 metadatum on this image: 2024-09-13T14:29:30.768 INFO:tasks.workunit.client.0.smithi096.stderr: 2024-09-13T14:29:30.768 INFO:tasks.workunit.client.0.smithi096.stderr:Key Value 2024-09-13T14:29:30.768 INFO:tasks.workunit.client.0.smithi096.stderr:key1 value1' = 'There is 1 metadatum on this image: 2024-09-13T14:29:30.768 INFO:tasks.workunit.client.0.smithi096.stderr: 2024-09-13T14:29:30.768 INFO:tasks.workunit.client.0.smithi096.stderr:Key Value 2024-09-13T14:29:30.768 INFO:tasks.workunit.client.0.smithi096.stderr:key1 value1' ']' 2024-09-13T14:29:30.768 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export testimg_import /tmp/rbd_import_export_79210/img_import 2024-09-13T14:29:30.803 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 100% complete...done. 2024-09-13T14:29:30.806 INFO:tasks.workunit.client.0.smithi096.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_79210/img /tmp/rbd_import_export_79210/img_import 2024-09-13T14:29:30.806 INFO:tasks.workunit.client.0.smithi096.stderr:+ cmp -l /tmp/rbd_import_export_79210/img /tmp/rbd_import_export_79210/img_import 2024-09-13T14:29:30.807 INFO:tasks.workunit.client.0.smithi096.stderr:++ stat /tmp/rbd_import_export_79210/img --format %b 2024-09-13T14:29:30.809 INFO:tasks.workunit.client.0.smithi096.stderr:+ origsize=24 2024-09-13T14:29:30.809 INFO:tasks.workunit.client.0.smithi096.stderr:++ stat /tmp/rbd_import_export_79210/img_import --format %b 2024-09-13T14:29:30.810 INFO:tasks.workunit.client.0.smithi096.stderr:+ exportsize=224 2024-09-13T14:29:30.810 INFO:tasks.workunit.client.0.smithi096.stderr:+ difference=200 2024-09-13T14:29:30.810 INFO:tasks.workunit.client.0.smithi096.stderr:+ difference=200 2024-09-13T14:29:30.810 INFO:tasks.workunit.client.0.smithi096.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2024-09-13T14:29:30.810 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_79210/img_snap 2024-09-13T14:29:30.845 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 100% complete...done. 2024-09-13T14:29:30.848 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_79210/img_snap_import 2024-09-13T14:29:30.885 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 100% complete...done. 2024-09-13T14:29:30.888 INFO:tasks.workunit.client.0.smithi096.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_79210/img_snap /tmp/rbd_import_export_79210/img_snap_import 2024-09-13T14:29:30.888 INFO:tasks.workunit.client.0.smithi096.stderr:+ cmp -l /tmp/rbd_import_export_79210/img_snap /tmp/rbd_import_export_79210/img_snap_import 2024-09-13T14:29:30.889 INFO:tasks.workunit.client.0.smithi096.stderr:++ stat /tmp/rbd_import_export_79210/img_snap --format %b 2024-09-13T14:29:30.890 INFO:tasks.workunit.client.0.smithi096.stderr:+ origsize=224 2024-09-13T14:29:30.890 INFO:tasks.workunit.client.0.smithi096.stderr:++ stat /tmp/rbd_import_export_79210/img_snap_import --format %b 2024-09-13T14:29:30.891 INFO:tasks.workunit.client.0.smithi096.stderr:+ exportsize=224 2024-09-13T14:29:30.891 INFO:tasks.workunit.client.0.smithi096.stderr:+ difference=0 2024-09-13T14:29:30.891 INFO:tasks.workunit.client.0.smithi096.stderr:+ difference=0 2024-09-13T14:29:30.892 INFO:tasks.workunit.client.0.smithi096.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-13T14:29:30.892 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/img_v2 2024-09-13T14:29:30.892 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/img_import 2024-09-13T14:29:30.893 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/img_snap 2024-09-13T14:29:30.894 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/img_snap_import 2024-09-13T14:29:30.895 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd snap rm testimg_import@snap 2024-09-13T14:29:30.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:30 smithi204 ceph-mon[32546]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T14:29:31.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:30 smithi096 ceph-mon[29229]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T14:29:31.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:30 smithi096 ceph-mon[39645]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T14:29:31.153 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-13T14:29:31.409 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:31.643 INFO:tasks.workunit.client.0.smithi096.stderr: Removing snap: 100% complete...done. 2024-09-13T14:29:31.650 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd remove testimg_import 2024-09-13T14:29:31.744 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 100% complete...done. 2024-09-13T14:29:31.748 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd snap rm testimg@snap 2024-09-13T14:29:31.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:31 smithi096 ceph-mon[29229]: pgmap v98: 73 pgs: 73 active+clean; 5.9 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 206 KiB/s rd, 93 KiB/s wr, 65 op/s 2024-09-13T14:29:31.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:31 smithi096 ceph-mon[29229]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T14:29:31.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:31 smithi096 ceph-mon[39645]: pgmap v98: 73 pgs: 73 active+clean; 5.9 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 206 KiB/s rd, 93 KiB/s wr, 65 op/s 2024-09-13T14:29:31.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:31 smithi096 ceph-mon[39645]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T14:29:31.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:31 smithi204 ceph-mon[32546]: pgmap v98: 73 pgs: 73 active+clean; 5.9 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 206 KiB/s rd, 93 KiB/s wr, 65 op/s 2024-09-13T14:29:31.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:31 smithi204 ceph-mon[32546]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T14:29:32.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:29:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:29:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:29:32.657 INFO:tasks.workunit.client.0.smithi096.stderr: Removing snap: 100% complete...done. 2024-09-13T14:29:32.665 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd rm testimg 2024-09-13T14:29:32.759 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 100% complete...done. 2024-09-13T14:29:32.762 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --order 20 /tmp/rbd_import_export_79210/img testimg 2024-09-13T14:29:32.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:32 smithi096 ceph-mon[29229]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T14:29:32.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:32 smithi096 ceph-mon[39645]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T14:29:32.826 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 100% complete...done. 2024-09-13T14:29:32.827 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T14:29:32.829 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_79210/img_v2 2024-09-13T14:29:32.870 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-13T14:29:32.873 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_79210/img_v2 testimg_import 2024-09-13T14:29:32.936 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-13T14:29:32.939 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd info testimg_import 2024-09-13T14:29:32.939 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep order 2024-09-13T14:29:32.939 INFO:tasks.workunit.client.0.smithi096.stderr:+ awk '{print $2}' 2024-09-13T14:29:32.939 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep 20 2024-09-13T14:29:32.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:32 smithi204 ceph-mon[32546]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T14:29:32.981 INFO:tasks.workunit.client.0.smithi096.stdout:20 2024-09-13T14:29:32.981 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/img_v2 2024-09-13T14:29:32.981 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd remove testimg_import 2024-09-13T14:29:33.089 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 100% complete...done. 2024-09-13T14:29:33.104 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd remove testimg 2024-09-13T14:29:33.195 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 100% complete...done. 2024-09-13T14:29:33.198 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_79210/img testimg 2024-09-13T14:29:33.252 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 100% complete...done. 2024-09-13T14:29:33.255 INFO:tasks.workunit.client.0.smithi096.stderr:++ rbd info testimg 2024-09-13T14:29:33.256 INFO:tasks.workunit.client.0.smithi096.stderr:++ grep features 2024-09-13T14:29:33.297 INFO:tasks.workunit.client.0.smithi096.stderr:+ FEATURES_BEFORE=' features: layering 2024-09-13T14:29:33.297 INFO:tasks.workunit.client.0.smithi096.stderr: op_features: ' 2024-09-13T14:29:33.298 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_79210/img_v2 2024-09-13T14:29:33.337 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-13T14:29:33.340 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_79210/img_v2 testimg_import 2024-09-13T14:29:33.397 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-13T14:29:33.402 INFO:tasks.workunit.client.0.smithi096.stderr:++ rbd info testimg_import 2024-09-13T14:29:33.402 INFO:tasks.workunit.client.0.smithi096.stderr:++ grep features 2024-09-13T14:29:33.441 INFO:tasks.workunit.client.0.smithi096.stderr:+ FEATURES_AFTER=' features: layering 2024-09-13T14:29:33.442 INFO:tasks.workunit.client.0.smithi096.stderr: op_features: ' 2024-09-13T14:29:33.442 INFO:tasks.workunit.client.0.smithi096.stderr:+ '[' ' features: layering 2024-09-13T14:29:33.442 INFO:tasks.workunit.client.0.smithi096.stderr: op_features: ' '!=' ' features: layering 2024-09-13T14:29:33.442 INFO:tasks.workunit.client.0.smithi096.stderr: op_features: ' ']' 2024-09-13T14:29:33.442 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/img_v2 2024-09-13T14:29:33.443 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd remove testimg_import 2024-09-13T14:29:33.514 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 100% complete...done. 2024-09-13T14:29:33.518 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd remove testimg 2024-09-13T14:29:33.587 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 100% complete...done. 2024-09-13T14:29:33.591 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_79210/img testimg 2024-09-13T14:29:33.658 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 100% complete...done. 2024-09-13T14:29:33.661 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_79210/img_v2 2024-09-13T14:29:33.708 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-13T14:29:33.712 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_79210/img_v2 testimg_import 2024-09-13T14:29:33.811 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-13T14:29:33.816 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd info testimg_import 2024-09-13T14:29:33.816 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep 'stripe unit' 2024-09-13T14:29:33.817 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep -Ei '(4 KiB|4096)' 2024-09-13T14:29:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:33 smithi096 ceph-mon[29229]: pgmap v101: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 133 KiB/s rd, 9.2 KiB/s wr, 134 op/s 2024-09-13T14:29:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:33 smithi096 ceph-mon[29229]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T14:29:33.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:33 smithi096 ceph-mon[39645]: pgmap v101: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 133 KiB/s rd, 9.2 KiB/s wr, 134 op/s 2024-09-13T14:29:33.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:33 smithi096 ceph-mon[39645]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T14:29:33.855 INFO:tasks.workunit.client.0.smithi096.stdout: stripe unit: 4 KiB 2024-09-13T14:29:33.855 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd info testimg_import 2024-09-13T14:29:33.855 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep 'stripe count' 2024-09-13T14:29:33.855 INFO:tasks.workunit.client.0.smithi096.stderr:+ awk '{print $3}' 2024-09-13T14:29:33.856 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep 1000 2024-09-13T14:29:33.899 INFO:tasks.workunit.client.0.smithi096.stdout:1000 2024-09-13T14:29:33.899 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/img_v2 2024-09-13T14:29:33.900 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd remove testimg_import 2024-09-13T14:29:33.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:33 smithi204 ceph-mon[32546]: pgmap v101: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 133 KiB/s rd, 9.2 KiB/s wr, 134 op/s 2024-09-13T14:29:33.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:33 smithi204 ceph-mon[32546]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T14:29:34.000 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-13T14:29:34.001 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd remove testimg 2024-09-13T14:29:34.101 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-13T14:29:34.106 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_79210/img testimg 2024-09-13T14:29:34.161 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 100% complete...done. 2024-09-13T14:29:34.163 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd snap create testimg@snap1 2024-09-13T14:29:34.270 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:29:34.667 INFO:tasks.workunit.client.0.smithi096.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-09-13T14:29:34.673 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd snap create testimg@snap2 2024-09-13T14:29:34.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:34 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1113067287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:35.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:34 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1113067287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:35.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:34 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1113067287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:35.676 INFO:tasks.workunit.client.0.smithi096.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-09-13T14:29:35.682 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd snap protect testimg@snap2 2024-09-13T14:29:35.724 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_79210/snap_protect 2024-09-13T14:29:35.764 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-13T14:29:35.767 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_79210/snap_protect testimg_import 2024-09-13T14:29:35.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:35 smithi204 ceph-mon[32546]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:35.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:35 smithi204 ceph-mon[32546]: from='client.25010 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:35 smithi204 ceph-mon[32546]: pgmap v103: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 11 KiB/s wr, 163 op/s 2024-09-13T14:29:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:35 smithi204 ceph-mon[32546]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T14:29:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:35 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:36.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:35 smithi096 ceph-mon[29229]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:36.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:35 smithi096 ceph-mon[29229]: from='client.25010 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:36.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:35 smithi096 ceph-mon[29229]: pgmap v103: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 11 KiB/s wr, 163 op/s 2024-09-13T14:29:36.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:35 smithi096 ceph-mon[29229]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T14:29:36.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:35 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:35 smithi096 ceph-mon[39645]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:35 smithi096 ceph-mon[39645]: from='client.25010 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:35 smithi096 ceph-mon[39645]: pgmap v103: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 11 KiB/s wr, 163 op/s 2024-09-13T14:29:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:35 smithi096 ceph-mon[39645]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T14:29:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:35 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:36.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:36 smithi204 ceph-mon[32546]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T14:29:37.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:36 smithi096 ceph-mon[29229]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T14:29:37.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:36 smithi096 ceph-mon[39645]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T14:29:37.185 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:29:37.692 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-13T14:29:37.695 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd info testimg_import@snap1 2024-09-13T14:29:37.695 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep 'protected: False' 2024-09-13T14:29:37.733 INFO:tasks.workunit.client.0.smithi096.stdout: protected: False 2024-09-13T14:29:37.734 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd info testimg_import@snap2 2024-09-13T14:29:37.734 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep 'protected: True' 2024-09-13T14:29:37.774 INFO:tasks.workunit.client.0.smithi096.stdout: protected: True 2024-09-13T14:29:37.774 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/snap_protect 2024-09-13T14:29:37.775 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd snap unprotect testimg@snap2 2024-09-13T14:29:37.821 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd snap unprotect testimg_import@snap2 2024-09-13T14:29:37.864 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd snap purge testimg 2024-09-13T14:29:37.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:37 smithi204 ceph-mon[32546]: pgmap v106: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 340 KiB/s rd, 41 KiB/s wr, 456 op/s 2024-09-13T14:29:37.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:37 smithi204 ceph-mon[32546]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T14:29:37.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:37 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3379484999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:38.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:29:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:29:37] "GET /metrics HTTP/1.1" 200 240120 "" "Prometheus/2.43.0" 2024-09-13T14:29:38.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:37 smithi096 ceph-mon[29229]: pgmap v106: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 340 KiB/s rd, 41 KiB/s wr, 456 op/s 2024-09-13T14:29:38.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:37 smithi096 ceph-mon[29229]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T14:29:38.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:37 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3379484999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:38.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:37 smithi096 ceph-mon[39645]: pgmap v106: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 340 KiB/s rd, 41 KiB/s wr, 456 op/s 2024-09-13T14:29:38.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:37 smithi096 ceph-mon[39645]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T14:29:38.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:37 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3379484999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:38.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:38 smithi204 ceph-mon[32546]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:38.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:38 smithi204 ceph-mon[32546]: from='client.25052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:38.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:38 smithi204 ceph-mon[32546]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T14:29:39.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:38 smithi096 ceph-mon[29229]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:39.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:38 smithi096 ceph-mon[29229]: from='client.25052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:39.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:38 smithi096 ceph-mon[29229]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T14:29:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:38 smithi096 ceph-mon[39645]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:38 smithi096 ceph-mon[39645]: from='client.25052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:38 smithi096 ceph-mon[39645]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T14:29:39.691 INFO:tasks.workunit.client.0.smithi096.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-09-13T14:29:39.699 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd snap purge testimg_import 2024-09-13T14:29:39.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:39 smithi204 ceph-mon[32546]: pgmap v109: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 418 KiB/s rd, 51 KiB/s wr, 562 op/s 2024-09-13T14:29:39.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:39 smithi204 ceph-mon[32546]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T14:29:40.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:39 smithi096 ceph-mon[29229]: pgmap v109: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 418 KiB/s rd, 51 KiB/s wr, 562 op/s 2024-09-13T14:29:40.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:39 smithi096 ceph-mon[29229]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T14:29:40.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:39 smithi096 ceph-mon[39645]: pgmap v109: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 418 KiB/s rd, 51 KiB/s wr, 562 op/s 2024-09-13T14:29:40.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:39 smithi096 ceph-mon[39645]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T14:29:40.086 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:29:40.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:40 smithi204 ceph-mon[32546]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T14:29:40.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:40 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2330707926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:40.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:40 smithi204 ceph-mon[32546]: from='client.15195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:41.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:40 smithi096 ceph-mon[29229]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T14:29:41.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:40 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2330707926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:41.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:40 smithi096 ceph-mon[29229]: from='client.15195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:41.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:40 smithi096 ceph-mon[39645]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T14:29:41.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:40 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2330707926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:41.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:40 smithi096 ceph-mon[39645]: from='client.15195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:41.277 INFO:tasks.workunit.client.0.smithi096.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-09-13T14:29:41.284 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd remove testimg 2024-09-13T14:29:41.372 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 100% complete...done. 2024-09-13T14:29:41.375 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd remove testimg_import 2024-09-13T14:29:41.458 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 100% complete...done. 2024-09-13T14:29:41.461 INFO:tasks.workunit.client.0.smithi096.stderr:+ tiered=0 2024-09-13T14:29:41.461 INFO:tasks.workunit.client.0.smithi096.stderr:+ ceph osd dump 2024-09-13T14:29:41.461 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep '^pool' 2024-09-13T14:29:41.462 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep tier 2024-09-13T14:29:41.462 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep ''\''rbd'\''' 2024-09-13T14:29:41.844 INFO:tasks.workunit.client.0.smithi096.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_79210/sparse1 2024-09-13T14:29:41.848 INFO:tasks.workunit.client.0.smithi096.stderr:1+0 records in 2024-09-13T14:29:41.849 INFO:tasks.workunit.client.0.smithi096.stderr:1+0 records out 2024-09-13T14:29:41.849 INFO:tasks.workunit.client.0.smithi096.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00344644 s, 304 MB/s 2024-09-13T14:29:41.849 INFO:tasks.workunit.client.0.smithi096.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_79210/sparse2 2024-09-13T14:29:41.853 INFO:tasks.workunit.client.0.smithi096.stderr:1+0 records in 2024-09-13T14:29:41.853 INFO:tasks.workunit.client.0.smithi096.stderr:1+0 records out 2024-09-13T14:29:41.853 INFO:tasks.workunit.client.0.smithi096.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00333885 s, 314 MB/s 2024-09-13T14:29:41.853 INFO:tasks.workunit.client.0.smithi096.stderr:+ truncate /tmp/rbd_import_export_79210/sparse2 -s 2M 2024-09-13T14:29:41.859 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd rm sparse1 2024-09-13T14:29:41.898 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 0% complete...failed. 2024-09-13T14:29:41.898 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: delete error: (2) No such file or directory 2024-09-13T14:29:41.901 INFO:tasks.workunit.client.0.smithi096.stderr:+ true 2024-09-13T14:29:41.901 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_79210/sparse1 2024-09-13T14:29:41.959 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-09-13T14:29:41.959 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:29:41.959 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T14:29:41.962 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd ls -l 2024-09-13T14:29:41.962 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep sparse1 2024-09-13T14:29:41.963 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-09-13T14:29:41.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:41 smithi204 ceph-mon[32546]: from='client.15201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:41.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:41 smithi204 ceph-mon[32546]: pgmap v112: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:41.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:41 smithi204 ceph-mon[32546]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T14:29:41.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:41 smithi204 ceph-mon[32546]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T14:29:42.003 INFO:tasks.workunit.client.0.smithi096.stdout:sparse1 2 MiB 2 2024-09-13T14:29:42.004 INFO:tasks.workunit.client.0.smithi096.stderr:++ objects sparse1 2024-09-13T14:29:42.004 INFO:tasks.workunit.client.0.smithi096.stderr:++ image=sparse1 2024-09-13T14:29:42.004 INFO:tasks.workunit.client.0.smithi096.stderr:+++ rbd info sparse1 2024-09-13T14:29:42.004 INFO:tasks.workunit.client.0.smithi096.stderr:+++ grep block_name_prefix 2024-09-13T14:29:42.004 INFO:tasks.workunit.client.0.smithi096.stderr:+++ awk '{print $NF;}' 2024-09-13T14:29:42.040 INFO:tasks.workunit.client.0.smithi096.stderr:++ prefix=rbd_data.3b7f3cbc3f65 2024-09-13T14:29:42.041 INFO:tasks.workunit.client.0.smithi096.stderr:+++ grep rbd_data.3b7f3cbc3f65 2024-09-13T14:29:42.041 INFO:tasks.workunit.client.0.smithi096.stderr:++++ get_image_data_pool sparse1 2024-09-13T14:29:42.041 INFO:tasks.workunit.client.0.smithi096.stderr:++++ image=sparse1 2024-09-13T14:29:42.041 INFO:tasks.workunit.client.0.smithi096.stderr:+++ sed -e 's/rbd_data.3b7f3cbc3f65\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-13T14:29:42.041 INFO:tasks.workunit.client.0.smithi096.stderr:+++ sort -u 2024-09-13T14:29:42.041 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ rbd info sparse1 2024-09-13T14:29:42.042 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ grep 'data_pool: ' 2024-09-13T14:29:42.042 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ awk -F: '{ print $NF }' 2024-09-13T14:29:42.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:41 smithi096 ceph-mon[29229]: from='client.15201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:42.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:41 smithi096 ceph-mon[29229]: pgmap v112: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:42.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:41 smithi096 ceph-mon[29229]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T14:29:42.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:41 smithi096 ceph-mon[29229]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T14:29:42.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:41 smithi096 ceph-mon[39645]: from='client.15201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:42.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:41 smithi096 ceph-mon[39645]: pgmap v112: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:42.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:41 smithi096 ceph-mon[39645]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T14:29:42.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:41 smithi096 ceph-mon[39645]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T14:29:42.080 INFO:tasks.workunit.client.0.smithi096.stderr:++++ data_pool= 2024-09-13T14:29:42.080 INFO:tasks.workunit.client.0.smithi096.stderr:++++ '[' -z ']' 2024-09-13T14:29:42.080 INFO:tasks.workunit.client.0.smithi096.stderr:++++ data_pool=rbd 2024-09-13T14:29:42.080 INFO:tasks.workunit.client.0.smithi096.stderr:++++ echo rbd 2024-09-13T14:29:42.080 INFO:tasks.workunit.client.0.smithi096.stderr:+++ rados ls -p rbd 2024-09-13T14:29:42.114 INFO:tasks.workunit.client.0.smithi096.stderr:++ objects=1 2024-09-13T14:29:42.114 INFO:tasks.workunit.client.0.smithi096.stderr:++ echo 1 2024-09-13T14:29:42.114 INFO:tasks.workunit.client.0.smithi096.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-09-13T14:29:42.114 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export sparse1 /tmp/rbd_import_export_79210/sparse1.out 2024-09-13T14:29:42.150 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-09-13T14:29:42.153 INFO:tasks.workunit.client.0.smithi096.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_79210/sparse1 /tmp/rbd_import_export_79210/sparse1.out 2024-09-13T14:29:42.153 INFO:tasks.workunit.client.0.smithi096.stderr:+ cmp -l /tmp/rbd_import_export_79210/sparse1 /tmp/rbd_import_export_79210/sparse1.out 2024-09-13T14:29:42.155 INFO:tasks.workunit.client.0.smithi096.stderr:++ stat /tmp/rbd_import_export_79210/sparse1 --format %b 2024-09-13T14:29:42.156 INFO:tasks.workunit.client.0.smithi096.stderr:+ origsize=2048 2024-09-13T14:29:42.156 INFO:tasks.workunit.client.0.smithi096.stderr:++ stat /tmp/rbd_import_export_79210/sparse1.out --format %b 2024-09-13T14:29:42.158 INFO:tasks.workunit.client.0.smithi096.stderr:+ exportsize=2048 2024-09-13T14:29:42.158 INFO:tasks.workunit.client.0.smithi096.stderr:+ difference=0 2024-09-13T14:29:42.158 INFO:tasks.workunit.client.0.smithi096.stderr:+ difference=0 2024-09-13T14:29:42.158 INFO:tasks.workunit.client.0.smithi096.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-13T14:29:42.158 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/sparse1.out 2024-09-13T14:29:42.158 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd rm sparse1 2024-09-13T14:29:42.241 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-09-13T14:29:42.244 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd rm sparse2 2024-09-13T14:29:42.283 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 0% complete...failed. 2024-09-13T14:29:42.283 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: delete error: (2) No such file or directory 2024-09-13T14:29:42.287 INFO:tasks.workunit.client.0.smithi096.stderr:+ true 2024-09-13T14:29:42.287 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_79210/sparse2 2024-09-13T14:29:42.347 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-09-13T14:29:42.347 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:29:42.347 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T14:29:42.350 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd ls -l 2024-09-13T14:29:42.351 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep sparse2 2024-09-13T14:29:42.351 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-09-13T14:29:42.391 INFO:tasks.workunit.client.0.smithi096.stdout:sparse2 2 MiB 2 2024-09-13T14:29:42.392 INFO:tasks.workunit.client.0.smithi096.stderr:++ objects sparse2 2024-09-13T14:29:42.392 INFO:tasks.workunit.client.0.smithi096.stderr:++ image=sparse2 2024-09-13T14:29:42.392 INFO:tasks.workunit.client.0.smithi096.stderr:+++ rbd info sparse2 2024-09-13T14:29:42.392 INFO:tasks.workunit.client.0.smithi096.stderr:+++ grep block_name_prefix 2024-09-13T14:29:42.392 INFO:tasks.workunit.client.0.smithi096.stderr:+++ awk '{print $NF;}' 2024-09-13T14:29:42.434 INFO:tasks.workunit.client.0.smithi096.stderr:++ prefix=rbd_data.3bb2f26f66ef 2024-09-13T14:29:42.435 INFO:tasks.workunit.client.0.smithi096.stderr:+++ grep rbd_data.3bb2f26f66ef 2024-09-13T14:29:42.435 INFO:tasks.workunit.client.0.smithi096.stderr:++++ get_image_data_pool sparse2 2024-09-13T14:29:42.435 INFO:tasks.workunit.client.0.smithi096.stderr:++++ image=sparse2 2024-09-13T14:29:42.435 INFO:tasks.workunit.client.0.smithi096.stderr:+++ sed -e 's/rbd_data.3bb2f26f66ef\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-13T14:29:42.435 INFO:tasks.workunit.client.0.smithi096.stderr:+++ sort -u 2024-09-13T14:29:42.436 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ rbd info sparse2 2024-09-13T14:29:42.436 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ grep 'data_pool: ' 2024-09-13T14:29:42.436 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ awk -F: '{ print $NF }' 2024-09-13T14:29:42.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:29:42 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:29:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:29:42.478 INFO:tasks.workunit.client.0.smithi096.stderr:++++ data_pool= 2024-09-13T14:29:42.478 INFO:tasks.workunit.client.0.smithi096.stderr:++++ '[' -z ']' 2024-09-13T14:29:42.478 INFO:tasks.workunit.client.0.smithi096.stderr:++++ data_pool=rbd 2024-09-13T14:29:42.478 INFO:tasks.workunit.client.0.smithi096.stderr:++++ echo rbd 2024-09-13T14:29:42.478 INFO:tasks.workunit.client.0.smithi096.stderr:+++ rados ls -p rbd 2024-09-13T14:29:42.515 INFO:tasks.workunit.client.0.smithi096.stderr:++ objects=0 2024-09-13T14:29:42.515 INFO:tasks.workunit.client.0.smithi096.stderr:++ echo 0 2024-09-13T14:29:42.515 INFO:tasks.workunit.client.0.smithi096.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-09-13T14:29:42.515 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export sparse2 /tmp/rbd_import_export_79210/sparse2.out 2024-09-13T14:29:42.553 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-09-13T14:29:42.558 INFO:tasks.workunit.client.0.smithi096.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_79210/sparse2 /tmp/rbd_import_export_79210/sparse2.out 2024-09-13T14:29:42.558 INFO:tasks.workunit.client.0.smithi096.stderr:+ cmp -l /tmp/rbd_import_export_79210/sparse2 /tmp/rbd_import_export_79210/sparse2.out 2024-09-13T14:29:42.560 INFO:tasks.workunit.client.0.smithi096.stderr:++ stat /tmp/rbd_import_export_79210/sparse2 --format %b 2024-09-13T14:29:42.561 INFO:tasks.workunit.client.0.smithi096.stderr:+ origsize=2048 2024-09-13T14:29:42.561 INFO:tasks.workunit.client.0.smithi096.stderr:++ stat /tmp/rbd_import_export_79210/sparse2.out --format %b 2024-09-13T14:29:42.562 INFO:tasks.workunit.client.0.smithi096.stderr:+ exportsize=2048 2024-09-13T14:29:42.562 INFO:tasks.workunit.client.0.smithi096.stderr:+ difference=0 2024-09-13T14:29:42.562 INFO:tasks.workunit.client.0.smithi096.stderr:+ difference=0 2024-09-13T14:29:42.562 INFO:tasks.workunit.client.0.smithi096.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-13T14:29:42.562 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/sparse2.out 2024-09-13T14:29:42.563 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd rm sparse2 2024-09-13T14:29:42.648 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-09-13T14:29:42.652 INFO:tasks.workunit.client.0.smithi096.stderr:+ truncate /tmp/rbd_import_export_79210/sparse1 -s 10M 2024-09-13T14:29:42.653 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --new-format --order 20 - sparse1 2024-09-13T14:29:42.743 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 100% complete...done. 2024-09-13T14:29:42.743 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:29:42.743 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T14:29:42.746 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd ls -l 2024-09-13T14:29:42.746 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep sparse1 2024-09-13T14:29:42.746 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep -Ei '(10 MiB|10240k)' 2024-09-13T14:29:42.789 INFO:tasks.workunit.client.0.smithi096.stdout:sparse1 10 MiB 2 2024-09-13T14:29:42.790 INFO:tasks.workunit.client.0.smithi096.stderr:++ objects sparse1 2024-09-13T14:29:42.790 INFO:tasks.workunit.client.0.smithi096.stderr:++ image=sparse1 2024-09-13T14:29:42.790 INFO:tasks.workunit.client.0.smithi096.stderr:+++ rbd info sparse1 2024-09-13T14:29:42.790 INFO:tasks.workunit.client.0.smithi096.stderr:+++ grep block_name_prefix 2024-09-13T14:29:42.790 INFO:tasks.workunit.client.0.smithi096.stderr:+++ awk '{print $NF;}' 2024-09-13T14:29:42.828 INFO:tasks.workunit.client.0.smithi096.stderr:++ prefix=rbd_data.629643c194ea 2024-09-13T14:29:42.829 INFO:tasks.workunit.client.0.smithi096.stderr:+++ grep rbd_data.629643c194ea 2024-09-13T14:29:42.829 INFO:tasks.workunit.client.0.smithi096.stderr:++++ get_image_data_pool sparse1 2024-09-13T14:29:42.829 INFO:tasks.workunit.client.0.smithi096.stderr:+++ sed -e 's/rbd_data.629643c194ea\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-13T14:29:42.829 INFO:tasks.workunit.client.0.smithi096.stderr:++++ image=sparse1 2024-09-13T14:29:42.829 INFO:tasks.workunit.client.0.smithi096.stderr:+++ sort -u 2024-09-13T14:29:42.829 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ rbd info sparse1 2024-09-13T14:29:42.829 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ grep 'data_pool: ' 2024-09-13T14:29:42.829 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ awk -F: '{ print $NF }' 2024-09-13T14:29:42.868 INFO:tasks.workunit.client.0.smithi096.stderr:++++ data_pool= 2024-09-13T14:29:42.869 INFO:tasks.workunit.client.0.smithi096.stderr:++++ '[' -z ']' 2024-09-13T14:29:42.869 INFO:tasks.workunit.client.0.smithi096.stderr:++++ data_pool=rbd 2024-09-13T14:29:42.869 INFO:tasks.workunit.client.0.smithi096.stderr:++++ echo rbd 2024-09-13T14:29:42.869 INFO:tasks.workunit.client.0.smithi096.stderr:+++ rados ls -p rbd 2024-09-13T14:29:42.904 INFO:tasks.workunit.client.0.smithi096.stderr:++ objects=1 2024-09-13T14:29:42.904 INFO:tasks.workunit.client.0.smithi096.stderr:++ echo 1 2024-09-13T14:29:42.905 INFO:tasks.workunit.client.0.smithi096.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-09-13T14:29:42.905 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export sparse1 /tmp/rbd_import_export_79210/sparse1.out 2024-09-13T14:29:42.942 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2024-09-13T14:29:42.947 INFO:tasks.workunit.client.0.smithi096.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_79210/sparse1 /tmp/rbd_import_export_79210/sparse1.out 2024-09-13T14:29:42.947 INFO:tasks.workunit.client.0.smithi096.stderr:+ cmp -l /tmp/rbd_import_export_79210/sparse1 /tmp/rbd_import_export_79210/sparse1.out 2024-09-13T14:29:42.954 INFO:tasks.workunit.client.0.smithi096.stderr:++ stat /tmp/rbd_import_export_79210/sparse1 --format %b 2024-09-13T14:29:42.955 INFO:tasks.workunit.client.0.smithi096.stderr:+ origsize=2048 2024-09-13T14:29:42.955 INFO:tasks.workunit.client.0.smithi096.stderr:++ stat /tmp/rbd_import_export_79210/sparse1.out --format %b 2024-09-13T14:29:42.956 INFO:tasks.workunit.client.0.smithi096.stderr:+ exportsize=2048 2024-09-13T14:29:42.957 INFO:tasks.workunit.client.0.smithi096.stderr:+ difference=0 2024-09-13T14:29:42.957 INFO:tasks.workunit.client.0.smithi096.stderr:+ difference=0 2024-09-13T14:29:42.957 INFO:tasks.workunit.client.0.smithi096.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-13T14:29:42.957 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/sparse1.out 2024-09-13T14:29:42.957 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd rm sparse1 2024-09-13T14:29:42.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:42 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2144545614' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T14:29:42.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:42 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2144545614' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T14:29:43.045 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-09-13T14:29:43.048 INFO:tasks.workunit.client.0.smithi096.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_79210/sparse2 oflag=append conv=notrunc 2024-09-13T14:29:43.056 INFO:tasks.workunit.client.0.smithi096.stderr:1+0 records in 2024-09-13T14:29:43.056 INFO:tasks.workunit.client.0.smithi096.stderr:1+0 records out 2024-09-13T14:29:43.057 INFO:tasks.workunit.client.0.smithi096.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00698053 s, 300 MB/s 2024-09-13T14:29:43.057 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --new-format --order 20 - sparse2 2024-09-13T14:29:43.068 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:29:43.069 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:42 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2144545614' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T14:29:43.069 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:42 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2144545614' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T14:29:43.069 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:42 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2144545614' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T14:29:43.069 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:42 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2144545614' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T14:29:43.147 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 100% complete...done. 2024-09-13T14:29:43.148 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:29:43.148 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T14:29:43.151 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd ls -l 2024-09-13T14:29:43.151 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep sparse2 2024-09-13T14:29:43.151 INFO:tasks.workunit.client.0.smithi096.stderr:+ grep -Ei '(4 MiB|4096k)' 2024-09-13T14:29:43.193 INFO:tasks.workunit.client.0.smithi096.stdout:sparse2 4 MiB 2 2024-09-13T14:29:43.194 INFO:tasks.workunit.client.0.smithi096.stderr:++ objects sparse2 2024-09-13T14:29:43.194 INFO:tasks.workunit.client.0.smithi096.stderr:++ image=sparse2 2024-09-13T14:29:43.194 INFO:tasks.workunit.client.0.smithi096.stderr:+++ rbd info sparse2 2024-09-13T14:29:43.194 INFO:tasks.workunit.client.0.smithi096.stderr:+++ grep block_name_prefix 2024-09-13T14:29:43.194 INFO:tasks.workunit.client.0.smithi096.stderr:+++ awk '{print $NF;}' 2024-09-13T14:29:43.232 INFO:tasks.workunit.client.0.smithi096.stderr:++ prefix=rbd_data.3c18bc12d135 2024-09-13T14:29:43.232 INFO:tasks.workunit.client.0.smithi096.stderr:+++ grep rbd_data.3c18bc12d135 2024-09-13T14:29:43.233 INFO:tasks.workunit.client.0.smithi096.stderr:++++ get_image_data_pool sparse2 2024-09-13T14:29:43.233 INFO:tasks.workunit.client.0.smithi096.stderr:++++ image=sparse2 2024-09-13T14:29:43.233 INFO:tasks.workunit.client.0.smithi096.stderr:+++ sed -e 's/rbd_data.3c18bc12d135\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-13T14:29:43.233 INFO:tasks.workunit.client.0.smithi096.stderr:+++ sort -u 2024-09-13T14:29:43.233 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ rbd info sparse2 2024-09-13T14:29:43.233 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ grep 'data_pool: ' 2024-09-13T14:29:43.233 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ awk -F: '{ print $NF }' 2024-09-13T14:29:43.284 INFO:tasks.workunit.client.0.smithi096.stderr:++++ data_pool= 2024-09-13T14:29:43.284 INFO:tasks.workunit.client.0.smithi096.stderr:++++ '[' -z ']' 2024-09-13T14:29:43.284 INFO:tasks.workunit.client.0.smithi096.stderr:++++ data_pool=rbd 2024-09-13T14:29:43.285 INFO:tasks.workunit.client.0.smithi096.stderr:++++ echo rbd 2024-09-13T14:29:43.285 INFO:tasks.workunit.client.0.smithi096.stderr:+++ rados ls -p rbd 2024-09-13T14:29:43.317 INFO:tasks.workunit.client.0.smithi096.stderr:++ objects='0 2024-09-13T14:29:43.317 INFO:tasks.workunit.client.0.smithi096.stderr:2 2024-09-13T14:29:43.317 INFO:tasks.workunit.client.0.smithi096.stderr:3' 2024-09-13T14:29:43.317 INFO:tasks.workunit.client.0.smithi096.stderr:++ echo 0 2 3 2024-09-13T14:29:43.317 INFO:tasks.workunit.client.0.smithi096.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2024-09-13T14:29:43.317 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export sparse2 /tmp/rbd_import_export_79210/sparse2.out 2024-09-13T14:29:43.360 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2024-09-13T14:29:43.362 INFO:tasks.workunit.client.0.smithi096.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_79210/sparse2 /tmp/rbd_import_export_79210/sparse2.out 2024-09-13T14:29:43.362 INFO:tasks.workunit.client.0.smithi096.stderr:+ cmp -l /tmp/rbd_import_export_79210/sparse2 /tmp/rbd_import_export_79210/sparse2.out 2024-09-13T14:29:43.365 INFO:tasks.workunit.client.0.smithi096.stderr:++ stat /tmp/rbd_import_export_79210/sparse2 --format %b 2024-09-13T14:29:43.366 INFO:tasks.workunit.client.0.smithi096.stderr:+ origsize=6144 2024-09-13T14:29:43.366 INFO:tasks.workunit.client.0.smithi096.stderr:++ stat /tmp/rbd_import_export_79210/sparse2.out --format %b 2024-09-13T14:29:43.367 INFO:tasks.workunit.client.0.smithi096.stderr:+ exportsize=6144 2024-09-13T14:29:43.368 INFO:tasks.workunit.client.0.smithi096.stderr:+ difference=0 2024-09-13T14:29:43.368 INFO:tasks.workunit.client.0.smithi096.stderr:+ difference=0 2024-09-13T14:29:43.368 INFO:tasks.workunit.client.0.smithi096.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-13T14:29:43.368 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/sparse2.out 2024-09-13T14:29:43.368 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd rm sparse2 2024-09-13T14:29:43.461 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2024-09-13T14:29:43.465 INFO:tasks.workunit.client.0.smithi096.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2024-09-13T14:29:43.465 INFO:tasks.workunit.client.0.smithi096.stdout:partially-sparse file imports to partially-sparse image 2024-09-13T14:29:43.465 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_79210/sparse1 sparse 2024-09-13T14:29:43.528 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2024-09-13T14:29:43.529 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:29:43.529 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T14:29:43.531 INFO:tasks.workunit.client.0.smithi096.stderr:++ objects sparse 2024-09-13T14:29:43.531 INFO:tasks.workunit.client.0.smithi096.stderr:++ image=sparse 2024-09-13T14:29:43.532 INFO:tasks.workunit.client.0.smithi096.stderr:+++ rbd info sparse 2024-09-13T14:29:43.532 INFO:tasks.workunit.client.0.smithi096.stderr:+++ grep block_name_prefix 2024-09-13T14:29:43.532 INFO:tasks.workunit.client.0.smithi096.stderr:+++ awk '{print $NF;}' 2024-09-13T14:29:43.571 INFO:tasks.workunit.client.0.smithi096.stderr:++ prefix=rbd_data.62fc866ad748 2024-09-13T14:29:43.572 INFO:tasks.workunit.client.0.smithi096.stderr:+++ grep rbd_data.62fc866ad748 2024-09-13T14:29:43.572 INFO:tasks.workunit.client.0.smithi096.stderr:++++ get_image_data_pool sparse 2024-09-13T14:29:43.572 INFO:tasks.workunit.client.0.smithi096.stderr:++++ image=sparse 2024-09-13T14:29:43.572 INFO:tasks.workunit.client.0.smithi096.stderr:+++ sed -e 's/rbd_data.62fc866ad748\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-13T14:29:43.572 INFO:tasks.workunit.client.0.smithi096.stderr:+++ sort -u 2024-09-13T14:29:43.573 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ rbd info sparse 2024-09-13T14:29:43.573 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ grep 'data_pool: ' 2024-09-13T14:29:43.573 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ awk -F: '{ print $NF }' 2024-09-13T14:29:43.613 INFO:tasks.workunit.client.0.smithi096.stderr:++++ data_pool= 2024-09-13T14:29:43.613 INFO:tasks.workunit.client.0.smithi096.stderr:++++ '[' -z ']' 2024-09-13T14:29:43.613 INFO:tasks.workunit.client.0.smithi096.stderr:++++ data_pool=rbd 2024-09-13T14:29:43.613 INFO:tasks.workunit.client.0.smithi096.stderr:++++ echo rbd 2024-09-13T14:29:43.613 INFO:tasks.workunit.client.0.smithi096.stderr:+++ rados ls -p rbd 2024-09-13T14:29:43.644 INFO:tasks.workunit.client.0.smithi096.stderr:++ objects=1 2024-09-13T14:29:43.645 INFO:tasks.workunit.client.0.smithi096.stderr:++ echo 1 2024-09-13T14:29:43.645 INFO:tasks.workunit.client.0.smithi096.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-09-13T14:29:43.645 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd rm sparse 2024-09-13T14:29:43.731 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-09-13T14:29:43.734 INFO:tasks.workunit.client.0.smithi096.stdout:zeros import through stdin to sparse image 2024-09-13T14:29:43.735 INFO:tasks.workunit.client.0.smithi096.stderr:+ echo 'zeros import through stdin to sparse image' 2024-09-13T14:29:43.735 INFO:tasks.workunit.client.0.smithi096.stderr:+ dd if=/dev/zero bs=1M count=4 2024-09-13T14:29:43.735 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd import --new-format - sparse 2024-09-13T14:29:43.783 INFO:tasks.workunit.client.0.smithi096.stderr:4+0 records in 2024-09-13T14:29:43.783 INFO:tasks.workunit.client.0.smithi096.stderr:4+0 records out 2024-09-13T14:29:43.783 INFO:tasks.workunit.client.0.smithi096.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0476381 s, 88.0 MB/s 2024-09-13T14:29:43.792 INFO:tasks.workunit.client.0.smithi096.stderr: Importing image: 100% complete...done. 2024-09-13T14:29:43.792 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:29:43.796 INFO:tasks.workunit.client.0.smithi096.stderr:++ objects sparse 2024-09-13T14:29:43.796 INFO:tasks.workunit.client.0.smithi096.stderr:++ image=sparse 2024-09-13T14:29:43.796 INFO:tasks.workunit.client.0.smithi096.stderr:+++ rbd info sparse 2024-09-13T14:29:43.796 INFO:tasks.workunit.client.0.smithi096.stderr:+++ grep block_name_prefix 2024-09-13T14:29:43.796 INFO:tasks.workunit.client.0.smithi096.stderr:+++ awk '{print $NF;}' 2024-09-13T14:29:43.831 INFO:tasks.workunit.client.0.smithi096.stderr:++ prefix=rbd_data.3c6c21d5d145 2024-09-13T14:29:43.832 INFO:tasks.workunit.client.0.smithi096.stderr:+++ grep rbd_data.3c6c21d5d145 2024-09-13T14:29:43.832 INFO:tasks.workunit.client.0.smithi096.stderr:++++ get_image_data_pool sparse 2024-09-13T14:29:43.832 INFO:tasks.workunit.client.0.smithi096.stderr:++++ image=sparse 2024-09-13T14:29:43.832 INFO:tasks.workunit.client.0.smithi096.stderr:+++ sed -e 's/rbd_data.3c6c21d5d145\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-13T14:29:43.832 INFO:tasks.workunit.client.0.smithi096.stderr:+++ sort -u 2024-09-13T14:29:43.832 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ rbd info sparse 2024-09-13T14:29:43.833 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ grep 'data_pool: ' 2024-09-13T14:29:43.833 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ awk -F: '{ print $NF }' 2024-09-13T14:29:43.868 INFO:tasks.workunit.client.0.smithi096.stderr:++++ data_pool= 2024-09-13T14:29:43.869 INFO:tasks.workunit.client.0.smithi096.stderr:++++ '[' -z ']' 2024-09-13T14:29:43.869 INFO:tasks.workunit.client.0.smithi096.stderr:++++ data_pool=rbd 2024-09-13T14:29:43.869 INFO:tasks.workunit.client.0.smithi096.stderr:++++ echo rbd 2024-09-13T14:29:43.869 INFO:tasks.workunit.client.0.smithi096.stderr:+++ rados ls -p rbd 2024-09-13T14:29:43.904 INFO:tasks.workunit.client.0.smithi096.stderr:++ objects= 2024-09-13T14:29:43.904 INFO:tasks.workunit.client.0.smithi096.stderr:++ echo 2024-09-13T14:29:43.904 INFO:tasks.workunit.client.0.smithi096.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2024-09-13T14:29:43.904 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd rm sparse 2024-09-13T14:29:43.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:43 smithi204 ceph-mon[32546]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 157 KiB/s rd, 11 KiB/s wr, 212 op/s 2024-09-13T14:29:43.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:43 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/806532803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:43.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:43 smithi204 ceph-mon[32546]: from='client.25256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:43.981 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 100% complete...done. 2024-09-13T14:29:43.985 INFO:tasks.workunit.client.0.smithi096.stdout:zeros export to sparse file 2024-09-13T14:29:43.985 INFO:tasks.workunit.client.0.smithi096.stderr:+ echo 'zeros export to sparse file' 2024-09-13T14:29:43.985 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd create --new-format sparse --size 4 2024-09-13T14:29:44.026 INFO:tasks.workunit.client.0.smithi096.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:29:44.029 INFO:tasks.workunit.client.0.smithi096.stderr:++ rbd info sparse 2024-09-13T14:29:44.029 INFO:tasks.workunit.client.0.smithi096.stderr:++ grep block_name_prefix 2024-09-13T14:29:44.029 INFO:tasks.workunit.client.0.smithi096.stderr:++ awk '{print $NF;}' 2024-09-13T14:29:44.072 INFO:tasks.workunit.client.0.smithi096.stderr:+ prefix=rbd_data.633ec68cd0b0 2024-09-13T14:29:44.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:43 smithi096 ceph-mon[29229]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 157 KiB/s rd, 11 KiB/s wr, 212 op/s 2024-09-13T14:29:44.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:43 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/806532803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:44.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:43 smithi096 ceph-mon[29229]: from='client.25256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:44.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:43 smithi096 ceph-mon[39645]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 157 KiB/s rd, 11 KiB/s wr, 212 op/s 2024-09-13T14:29:44.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:43 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/806532803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:44.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:43 smithi096 ceph-mon[39645]: from='client.25256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:44.073 INFO:tasks.workunit.client.0.smithi096.stderr:+ dd if=/dev/zero bs=4M count=1 2024-09-13T14:29:44.073 INFO:tasks.workunit.client.0.smithi096.stderr:++ get_image_data_pool sparse 2024-09-13T14:29:44.074 INFO:tasks.workunit.client.0.smithi096.stderr:++ image=sparse 2024-09-13T14:29:44.074 INFO:tasks.workunit.client.0.smithi096.stderr:+++ rbd info sparse 2024-09-13T14:29:44.074 INFO:tasks.workunit.client.0.smithi096.stderr:+++ grep 'data_pool: ' 2024-09-13T14:29:44.074 INFO:tasks.workunit.client.0.smithi096.stderr:+++ awk -F: '{ print $NF }' 2024-09-13T14:29:44.114 INFO:tasks.workunit.client.0.smithi096.stderr:++ data_pool= 2024-09-13T14:29:44.114 INFO:tasks.workunit.client.0.smithi096.stderr:++ '[' -z ']' 2024-09-13T14:29:44.115 INFO:tasks.workunit.client.0.smithi096.stderr:++ data_pool=rbd 2024-09-13T14:29:44.115 INFO:tasks.workunit.client.0.smithi096.stderr:++ echo rbd 2024-09-13T14:29:44.115 INFO:tasks.workunit.client.0.smithi096.stderr:+ rados -p rbd put rbd_data.633ec68cd0b0.000000000000 - 2024-09-13T14:29:44.144 INFO:tasks.workunit.client.0.smithi096.stderr:1+0 records in 2024-09-13T14:29:44.144 INFO:tasks.workunit.client.0.smithi096.stderr:1+0 records out 2024-09-13T14:29:44.145 INFO:tasks.workunit.client.0.smithi096.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0714026 s, 58.7 MB/s 2024-09-13T14:29:44.165 INFO:tasks.workunit.client.0.smithi096.stderr:++ objects sparse 2024-09-13T14:29:44.165 INFO:tasks.workunit.client.0.smithi096.stderr:++ image=sparse 2024-09-13T14:29:44.166 INFO:tasks.workunit.client.0.smithi096.stderr:+++ rbd info sparse 2024-09-13T14:29:44.166 INFO:tasks.workunit.client.0.smithi096.stderr:+++ grep block_name_prefix 2024-09-13T14:29:44.166 INFO:tasks.workunit.client.0.smithi096.stderr:+++ awk '{print $NF;}' 2024-09-13T14:29:44.204 INFO:tasks.workunit.client.0.smithi096.stderr:++ prefix=rbd_data.633ec68cd0b0 2024-09-13T14:29:44.205 INFO:tasks.workunit.client.0.smithi096.stderr:+++ grep rbd_data.633ec68cd0b0 2024-09-13T14:29:44.205 INFO:tasks.workunit.client.0.smithi096.stderr:++++ get_image_data_pool sparse 2024-09-13T14:29:44.205 INFO:tasks.workunit.client.0.smithi096.stderr:++++ image=sparse 2024-09-13T14:29:44.205 INFO:tasks.workunit.client.0.smithi096.stderr:+++ sed -e 's/rbd_data.633ec68cd0b0\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-13T14:29:44.205 INFO:tasks.workunit.client.0.smithi096.stderr:+++ sort -u 2024-09-13T14:29:44.206 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ rbd info sparse 2024-09-13T14:29:44.206 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ grep 'data_pool: ' 2024-09-13T14:29:44.206 INFO:tasks.workunit.client.0.smithi096.stderr:+++++ awk -F: '{ print $NF }' 2024-09-13T14:29:44.242 INFO:tasks.workunit.client.0.smithi096.stderr:++++ data_pool= 2024-09-13T14:29:44.242 INFO:tasks.workunit.client.0.smithi096.stderr:++++ '[' -z ']' 2024-09-13T14:29:44.242 INFO:tasks.workunit.client.0.smithi096.stderr:++++ data_pool=rbd 2024-09-13T14:29:44.242 INFO:tasks.workunit.client.0.smithi096.stderr:++++ echo rbd 2024-09-13T14:29:44.242 INFO:tasks.workunit.client.0.smithi096.stderr:+++ rados ls -p rbd 2024-09-13T14:29:44.277 INFO:tasks.workunit.client.0.smithi096.stderr:++ objects=0 2024-09-13T14:29:44.277 INFO:tasks.workunit.client.0.smithi096.stderr:++ echo 0 2024-09-13T14:29:44.277 INFO:tasks.workunit.client.0.smithi096.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-09-13T14:29:44.277 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/sparse 2024-09-13T14:29:44.278 INFO:tasks.workunit.client.0.smithi096.stderr:rm: cannot remove '/tmp/rbd_import_export_79210/sparse': No such file or directory 2024-09-13T14:29:44.278 INFO:tasks.workunit.client.0.smithi096.stderr:+ true 2024-09-13T14:29:44.278 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd export sparse /tmp/rbd_import_export_79210/sparse 2024-09-13T14:29:44.315 INFO:tasks.workunit.client.0.smithi096.stderr: Exporting image: 100% complete...done. 2024-09-13T14:29:44.319 INFO:tasks.workunit.client.0.smithi096.stderr:++ stat /tmp/rbd_import_export_79210/sparse --format=%b 2024-09-13T14:29:44.320 INFO:tasks.workunit.client.0.smithi096.stderr:+ '[' 0 = 0 ']' 2024-09-13T14:29:44.320 INFO:tasks.workunit.client.0.smithi096.stderr:+ rbd rm sparse 2024-09-13T14:29:44.405 INFO:tasks.workunit.client.0.smithi096.stderr: Removing image: 100% complete...done. 2024-09-13T14:29:44.408 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm /tmp/rbd_import_export_79210/sparse /tmp/rbd_import_export_79210/sparse1 /tmp/rbd_import_export_79210/sparse2 /tmp/rbd_import_export_79210/sparse3 2024-09-13T14:29:44.410 INFO:tasks.workunit.client.0.smithi096.stderr:rm: cannot remove '/tmp/rbd_import_export_79210/sparse3': No such file or directory 2024-09-13T14:29:44.410 INFO:tasks.workunit.client.0.smithi096.stderr:+ true 2024-09-13T14:29:44.410 INFO:tasks.workunit.client.0.smithi096.stdout:OK 2024-09-13T14:29:44.411 INFO:tasks.workunit.client.0.smithi096.stderr:+ echo OK 2024-09-13T14:29:44.411 INFO:tasks.workunit.client.0.smithi096.stderr:+ rm -rf /tmp/rbd_import_export_79210 2024-09-13T14:29:44.412 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T14:29:44.412 DEBUG:teuthology.orchestra.run.smithi096:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T14:29:44.439 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2024-09-13T14:29:44.439 DEBUG:teuthology.orchestra.run.smithi096:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-13T14:29:44.875 DEBUG:teuthology.parallel:result is None 2024-09-13T14:29:44.876 DEBUG:teuthology.orchestra.run.smithi096:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T14:29:44.904 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T14:29:44.904 DEBUG:teuthology.orchestra.run.smithi096:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:29:44.959 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T14:29:44.960 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-13T14:29:44.960 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2024-09-13T14:29:44.960 DEBUG:teuthology.parallel:result is None 2024-09-13T14:29:44.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:44 smithi204 ceph-mon[32546]: from='client.15390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:45.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:44 smithi096 ceph-mon[29229]: from='client.15390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:45.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:44 smithi096 ceph-mon[39645]: from='client.15390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:45.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:45 smithi204 ceph-mon[32546]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 107 KiB/s rd, 7.8 KiB/s wr, 144 op/s 2024-09-13T14:29:46.031 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:29:46.032 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:45 smithi096 ceph-mon[29229]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 107 KiB/s rd, 7.8 KiB/s wr, 144 op/s 2024-09-13T14:29:46.032 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:45 smithi096 ceph-mon[39645]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 107 KiB/s rd, 7.8 KiB/s wr, 144 op/s 2024-09-13T14:29:46.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:46 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3772755671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:46.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:46 smithi204 ceph-mon[32546]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:46.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:46 smithi204 ceph-mon[32546]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T14:29:47.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:46 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3772755671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:47.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:46 smithi096 ceph-mon[29229]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:47.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:46 smithi096 ceph-mon[29229]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T14:29:47.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:46 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3772755671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:47.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:46 smithi096 ceph-mon[39645]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:47.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:46 smithi096 ceph-mon[39645]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T14:29:47.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:47 smithi204 ceph-mon[32546]: from='client.25478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:47.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:47 smithi204 ceph-mon[32546]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.9 MiB/s wr, 521 op/s 2024-09-13T14:29:48.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:47 smithi096 ceph-mon[29229]: from='client.25478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:48.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:47 smithi096 ceph-mon[29229]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.9 MiB/s wr, 521 op/s 2024-09-13T14:29:48.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:29:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:29:47] "GET /metrics HTTP/1.1" 200 240120 "" "Prometheus/2.43.0" 2024-09-13T14:29:48.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:47 smithi096 ceph-mon[39645]: from='client.25478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:48.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:47 smithi096 ceph-mon[39645]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.9 MiB/s wr, 521 op/s 2024-09-13T14:29:48.936 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:29:48.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:48 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2482244399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:49.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:48 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2482244399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:49.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:48 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2482244399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:49.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:49 smithi204 ceph-mon[32546]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.4 MiB/s wr, 397 op/s 2024-09-13T14:29:49.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:49 smithi204 ceph-mon[32546]: from='client.15585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:50.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:49 smithi096 ceph-mon[29229]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.4 MiB/s wr, 397 op/s 2024-09-13T14:29:50.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:49 smithi096 ceph-mon[29229]: from='client.15585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:50.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:49 smithi096 ceph-mon[39645]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.4 MiB/s wr, 397 op/s 2024-09-13T14:29:50.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:49 smithi096 ceph-mon[39645]: from='client.15585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:50.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:50 smithi204 ceph-mon[32546]: from='client.25496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:50.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:50 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:51.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:50 smithi096 ceph-mon[29229]: from='client.25496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:51.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:50 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:51.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:50 smithi096 ceph-mon[39645]: from='client.25496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:51.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:50 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:51.837 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:29:51.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:51 smithi204 ceph-mon[32546]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 913 KiB/s rd, 1.2 MiB/s wr, 336 op/s 2024-09-13T14:29:51.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:51 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/745212629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:52.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:51 smithi096 ceph-mon[29229]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 913 KiB/s rd, 1.2 MiB/s wr, 336 op/s 2024-09-13T14:29:52.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:51 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/745212629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:52.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:51 smithi096 ceph-mon[39645]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 913 KiB/s rd, 1.2 MiB/s wr, 336 op/s 2024-09-13T14:29:52.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:51 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/745212629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:52.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:29:52 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:29:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:29:53.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:52 smithi096 ceph-mon[29229]: from='client.25508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:53.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:52 smithi096 ceph-mon[29229]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:53.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:52 smithi096 ceph-mon[39645]: from='client.25508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:53.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:52 smithi096 ceph-mon[39645]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:53.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:52 smithi204 ceph-mon[32546]: from='client.25508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:53.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:52 smithi204 ceph-mon[32546]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:54.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:53 smithi096 ceph-mon[29229]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2024-09-13T14:29:54.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:53 smithi096 ceph-mon[39645]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2024-09-13T14:29:54.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:53 smithi204 ceph-mon[32546]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2024-09-13T14:29:54.749 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:29:55.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:54 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3366644036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:55.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:54 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3366644036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:55.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:54 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3366644036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:56.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:55 smithi096 ceph-mon[29229]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2024-09-13T14:29:56.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:55 smithi096 ceph-mon[29229]: from='client.15621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:56.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:55 smithi096 ceph-mon[29229]: from='client.15627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:55 smithi096 ceph-mon[39645]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2024-09-13T14:29:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:55 smithi096 ceph-mon[39645]: from='client.15621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:55 smithi096 ceph-mon[39645]: from='client.15627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:56.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:55 smithi204 ceph-mon[32546]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2024-09-13T14:29:56.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:55 smithi204 ceph-mon[32546]: from='client.15621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:56.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:55 smithi204 ceph-mon[32546]: from='client.15627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:57.797 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:29:58.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:57 smithi096 ceph-mon[29229]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 763 KiB/s rd, 1.1 MiB/s wr, 221 op/s 2024-09-13T14:29:58.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:57 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/539871689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:58.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:29:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:29:57] "GET /metrics HTTP/1.1" 200 240374 "" "Prometheus/2.43.0" 2024-09-13T14:29:58.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:57 smithi096 ceph-mon[39645]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 763 KiB/s rd, 1.1 MiB/s wr, 221 op/s 2024-09-13T14:29:58.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:57 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/539871689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:58.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:57 smithi204 ceph-mon[32546]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 763 KiB/s rd, 1.1 MiB/s wr, 221 op/s 2024-09-13T14:29:58.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:57 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/539871689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:59.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:58 smithi096 ceph-mon[29229]: from='client.15639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:59.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:58 smithi096 ceph-mon[29229]: from='client.25550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:59.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:58 smithi096 ceph-mon[39645]: from='client.15639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:59.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:58 smithi096 ceph-mon[39645]: from='client.25550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:59.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:58 smithi204 ceph-mon[32546]: from='client.15639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:59.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:58 smithi204 ceph-mon[32546]: from='client.25550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:00.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:29:59 smithi096 ceph-mon[29229]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:00.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:29:59 smithi096 ceph-mon[39645]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:00.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:29:59 smithi204 ceph-mon[32546]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:00.902 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:00.992 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:00 smithi204 ceph-mon[32546]: overall HEALTH_OK 2024-09-13T14:30:00.992 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:00 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/4190677641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:01.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:00 smithi096 ceph-mon[29229]: overall HEALTH_OK 2024-09-13T14:30:01.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:00 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/4190677641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:01.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:00 smithi096 ceph-mon[39645]: overall HEALTH_OK 2024-09-13T14:30:01.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:00 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/4190677641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:02.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:01 smithi096 ceph-mon[29229]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:02.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:01 smithi096 ceph-mon[29229]: from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:02.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:01 smithi096 ceph-mon[39645]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:02.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:01 smithi096 ceph-mon[39645]: from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:02.119 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:01 smithi204 ceph-mon[32546]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:02.119 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:01 smithi204 ceph-mon[32546]: from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:02.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:02 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:30:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:30:03.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:02 smithi096 ceph-mon[29229]: from='client.25568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:03.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:02 smithi096 ceph-mon[39645]: from='client.25568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:03.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:02 smithi204 ceph-mon[32546]: from='client.25568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:03.872 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:04.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:03 smithi096 ceph-mon[39645]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:04.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:03 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/181789110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:04.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:03 smithi096 ceph-mon[29229]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:04.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:03 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/181789110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:04.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:03 smithi204 ceph-mon[32546]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:04.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:03 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/181789110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:05.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:04 smithi096 ceph-mon[39645]: from='client.25580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:05.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:04 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:05.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:04 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:05.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:04 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:04 smithi096 ceph-mon[29229]: from='client.25580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:04 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:04 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:04 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:05.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:04 smithi204 ceph-mon[32546]: from='client.25580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:05.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:04 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:05.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:04 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:05.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:04 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:06.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:05 smithi096 ceph-mon[29229]: from='client.25586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:06.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:05 smithi096 ceph-mon[29229]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (unknown) 2024-09-13T14:30:06.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:05 smithi096 ceph-mon[29229]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:30:06.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:05 smithi096 ceph-mon[29229]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T14:30:06.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:05 smithi096 ceph-mon[29229]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:06.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:05 smithi096 ceph-mon[29229]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f on smithi204 2024-09-13T14:30:06.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:05 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:05 smithi096 ceph-mon[39645]: from='client.25586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:05 smithi096 ceph-mon[39645]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (unknown) 2024-09-13T14:30:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:05 smithi096 ceph-mon[39645]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:30:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:05 smithi096 ceph-mon[39645]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T14:30:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:05 smithi096 ceph-mon[39645]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:05 smithi096 ceph-mon[39645]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f on smithi204 2024-09-13T14:30:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:05 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:06.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:05 smithi204 ceph-mon[32546]: from='client.25586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:06.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:05 smithi204 ceph-mon[32546]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (unknown) 2024-09-13T14:30:06.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:05 smithi204 ceph-mon[32546]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:30:06.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:05 smithi204 ceph-mon[32546]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T14:30:06.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:05 smithi204 ceph-mon[32546]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:06.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:05 smithi204 ceph-mon[32546]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f on smithi204 2024-09-13T14:30:06.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:05 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:06.718 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:06 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2672686829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:07.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:06 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2672686829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:07.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:06 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2672686829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:08.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:30:07] "GET /metrics HTTP/1.1" 200 240312 "" "Prometheus/2.43.0" 2024-09-13T14:30:08.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:07 smithi096 ceph-mon[29229]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:08.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:07 smithi096 ceph-mon[29229]: from='client.25598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:08.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:07 smithi096 ceph-mon[29229]: from='client.25604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:08.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:07 smithi096 ceph-mon[39645]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:08.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:07 smithi096 ceph-mon[39645]: from='client.25598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:08.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:07 smithi096 ceph-mon[39645]: from='client.25604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:08.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:07 smithi204 ceph-mon[32546]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:08.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:07 smithi204 ceph-mon[32546]: from='client.25598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:08.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:07 smithi204 ceph-mon[32546]: from='client.25604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:09.546 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:09.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:09 smithi096 ceph-mon[29229]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:09.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:09 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/776750228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:09.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:09 smithi096 ceph-mon[39645]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:09.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:09 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/776750228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:10.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:09 smithi204 ceph-mon[32546]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:10.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:09 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/776750228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:11.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:10 smithi096 ceph-mon[29229]: from='client.25616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:11.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:10 smithi096 ceph-mon[29229]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:11.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:10 smithi096 ceph-mon[39645]: from='client.25616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:11.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:10 smithi096 ceph-mon[39645]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:11.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:10 smithi204 ceph-mon[32546]: from='client.25616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:11.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:10 smithi204 ceph-mon[32546]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:12.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:11 smithi096 ceph-mon[29229]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:12.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:11 smithi096 ceph-mon[39645]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:12.118 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:11 smithi204 ceph-mon[32546]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:12.379 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:12.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:12 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:30:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:30:13.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:12 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/58037511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:13.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:12 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/58037511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:13.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:12 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/58037511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:14.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:13 smithi096 ceph-mon[29229]: from='client.15729 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:14.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:13 smithi096 ceph-mon[29229]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:14.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:13 smithi096 ceph-mon[29229]: from='client.25640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:14.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:13 smithi096 ceph-mon[39645]: from='client.15729 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:14.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:13 smithi096 ceph-mon[39645]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:14.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:13 smithi096 ceph-mon[39645]: from='client.25640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:14.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:13 smithi204 ceph-mon[32546]: from='client.15729 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:14.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:13 smithi204 ceph-mon[32546]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:14.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:13 smithi204 ceph-mon[32546]: from='client.25640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:15.202 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:16.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:15 smithi096 ceph-mon[29229]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:16.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:15 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2354977065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:16.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:15 smithi096 ceph-mon[39645]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:16.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:15 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2354977065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:16.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:15 smithi204 ceph-mon[32546]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:16.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:15 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2354977065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:17.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:16 smithi096 ceph-mon[29229]: from='client.25652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:17.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:16 smithi096 ceph-mon[29229]: from='client.25658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:17.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:16 smithi096 ceph-mon[39645]: from='client.25652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:17.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:16 smithi096 ceph-mon[39645]: from='client.25658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:16 smithi204 ceph-mon[32546]: from='client.25652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:16 smithi204 ceph-mon[32546]: from='client.25658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:18.034 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:18.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:17 smithi096 ceph-mon[29229]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:18.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:17 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/665690844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:18.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:30:17] "GET /metrics HTTP/1.1" 200 240312 "" "Prometheus/2.43.0" 2024-09-13T14:30:18.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:17 smithi096 ceph-mon[39645]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:18.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:17 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/665690844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:18.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:17 smithi204 ceph-mon[32546]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:18.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:17 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/665690844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:19.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:18 smithi096 ceph-mon[39645]: from='client.25670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:19.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:18 smithi096 ceph-mon[29229]: from='client.25670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:19.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:18 smithi204 ceph-mon[32546]: from='client.25670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:20.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:19 smithi096 ceph-mon[39645]: from='client.25676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:20.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:19 smithi096 ceph-mon[39645]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:20.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:19 smithi096 ceph-mon[29229]: from='client.25676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:20.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:19 smithi096 ceph-mon[29229]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:20.119 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:19 smithi204 ceph-mon[32546]: from='client.25676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:20.119 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:19 smithi204 ceph-mon[32546]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:20.872 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:21.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:20 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:21.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:20 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1712491476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:21.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:20 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:21.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:20 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1712491476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:21.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:20 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:21.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:20 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1712491476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:22.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:21 smithi096 ceph-mon[29229]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:22.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:21 smithi096 ceph-mon[29229]: from='client.25688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:22.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:21 smithi096 ceph-mon[29229]: from='client.15789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:21 smithi096 ceph-mon[39645]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:21 smithi096 ceph-mon[39645]: from='client.25688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:21 smithi096 ceph-mon[39645]: from='client.15789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:22.118 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:21 smithi204 ceph-mon[32546]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:22.119 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:21 smithi204 ceph-mon[32546]: from='client.25688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:22.119 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:21 smithi204 ceph-mon[32546]: from='client.15789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:22.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:22 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:30:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:30:23.698 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:24.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:23 smithi096 ceph-mon[29229]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:24.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:23 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3719063880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:24.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:23 smithi096 ceph-mon[39645]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:24.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:23 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3719063880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:24.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:23 smithi204 ceph-mon[32546]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:24.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:23 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3719063880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:25.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:24 smithi096 ceph-mon[29229]: from='client.25706 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:25.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:24 smithi096 ceph-mon[29229]: from='client.25712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:25.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:24 smithi096 ceph-mon[39645]: from='client.25706 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:25.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:24 smithi096 ceph-mon[39645]: from='client.25712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:24 smithi204 ceph-mon[32546]: from='client.25706 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:24 smithi204 ceph-mon[32546]: from='client.25712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:26.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:25 smithi096 ceph-mon[29229]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:26.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:25 smithi096 ceph-mon[39645]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:26.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:25 smithi204 ceph-mon[32546]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:26.531 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:27.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:26 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2665006847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:27.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:26 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2665006847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:27.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:26 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2665006847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:28.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:30:27] "GET /metrics HTTP/1.1" 200 240257 "" "Prometheus/2.43.0" 2024-09-13T14:30:28.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:27 smithi096 ceph-mon[29229]: from='client.25724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:28.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:27 smithi096 ceph-mon[29229]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:28.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:27 smithi096 ceph-mon[29229]: from='client.25730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:28.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:27 smithi096 ceph-mon[39645]: from='client.25724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:28.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:27 smithi096 ceph-mon[39645]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:28.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:27 smithi096 ceph-mon[39645]: from='client.25730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:28.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:27 smithi204 ceph-mon[32546]: from='client.25724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:28.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:27 smithi204 ceph-mon[32546]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:28.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:27 smithi204 ceph-mon[32546]: from='client.25730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:29.369 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:30.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:29 smithi096 ceph-mon[39645]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:30.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:29 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1296818665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:30.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:29 smithi096 ceph-mon[29229]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:30.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:29 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1296818665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:30.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:29 smithi204 ceph-mon[32546]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:30.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:29 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1296818665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:31.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:30 smithi204 ceph-mon[32546]: from='client.25742 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:31.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:30 smithi204 ceph-mon[32546]: from='client.15843 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:31.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:30 smithi096 ceph-mon[29229]: from='client.25742 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:31.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:30 smithi096 ceph-mon[29229]: from='client.15843 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:31.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:30 smithi096 ceph-mon[39645]: from='client.25742 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:31.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:30 smithi096 ceph-mon[39645]: from='client.15843 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:32.119 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:31 smithi204 ceph-mon[32546]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:32.209 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:32.209 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:31 smithi096 ceph-mon[29229]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:32.209 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:31 smithi096 ceph-mon[39645]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:32.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[34402]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:30:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:30:33.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:32 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1282545328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:33.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:32 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1282545328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:33.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:32 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1282545328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:34.090 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:33 smithi204 ceph-mon[32546]: from='client.15855 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:34.090 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:33 smithi204 ceph-mon[32546]: from='client.15861 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:34.090 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:33 smithi204 ceph-mon[32546]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:34.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:33 smithi096 ceph-mon[29229]: from='client.15855 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:34.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:33 smithi096 ceph-mon[29229]: from='client.15861 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:34.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:33 smithi096 ceph-mon[29229]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:34.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:33 smithi096 ceph-mon[39645]: from='client.15855 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:34.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:33 smithi096 ceph-mon[39645]: from='client.15861 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:34.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:33 smithi096 ceph-mon[39645]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:35.059 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:35.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:34 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3762369250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:35.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:34 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3762369250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:35.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:34 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3762369250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[39645]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[39645]: Upgrade: Updating mgr.x 2024-09-13T14:30:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:30:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:30:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:30:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[39645]: Deploying daemon mgr.x on smithi204 2024-09-13T14:30:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[39645]: from='client.15873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:36.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[29229]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:36.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[29229]: Upgrade: Updating mgr.x 2024-09-13T14:30:36.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:36.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:30:36.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:30:36.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:30:36.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[29229]: Deploying daemon mgr.x on smithi204 2024-09-13T14:30:36.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[29229]: from='client.15873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:36.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:36 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:36.349 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:36 smithi204 ceph-mon[32546]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:36.350 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:36 smithi204 ceph-mon[32546]: Upgrade: Updating mgr.x 2024-09-13T14:30:36.350 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:36 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:36.350 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:36 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:30:36.350 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:36 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:30:36.350 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:36 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:30:36.350 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:36 smithi204 ceph-mon[32546]: Deploying daemon mgr.x on smithi204 2024-09-13T14:30:36.350 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:36 smithi204 ceph-mon[32546]: from='client.15873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:36.350 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:36 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:36.718 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:36 smithi204 systemd[1]: Stopping Ceph mgr.x for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:30:37.121 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:37 smithi204 ceph-mon[32546]: from='client.15879 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:37.121 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:36 smithi204 podman[65294]: 2024-09-13 14:30:36.840586238 +0000 UTC m=+0.145596873 container died f7bdd6b72e4d42f5a0c14edf836cea3d22491f063aa51744f2dc765eec1dfda5 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:30:37.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:37 smithi096 ceph-mon[29229]: from='client.15879 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:37.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:37 smithi096 ceph-mon[39645]: from='client.15879 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:37.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:37 smithi204 podman[65294]: 2024-09-13 14:30:37.211254604 +0000 UTC m=+0.516265239 container cleanup f7bdd6b72e4d42f5a0c14edf836cea3d22491f063aa51744f2dc765eec1dfda5 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T14:30:37.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:37 smithi204 bash[65294]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x 2024-09-13T14:30:37.834 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:37 smithi204 podman[65306]: 2024-09-13 14:30:37.479391661 +0000 UTC m=+0.634512538 container remove f7bdd6b72e4d42f5a0c14edf836cea3d22491f063aa51744f2dc765eec1dfda5 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, RELEASE=quincy-bbdadfa) 2024-09-13T14:30:37.834 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:37 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:30:37.834 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:37 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T14:30:37.834 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:37 smithi204 systemd[1]: Stopped Ceph mgr.x for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:30:37.834 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:37 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.x.service: Consumed 10.290s CPU time. 2024-09-13T14:30:37.891 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:38.163 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:38 smithi204 ceph-mon[32546]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:38.163 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:38 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3924887619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:38.164 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:37 smithi204 systemd[1]: Starting Ceph mgr.x for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:30:38.164 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:38 smithi204 podman[65433]: 2024-09-13 14:30:37.94553761 +0000 UTC m=+0.015776414 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:30:38.164 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:38 smithi204 podman[65433]: 2024-09-13 14:30:38.07254635 +0000 UTC m=+0.142785186 container create 2a60f8e84e6f375ec5cf16394db9e90ed8166ec0bf38aeddd4266f1bfece6dd8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:30:38.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:38 smithi096 ceph-mon[39645]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:38.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:38 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3924887619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:38.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:38 smithi096 ceph-mon[29229]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:38.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:38 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3924887619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:38.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:30:37] "GET /metrics HTTP/1.1" 200 240252 "" "Prometheus/2.43.0" 2024-09-13T14:30:38.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:38 smithi204 podman[65433]: 2024-09-13 14:30:38.271211111 +0000 UTC m=+0.341449906 container init 2a60f8e84e6f375ec5cf16394db9e90ed8166ec0bf38aeddd4266f1bfece6dd8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:30:38.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:38 smithi204 podman[65433]: 2024-09-13 14:30:38.273998324 +0000 UTC m=+0.344237118 container start 2a60f8e84e6f375ec5cf16394db9e90ed8166ec0bf38aeddd4266f1bfece6dd8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:30:38.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:38 smithi204 ceph-mgr[65452]: -- 172.21.15.204:0/3490267556 <== mon.1 v2:172.21.15.204:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cd016c90e0 con 0x55cd016a7400 2024-09-13T14:30:38.469 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:38 smithi204 bash[65433]: 2a60f8e84e6f375ec5cf16394db9e90ed8166ec0bf38aeddd4266f1bfece6dd8 2024-09-13T14:30:38.469 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:38 smithi204 systemd[1]: Started Ceph mgr.x for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:30:39.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:39 smithi096 ceph-mon[29229]: from='client.15891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:39.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:39 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:39.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:39 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:39.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:39 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:39.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:39 smithi096 ceph-mon[39645]: from='client.15891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:39.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:39 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:39.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:39 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:39.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:39 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:39.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:39 smithi204 ceph-mon[32546]: from='client.15891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:39.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:39 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:39.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:39 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:39.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:39 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:40.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:40 smithi204 ceph-mon[32546]: from='client.15897 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:40.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:40 smithi204 ceph-mon[32546]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:40.244 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:40 smithi096 ceph-mon[29229]: from='client.15897 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:40.244 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:40 smithi096 ceph-mon[29229]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:40.244 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:40 smithi096 ceph-mon[39645]: from='client.15897 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:40.244 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:40 smithi096 ceph-mon[39645]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:40.749 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:41.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:41 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3948776948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:41 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3948776948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:41 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3948776948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:42.308 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:42 smithi204 ceph-mon[32546]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:42.308 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:42 smithi204 ceph-mon[32546]: from='client.15912 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:42.308 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:42 smithi204 ceph-mon[32546]: from='client.25823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:42.308 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:42 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:42.308 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:42 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:42.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:42 smithi096 ceph-mon[29229]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:42.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:42 smithi096 ceph-mon[29229]: from='client.15912 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:42.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:42 smithi096 ceph-mon[29229]: from='client.25823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:42.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:42 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:42.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:42 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:42.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:42 smithi096 ceph-mon[39645]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:42.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:42 smithi096 ceph-mon[39645]: from='client.15912 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:42.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:42 smithi096 ceph-mon[39645]: from='client.25823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:42.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:42 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:42.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:42 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:43.593 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:43.876 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: [13/Sep/2024:14:30:43] ENGINE Bus STARTING 2024-09-13T14:30:43.876 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: CherryPy Checker: 2024-09-13T14:30:43.876 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: The Application mounted at '' has an empty config. 2024-09-13T14:30:43.876 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: 2024-09-13T14:30:43.876 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: [13/Sep/2024:14:30:43] ENGINE Serving on http://:::9283 2024-09-13T14:30:43.876 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: [13/Sep/2024:14:30:43] ENGINE Bus STARTED 2024-09-13T14:30:44.130 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-mon[32546]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:44.130 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:44.130 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:44.130 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2898366899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:44.130 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:44.130 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:44.130 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-mon[32546]: Standby manager daemon x restarted 2024-09-13T14:30:44.130 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-mon[32546]: Standby manager daemon x started 2024-09-13T14:30:44.131 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:30:44.131 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:30:44.131 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:30:44.131 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:43 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:30:44.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[39645]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:44.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:44.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:44.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2898366899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:44.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:44.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:44.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[39645]: Standby manager daemon x restarted 2024-09-13T14:30:44.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[39645]: Standby manager daemon x started 2024-09-13T14:30:44.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:30:44.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:30:44.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:30:44.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:30:44.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[29229]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:44.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:44.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:44.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2898366899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:44.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:44.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:44.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[29229]: Standby manager daemon x restarted 2024-09-13T14:30:44.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[29229]: Standby manager daemon x started 2024-09-13T14:30:44.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:30:44.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:30:44.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:30:44.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:43 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:30:45.141 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:44 smithi204 ceph-mon[32546]: from='client.25835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:45.141 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:44 smithi204 ceph-mon[32546]: from='client.25841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:45.142 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:44 smithi204 ceph-mon[32546]: mgrmap e21: y(active, since 3m), standbys: x 2024-09-13T14:30:45.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:44 smithi096 ceph-mon[29229]: from='client.25835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:45.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:44 smithi096 ceph-mon[29229]: from='client.25841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:45.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:44 smithi096 ceph-mon[29229]: mgrmap e21: y(active, since 3m), standbys: x 2024-09-13T14:30:45.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:44 smithi096 ceph-mon[39645]: from='client.25835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:45.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:44 smithi096 ceph-mon[39645]: from='client.25841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:45.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:44 smithi096 ceph-mon[39645]: mgrmap e21: y(active, since 3m), standbys: x 2024-09-13T14:30:46.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:45 smithi204 ceph-mon[32546]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:45 smithi096 ceph-mon[29229]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:46.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:45 smithi096 ceph-mon[39645]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:46.425 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:47.188 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:46 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3019038788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:47.188 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:46 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3019038788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:47.188 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:47.052+0000 7f89f7cf2640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-13T14:30:47.188 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ignoring --setuser ceph since I am not root 2024-09-13T14:30:47.188 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ignoring --setgroup ceph since I am not root 2024-09-13T14:30:47.189 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mgr[29462]: -- 172.21.15.96:0/636647812 <== mon.0 v2:172.21.15.96:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5621e3872340 con 0x5621e387f000 2024-09-13T14:30:47.189 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:47.186+0000 7f1853d07ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:30:47.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:46 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3019038788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:47.218 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: [13/Sep/2024:14:30:47] ENGINE Bus STOPPING 2024-09-13T14:30:47.564 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:47.241+0000 7f1853d07ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:30:47.564 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:47.304+0000 7f1853d07ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:30:47.565 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:47.427+0000 7f1853d07ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:30:47.565 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:47.562+0000 7f1853d07ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:30:47.718 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: [13/Sep/2024:14:30:47] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:30:47.718 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: [13/Sep/2024:14:30:47] ENGINE Bus STOPPED 2024-09-13T14:30:48.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:47.813+0000 7f1853d07ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:30:48.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:47.872+0000 7f1853d07ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:30:48.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:48.035+0000 7f1853d07ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:30:48.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:48.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:48.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:30:48.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:30:48.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:48.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:48.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:30:48.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: Activating manager daemon x 2024-09-13T14:30:48.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T14:30:48.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T14:30:48.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: mgrmap e22: x(active, starting, since 0.0064552s) 2024-09-13T14:30:48.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:30:48.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:30:48.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:30:48.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi204.hbwbot"}]: dispatch 2024-09-13T14:30:48.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi096.dqiqpz"}]: dispatch 2024-09-13T14:30:48.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:30:48.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:30:48.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:30:48.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:30:48.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:30:48.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:30:48.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:30:48.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:30:48.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:30:48.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:30:48.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:30:48.076 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:30:48.076 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[39645]: Manager daemon x is now available 2024-09-13T14:30:48.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:48.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:48.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:30:48.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:30:48.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:48.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:48.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:30:48.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: Activating manager daemon x 2024-09-13T14:30:48.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T14:30:48.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T14:30:48.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: mgrmap e22: x(active, starting, since 0.0064552s) 2024-09-13T14:30:48.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:30:48.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:30:48.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:30:48.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi204.hbwbot"}]: dispatch 2024-09-13T14:30:48.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi096.dqiqpz"}]: dispatch 2024-09-13T14:30:48.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:30:48.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:30:48.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:30:48.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:30:48.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:30:48.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:30:48.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:30:48.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:30:48.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:30:48.079 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:30:48.079 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:30:48.079 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:30:48.079 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:47 smithi096 ceph-mon[29229]: Manager daemon x is now available 2024-09-13T14:30:48.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:48.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:48.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:30:48.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:30:48.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' 2024-09-13T14:30:48.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:48.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:30:48.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: Activating manager daemon x 2024-09-13T14:30:48.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T14:30:48.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.24335 172.21.15.96:0/1773651079' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T14:30:48.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: mgrmap e22: x(active, starting, since 0.0064552s) 2024-09-13T14:30:48.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:30:48.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:30:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:30:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi204.hbwbot"}]: dispatch 2024-09-13T14:30:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi096.dqiqpz"}]: dispatch 2024-09-13T14:30:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:30:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:30:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:30:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:30:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:30:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:30:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:30:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:30:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:30:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:30:48.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:30:48.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:30:48.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:47 smithi204 ceph-mon[32546]: Manager daemon x is now available 2024-09-13T14:30:48.660 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: [13/Sep/2024:14:30:48] ENGINE Bus STARTING 2024-09-13T14:30:48.660 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: 2024-09-13T14:30:48.340+0000 7f6a83e88640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:48.660 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: 2024-09-13T14:30:48.340+0000 7f6a83e88640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:48.660 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: 2024-09-13T14:30:48.340+0000 7f6a83e88640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:48.660 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: 2024-09-13T14:30:48.340+0000 7f6a83e88640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:48.661 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: 2024-09-13T14:30:48.340+0000 7f6a83e88640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:48.661 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: 2024-09-13T14:30:48.341+0000 7f6a80681640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:48.661 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: 2024-09-13T14:30:48.341+0000 7f6a80681640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:48.661 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: 2024-09-13T14:30:48.341+0000 7f6a80681640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:48.661 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: 2024-09-13T14:30:48.341+0000 7f6a80681640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:48.661 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: 2024-09-13T14:30:48.341+0000 7f6a80681640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:48.662 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: [13/Sep/2024:14:30:48] ENGINE Serving on http://:::9283 2024-09-13T14:30:48.662 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: [13/Sep/2024:14:30:48] ENGINE Bus STARTED 2024-09-13T14:30:49.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:48.919+0000 7f1853d07ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:30:49.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:48 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:48.985+0000 7f1853d07ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:30:49.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: mgrmap e23: x(active, since 1.00885s) 2024-09-13T14:30:49.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: Checking for cert/key for grafana.a 2024-09-13T14:30:49.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: Migrating grafana.a cert to cert store 2024-09-13T14:30:49.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: Migrating grafana.a key to cert store 2024-09-13T14:30:49.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:49.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:49.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:30:49.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:30:49.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:30:49.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:30:49.326 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:30:49.326 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:30:49.326 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3765868822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:49.326 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: mgrmap e23: x(active, since 1.00885s) 2024-09-13T14:30:49.326 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.327 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.327 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: Checking for cert/key for grafana.a 2024-09-13T14:30:49.327 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: Migrating grafana.a cert to cert store 2024-09-13T14:30:49.327 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.327 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: Migrating grafana.a key to cert store 2024-09-13T14:30:49.328 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.328 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.328 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.328 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:49.328 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:49.328 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:30:49.329 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:30:49.329 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:30:49.329 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:30:49.329 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:30:49.330 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:30:49.330 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3765868822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:49.338 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: mgrmap e23: x(active, since 1.00885s) 2024-09-13T14:30:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: Checking for cert/key for grafana.a 2024-09-13T14:30:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: Migrating grafana.a cert to cert store 2024-09-13T14:30:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: Migrating grafana.a key to cert store 2024-09-13T14:30:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:30:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:30:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:30:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:30:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:30:49.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:30:49.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3765868822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:49.969 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: 2024-09-13T14:30:49.573+0000 7f6a6cbe0640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:30:49] ENGINE Error in HTTPServer.serve 2024-09-13T14:30:49.969 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: Traceback (most recent call last): 2024-09-13T14:30:49.969 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:30:49.969 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: self._connections.run(self.expiration_interval) 2024-09-13T14:30:49.969 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:30:49.969 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: self._run(expiration_interval) 2024-09-13T14:30:49.969 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:30:49.970 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:30:49.970 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:30:49.970 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:30:49.970 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:30:49.970 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: s = self.context.wrap_socket( 2024-09-13T14:30:49.970 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:30:49.970 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: return self.sslsocket_class._create( 2024-09-13T14:30:49.970 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:30:49.971 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: self.do_handshake() 2024-09-13T14:30:49.971 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:30:49.971 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: self._sslobj.do_handshake() 2024-09-13T14:30:49.971 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:30:49.971 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: 2024-09-13T14:30:49.981 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:49.722+0000 7f1853d07ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:30:49.981 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:49.837+0000 7f1853d07ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:30:50.239 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-mon[29229]: Deploying cephadm binary to smithi096 2024-09-13T14:30:50.239 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-mon[29229]: Deploying cephadm binary to smithi204 2024-09-13T14:30:50.239 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-mon[29229]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:50.240 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-mon[39645]: Deploying cephadm binary to smithi096 2024-09-13T14:30:50.240 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-mon[39645]: Deploying cephadm binary to smithi204 2024-09-13T14:30:50.240 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-mon[39645]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:50.240 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:49 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:49.979+0000 7f1853d07ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:30:50.240 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:50.045+0000 7f1853d07ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:30:50.240 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:50.174+0000 7f1853d07ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:30:50.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:50 smithi204 ceph-mon[32546]: Deploying cephadm binary to smithi096 2024-09-13T14:30:50.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:50 smithi204 ceph-mon[32546]: Deploying cephadm binary to smithi204 2024-09-13T14:30:50.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:50 smithi204 ceph-mon[32546]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:50.513 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:50.242+0000 7f1853d07ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:30:50.513 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:50.303+0000 7f1853d07ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:30:50.513 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:50.445+0000 7f1853d07ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:30:50.513 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:50.511+0000 7f1853d07ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:30:50.796 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:50.578+0000 7f1853d07ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:30:50.797 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:50.658+0000 7f1853d07ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:30:50.797 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:50.721+0000 7f1853d07ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:30:50.797 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:50.794+0000 7f1853d07ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:30:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[29229]: from='client.25907 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[29229]: [13/Sep/2024:14:30:49] ENGINE Bus STARTING 2024-09-13T14:30:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[29229]: [13/Sep/2024:14:30:49] ENGINE Serving on https://172.21.15.204:7150 2024-09-13T14:30:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[29229]: [13/Sep/2024:14:30:49] ENGINE Error in HTTPServer.serve 2024-09-13T14:30:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout: Traceback (most recent call last): 2024-09-13T14:30:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:30:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:30:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:30:51.324 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._run(expiration_interval) 2024-09-13T14:30:51.324 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:30:51.324 INFO:journalctl@ceph.mon.a.smithi096.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:30:51.324 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:30:51.324 INFO:journalctl@ceph.mon.a.smithi096.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:30:51.324 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:30:51.324 INFO:journalctl@ceph.mon.a.smithi096.stdout: s = self.context.wrap_socket( 2024-09-13T14:30:51.324 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:30:51.325 INFO:journalctl@ceph.mon.a.smithi096.stdout: return self.sslsocket_class._create( 2024-09-13T14:30:51.325 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:30:51.325 INFO:journalctl@ceph.mon.a.smithi096.stdout: self.do_handshake() 2024-09-13T14:30:51.325 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:30:51.325 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._sslobj.do_handshake() 2024-09-13T14:30:51.325 INFO:journalctl@ceph.mon.a.smithi096.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:30:51.326 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[29229]: from='client.25913 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:51.326 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[29229]: [13/Sep/2024:14:30:49] ENGINE Serving on http://172.21.15.204:8765 2024-09-13T14:30:51.326 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[29229]: [13/Sep/2024:14:30:49] ENGINE Bus STARTED 2024-09-13T14:30:51.326 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[29229]: mgrmap e24: x(active, since 3s) 2024-09-13T14:30:51.326 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[39645]: from='client.25907 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:51.327 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[39645]: [13/Sep/2024:14:30:49] ENGINE Bus STARTING 2024-09-13T14:30:51.327 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[39645]: [13/Sep/2024:14:30:49] ENGINE Serving on https://172.21.15.204:7150 2024-09-13T14:30:51.327 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[39645]: [13/Sep/2024:14:30:49] ENGINE Error in HTTPServer.serve 2024-09-13T14:30:51.327 INFO:journalctl@ceph.mon.c.smithi096.stdout: Traceback (most recent call last): 2024-09-13T14:30:51.327 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:30:51.327 INFO:journalctl@ceph.mon.c.smithi096.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:30:51.327 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:30:51.328 INFO:journalctl@ceph.mon.c.smithi096.stdout: self._run(expiration_interval) 2024-09-13T14:30:51.328 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:30:51.328 INFO:journalctl@ceph.mon.c.smithi096.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:30:51.328 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:30:51.329 INFO:journalctl@ceph.mon.c.smithi096.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:30:51.329 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:30:51.329 INFO:journalctl@ceph.mon.c.smithi096.stdout: s = self.context.wrap_socket( 2024-09-13T14:30:51.329 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:30:51.329 INFO:journalctl@ceph.mon.c.smithi096.stdout: return self.sslsocket_class._create( 2024-09-13T14:30:51.329 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:30:51.329 INFO:journalctl@ceph.mon.c.smithi096.stdout: self.do_handshake() 2024-09-13T14:30:51.329 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:30:51.330 INFO:journalctl@ceph.mon.c.smithi096.stdout: self._sslobj.do_handshake() 2024-09-13T14:30:51.330 INFO:journalctl@ceph.mon.c.smithi096.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:30:51.330 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[39645]: from='client.25913 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:51.330 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[39645]: [13/Sep/2024:14:30:49] ENGINE Serving on http://172.21.15.204:8765 2024-09-13T14:30:51.330 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[39645]: [13/Sep/2024:14:30:49] ENGINE Bus STARTED 2024-09-13T14:30:51.330 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-mon[39645]: mgrmap e24: x(active, since 3s) 2024-09-13T14:30:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:51 smithi204 ceph-mon[32546]: from='client.25907 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:51 smithi204 ceph-mon[32546]: [13/Sep/2024:14:30:49] ENGINE Bus STARTING 2024-09-13T14:30:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:51 smithi204 ceph-mon[32546]: [13/Sep/2024:14:30:49] ENGINE Serving on https://172.21.15.204:7150 2024-09-13T14:30:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:51 smithi204 ceph-mon[32546]: [13/Sep/2024:14:30:49] ENGINE Error in HTTPServer.serve 2024-09-13T14:30:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout: Traceback (most recent call last): 2024-09-13T14:30:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:30:51.470 INFO:journalctl@ceph.mon.b.smithi204.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:30:51.470 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:30:51.470 INFO:journalctl@ceph.mon.b.smithi204.stdout: self._run(expiration_interval) 2024-09-13T14:30:51.470 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:30:51.470 INFO:journalctl@ceph.mon.b.smithi204.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:30:51.470 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:30:51.470 INFO:journalctl@ceph.mon.b.smithi204.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:30:51.471 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:30:51.471 INFO:journalctl@ceph.mon.b.smithi204.stdout: s = self.context.wrap_socket( 2024-09-13T14:30:51.471 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:30:51.471 INFO:journalctl@ceph.mon.b.smithi204.stdout: return self.sslsocket_class._create( 2024-09-13T14:30:51.471 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:30:51.471 INFO:journalctl@ceph.mon.b.smithi204.stdout: self.do_handshake() 2024-09-13T14:30:51.471 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:30:51.472 INFO:journalctl@ceph.mon.b.smithi204.stdout: self._sslobj.do_handshake() 2024-09-13T14:30:51.472 INFO:journalctl@ceph.mon.b.smithi204.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:30:51.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:51 smithi204 ceph-mon[32546]: from='client.25913 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:51.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:51 smithi204 ceph-mon[32546]: [13/Sep/2024:14:30:49] ENGINE Serving on http://172.21.15.204:8765 2024-09-13T14:30:51.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:51 smithi204 ceph-mon[32546]: [13/Sep/2024:14:30:49] ENGINE Bus STARTED 2024-09-13T14:30:51.472 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:51 smithi204 ceph-mon[32546]: mgrmap e24: x(active, since 3s) 2024-09-13T14:30:51.694 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:51.585+0000 7f1853d07ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:30:51.694 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:30:51] ENGINE Bus STARTING 2024-09-13T14:30:51.694 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: CherryPy Checker: 2024-09-13T14:30:51.694 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: The Application mounted at '' has an empty config. 2024-09-13T14:30:51.695 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: 2024-09-13T14:30:51.695 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:30:51] ENGINE Serving on http://:::9283 2024-09-13T14:30:51.695 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:51 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: [13/Sep/2024:14:30:51] ENGINE Bus STARTED 2024-09-13T14:30:52.261 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:52.261 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:52 smithi096 ceph-mon[29229]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:52.262 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:52 smithi096 ceph-mon[29229]: Standby manager daemon y started 2024-09-13T14:30:52.262 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:52 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.96:0/2314489145' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:30:52.262 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:52 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.96:0/2314489145' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:30:52.262 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:52 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.96:0/2314489145' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:30:52.262 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:52 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.96:0/2314489145' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:30:52.262 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:52 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2591848983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:52.262 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:52 smithi096 ceph-mon[39645]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:52.262 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:52 smithi096 ceph-mon[39645]: Standby manager daemon y started 2024-09-13T14:30:52.262 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:52 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.96:0/2314489145' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:30:52.262 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:52 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.96:0/2314489145' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:30:52.262 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:52 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.96:0/2314489145' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:30:52.263 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:52 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.96:0/2314489145' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:30:52.263 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:52 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2591848983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:52.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:30:52 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:30:52] "GET /metrics HTTP/1.1" 200 33508 "" "Prometheus/2.43.0" 2024-09-13T14:30:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:52 smithi204 ceph-mon[32546]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:52 smithi204 ceph-mon[32546]: Standby manager daemon y started 2024-09-13T14:30:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:52 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.96:0/2314489145' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:30:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:52 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.96:0/2314489145' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:30:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:52 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.96:0/2314489145' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:30:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:52 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.96:0/2314489145' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:30:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:52 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2591848983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: mgrmap e25: x(active, since 5s), standbys: y 2024-09-13T14:30:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:30:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: from='client.16020 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:53 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:53.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: mgrmap e25: x(active, since 5s), standbys: y 2024-09-13T14:30:53.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:30:53.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: from='client.16020 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:53.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:53.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:53.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: mgrmap e25: x(active, since 5s), standbys: y 2024-09-13T14:30:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:30:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: from='client.16020 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:30:53.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:53 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:30:54.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:54 smithi204 ceph-mon[32546]: from='client.16026 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:54 smithi204 ceph-mon[32546]: Adjusting osd_memory_target on smithi096 to 2480M 2024-09-13T14:30:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:54 smithi204 ceph-mon[32546]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 614 B/s wr, 15 op/s 2024-09-13T14:30:54.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:54 smithi096 ceph-mon[29229]: from='client.16026 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:54.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:54 smithi096 ceph-mon[29229]: Adjusting osd_memory_target on smithi096 to 2480M 2024-09-13T14:30:54.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:54 smithi096 ceph-mon[29229]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 614 B/s wr, 15 op/s 2024-09-13T14:30:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:54 smithi096 ceph-mon[39645]: from='client.16026 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:54 smithi096 ceph-mon[39645]: Adjusting osd_memory_target on smithi096 to 2480M 2024-09-13T14:30:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:54 smithi096 ceph-mon[39645]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 614 B/s wr, 15 op/s 2024-09-13T14:30:55.184 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:55 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3156068364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:55.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:55 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3156068364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:55.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:55 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3156068364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:56.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:56 smithi204 ceph-mon[32546]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 438 B/s wr, 10 op/s 2024-09-13T14:30:56.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:56 smithi204 ceph-mon[32546]: from='client.16038 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:56.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:56 smithi096 ceph-mon[29229]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 438 B/s wr, 10 op/s 2024-09-13T14:30:56.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:56 smithi096 ceph-mon[29229]: from='client.16038 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:56.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:56 smithi096 ceph-mon[39645]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 438 B/s wr, 10 op/s 2024-09-13T14:30:56.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:56 smithi096 ceph-mon[39645]: from='client.16038 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:57 smithi204 ceph-mon[32546]: from='client.25955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:57.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:57 smithi096 ceph-mon[29229]: from='client.25955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:57.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:57 smithi096 ceph-mon[39645]: from='client.25955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:58.089 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:30:58.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:58 smithi096 ceph-mon[29229]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 8 op/s 2024-09-13T14:30:58.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:58 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3104120509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:58.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:30:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:30:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:30:58.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:58 smithi096 ceph-mon[39645]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 8 op/s 2024-09-13T14:30:58.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:58 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3104120509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:58.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:58 smithi204 ceph-mon[32546]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 8 op/s 2024-09-13T14:30:58.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:58 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3104120509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:30:59 smithi204 ceph-mon[32546]: from='client.16056 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:30:59 smithi096 ceph-mon[29229]: from='client.16056 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:59.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:30:59 smithi096 ceph-mon[39645]: from='client.16056 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:00 smithi204 ceph-mon[32546]: from='client.16062 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:00 smithi204 ceph-mon[32546]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 6 op/s 2024-09-13T14:31:00.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:00 smithi096 ceph-mon[29229]: from='client.16062 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:00.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:00 smithi096 ceph-mon[29229]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 6 op/s 2024-09-13T14:31:00.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:00 smithi096 ceph-mon[39645]: from='client.16062 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:00.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:00 smithi096 ceph-mon[39645]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 6 op/s 2024-09-13T14:31:00.986 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:01.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:01 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/526123267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:01.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:01 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/526123267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:01.333 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:01 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/526123267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:02.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:02 smithi204 ceph-mon[32546]: from='client.25985 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:02.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:02 smithi204 ceph-mon[32546]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-13T14:31:02.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:02 smithi204 ceph-mon[32546]: from='client.16080 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:02.469 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:31:02 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:31:02] "GET /metrics HTTP/1.1" 200 33508 "" "Prometheus/2.43.0" 2024-09-13T14:31:02.491 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:02 smithi096 ceph-mon[39645]: from='client.25985 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:02.492 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:02 smithi096 ceph-mon[39645]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-13T14:31:02.492 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:02 smithi096 ceph-mon[39645]: from='client.16080 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:02.492 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:02 smithi096 ceph-mon[29229]: from='client.25985 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:02.492 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:02 smithi096 ceph-mon[29229]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-13T14:31:02.492 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:02 smithi096 ceph-mon[29229]: from='client.16080 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:03.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:03 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:03.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:03 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:03.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:03 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:03.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:03 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:03.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:03 smithi204 ceph-mon[32546]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-13T14:31:03.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:03 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:03.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:03 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:03.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:03 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:03.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:03 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:03.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:03 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:03.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:03 smithi096 ceph-mon[29229]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-13T14:31:03.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:03 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:03.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:03 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:03.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:03 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:03.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:03 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:03.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:03 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:03.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:03 smithi096 ceph-mon[39645]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-13T14:31:03.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:03 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:03.887 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:04.627 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:04 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1642862529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:04.627 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:04 smithi204 ceph-mon[32546]: from='client.16092 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:04.627 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:04 smithi204 ceph-mon[32546]: from='client.26009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:04.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:04 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1642862529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:04.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:04 smithi096 ceph-mon[29229]: from='client.16092 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:04.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:04 smithi096 ceph-mon[29229]: from='client.26009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:04.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:04 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1642862529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:04.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:04 smithi096 ceph-mon[39645]: from='client.16092 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:04.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:04 smithi096 ceph-mon[39645]: from='client.26009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:05 smithi204 ceph-mon[32546]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:05.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:05 smithi096 ceph-mon[29229]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:05.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:05 smithi096 ceph-mon[39645]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:06.810 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:07.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: Adjusting osd_memory_target on smithi204 to 2480M 2024-09-13T14:31:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:31:07.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:31:07.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:31:07.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2576396568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:07.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:07.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:07 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:07.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:07.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: Adjusting osd_memory_target on smithi204 to 2480M 2024-09-13T14:31:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:31:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:31:07.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:31:07.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2576396568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:07.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:07.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:07.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:07.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:07.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:31:07.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: Adjusting osd_memory_target on smithi204 to 2480M 2024-09-13T14:31:07.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:07.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:07.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:31:07.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:31:07.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:31:07.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2576396568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:07.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:07.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.227 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:31:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:31:08.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:31:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:31:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:31:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:31:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:31:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: from='client.16110 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:31:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: from='client.16116 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:31:08.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:31:08.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:31:08.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:31:08.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:31:08.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: from='client.16110 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:08.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:31:08.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:08.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: from='client.16116 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:08.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:08 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:31:08.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:31:08.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:31:08.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:31:08.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:31:08.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: from='client.16110 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:08.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:31:08.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:08.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: from='client.16116 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:08.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:08.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:08 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:09.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:09 smithi096 ceph-mon[29229]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:31:09.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:09 smithi096 ceph-mon[29229]: Deploying daemon alertmanager.a on smithi096 2024-09-13T14:31:09.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:09 smithi096 ceph-mon[39645]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:31:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:09 smithi096 ceph-mon[39645]: Deploying daemon alertmanager.a on smithi096 2024-09-13T14:31:09.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:09 smithi204 ceph-mon[32546]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:31:09.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:09 smithi204 ceph-mon[32546]: Deploying daemon alertmanager.a on smithi096 2024-09-13T14:31:09.774 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:10.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:10 smithi096 ceph-mon[29229]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:10.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:10 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3194520824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:10.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:10 smithi096 ceph-mon[39645]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:10.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:10 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3194520824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:10.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:10 smithi204 ceph-mon[32546]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:10.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:10 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3194520824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:11.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:11 smithi096 ceph-mon[29229]: from='client.16128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:11.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:11 smithi096 ceph-mon[29229]: from='client.26045 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:11.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:11 smithi096 ceph-mon[39645]: from='client.16128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:11.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:11 smithi096 ceph-mon[39645]: from='client.26045 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:11 smithi204 ceph-mon[32546]: from='client.16128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:11 smithi204 ceph-mon[32546]: from='client.26045 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:12.376 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:12 smithi096 ceph-mon[29229]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:12.376 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:12 smithi096 ceph-mon[39645]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:12.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:12 smithi204 ceph-mon[32546]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:12.469 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:31:12 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:31:12] "GET /metrics HTTP/1.1" 200 33857 "" "Prometheus/2.43.0" 2024-09-13T14:31:12.690 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:13.519 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:13 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/4069715926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:13.519 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:13 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/4069715926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:13.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:13 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/4069715926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:14.481 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:14 smithi096 ceph-mon[29229]: from='client.26057 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:14.481 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:14 smithi096 ceph-mon[29229]: from='client.26063 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:14.481 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:14 smithi096 ceph-mon[29229]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:14.481 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:14 smithi096 ceph-mon[39645]: from='client.26057 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:14.481 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:14 smithi096 ceph-mon[39645]: from='client.26063 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:14.481 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:14 smithi096 ceph-mon[39645]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:14.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:14 smithi204 ceph-mon[32546]: from='client.26057 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:14.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:14 smithi204 ceph-mon[32546]: from='client.26063 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:14.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:14 smithi204 ceph-mon[32546]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:14.822 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:14 smithi096 systemd[1]: Stopping Ceph alertmanager.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:31:15.278 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:14 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[72700]: ts=2024-09-13T14:31:14.944Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:31:15.278 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:14 smithi096 podman[87492]: 2024-09-13 14:31:14.985071191 +0000 UTC m=+0.157292810 container died c2fb084ac3b6e330733e37f7bd38b11aca67a7db23c80816f46203a383e6a51b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:15.572 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:15 smithi096 podman[87492]: 2024-09-13 14:31:15.377155643 +0000 UTC m=+0.549377266 container cleanup c2fb084ac3b6e330733e37f7bd38b11aca67a7db23c80816f46203a383e6a51b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:15.572 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:15 smithi096 bash[87492]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T14:31:15.636 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:16.070 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:15 smithi096 podman[87506]: 2024-09-13 14:31:15.628586597 +0000 UTC m=+0.638853766 container remove c2fb084ac3b6e330733e37f7bd38b11aca67a7db23c80816f46203a383e6a51b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:16.070 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:15 smithi096 podman[87506]: 2024-09-13 14:31:15.712703571 +0000 UTC m=+0.722970739 volume remove 6ecc162b67749b319951cbd4a559f2655f4af512d418429fd3a9870b59d17ab5 2024-09-13T14:31:16.070 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:15 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T14:31:16.070 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:15 smithi096 systemd[1]: Stopped Ceph alertmanager.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:31:16.070 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:15 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 1.041s CPU time. 2024-09-13T14:31:16.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:16 smithi096 ceph-mon[29229]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:16.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:16 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1875477239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:16.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:16 smithi096 ceph-mon[39645]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:16.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:16 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1875477239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:16.323 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 systemd[1]: Starting Ceph alertmanager.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:31:16.323 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 podman[87702]: 2024-09-13 14:31:16.256017979 +0000 UTC m=+0.095780956 volume create 8d0e8a14e572f640da0e9347ec1093af31e275f22776f36071fa49466c78834c 2024-09-13T14:31:16.323 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 podman[87702]: 2024-09-13 14:31:16.176068497 +0000 UTC m=+0.015831473 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-09-13T14:31:16.668 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 podman[87702]: 2024-09-13 14:31:16.35672507 +0000 UTC m=+0.196488046 container create e04c10a0dca29b51d31413f06f949393267b8971acc674e6450577429d4584fd (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:16.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:16 smithi204 ceph-mon[32546]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:16.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:16 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1875477239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:17.072 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 podman[87702]: 2024-09-13 14:31:16.668081484 +0000 UTC m=+0.507844461 container init e04c10a0dca29b51d31413f06f949393267b8971acc674e6450577429d4584fd (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:17.072 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 podman[87702]: 2024-09-13 14:31:16.671238445 +0000 UTC m=+0.511001422 container start e04c10a0dca29b51d31413f06f949393267b8971acc674e6450577429d4584fd (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:17.073 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[87717]: ts=2024-09-13T14:31:16.681Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-09-13T14:31:17.073 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[87717]: ts=2024-09-13T14:31:16.681Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-09-13T14:31:17.073 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[87717]: ts=2024-09-13T14:31:16.682Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.96 port=9094 2024-09-13T14:31:17.073 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[87717]: ts=2024-09-13T14:31:16.683Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T14:31:17.073 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[87717]: ts=2024-09-13T14:31:16.705Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:31:17.073 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[87717]: ts=2024-09-13T14:31:16.705Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:31:17.073 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[87717]: ts=2024-09-13T14:31:16.707Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-09-13T14:31:17.073 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[87717]: ts=2024-09-13T14:31:16.707Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T14:31:17.073 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 bash[87702]: e04c10a0dca29b51d31413f06f949393267b8971acc674e6450577429d4584fd 2024-09-13T14:31:17.073 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:16 smithi096 systemd[1]: Started Ceph alertmanager.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:31:17.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:17 smithi096 ceph-mon[29229]: from='client.16164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:17.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:17 smithi096 ceph-mon[29229]: from='client.26081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:17.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:17 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:17.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:17 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:17.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:17 smithi096 ceph-mon[39645]: from='client.16164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:17.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:17 smithi096 ceph-mon[39645]: from='client.26081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:17.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:17 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:17.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:17 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:17.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:17 smithi204 ceph-mon[32546]: from='client.16164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:17.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:17 smithi204 ceph-mon[32546]: from='client.26081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:17.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:17 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:17.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:17 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:18.301 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:18 smithi096 systemd[1]: Stopping Ceph node-exporter.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:31:18.301 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:31:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:31:18.553 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:18 smithi096 podman[87901]: 2024-09-13 14:31:18.37827636 +0000 UTC m=+0.137027735 container died 4e13850242054193620c4422811e60f54fdcb07b990ec8218c0e671132db0de8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:18.553 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:18 smithi096 ceph-mon[29229]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T14:31:18.553 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:18 smithi096 ceph-mon[29229]: Deploying daemon node-exporter.a on smithi096 2024-09-13T14:31:18.553 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:18 smithi096 ceph-mon[29229]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:18.553 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:18 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:18.553 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:18 smithi096 ceph-mon[39645]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T14:31:18.554 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:18 smithi096 ceph-mon[39645]: Deploying daemon node-exporter.a on smithi096 2024-09-13T14:31:18.554 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:18 smithi096 ceph-mon[39645]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:18.554 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:18 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:18.611 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:18.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:18 smithi204 ceph-mon[32546]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T14:31:18.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:18 smithi204 ceph-mon[32546]: Deploying daemon node-exporter.a on smithi096 2024-09-13T14:31:18.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:18 smithi204 ceph-mon[32546]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:18.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:18 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:18.822 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:18 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[87717]: ts=2024-09-13T14:31:18.684Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000276097s 2024-09-13T14:31:19.305 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:18 smithi096 podman[87901]: 2024-09-13 14:31:18.901145797 +0000 UTC m=+0.659897170 container cleanup 4e13850242054193620c4422811e60f54fdcb07b990ec8218c0e671132db0de8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:19.305 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:18 smithi096 bash[87901]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T14:31:19.305 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:19 smithi096 podman[87918]: 2024-09-13 14:31:19.119326282 +0000 UTC m=+0.741230421 container remove 4e13850242054193620c4422811e60f54fdcb07b990ec8218c0e671132db0de8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:19.305 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:19 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:31:19.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:19 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/885936433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:19.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:19 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/885936433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:19.573 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:19 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T14:31:19.575 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:19 smithi096 systemd[1]: Stopped Ceph node-exporter.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:31:19.576 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:19 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 2.089s CPU time. 2024-09-13T14:31:19.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:19 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/885936433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:20.072 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:19 smithi096 systemd[1]: Starting Ceph node-exporter.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:31:20.072 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:19 smithi096 bash[88102]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-13T14:31:20.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:20 smithi096 ceph-mon[39645]: from='client.16182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:20.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:20 smithi096 ceph-mon[39645]: from='client.16188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:20.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:20 smithi096 ceph-mon[39645]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:20.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:20 smithi096 ceph-mon[29229]: from='client.16182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:20.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:20 smithi096 ceph-mon[29229]: from='client.16188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:20.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:20 smithi096 ceph-mon[29229]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:20.573 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:20 smithi096 bash[88102]: Getting image source signatures 2024-09-13T14:31:20.574 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:20 smithi096 bash[88102]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-13T14:31:20.574 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:20 smithi096 bash[88102]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-13T14:31:20.574 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:20 smithi096 bash[88102]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-13T14:31:20.620 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:20 smithi204 ceph-mon[32546]: from='client.16182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:20.620 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:20 smithi204 ceph-mon[32546]: from='client.16188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:20.620 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:20 smithi204 ceph-mon[32546]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:21.571 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:21 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2288194380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:21.571 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:21 smithi096 bash[88102]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-13T14:31:21.571 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:21 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2288194380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:21.572 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:21.634 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:21 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2288194380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:21.822 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:21 smithi096 bash[88102]: Writing manifest to image destination 2024-09-13T14:31:22.158 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:21 smithi096 podman[88102]: 2024-09-13 14:31:21.807739832 +0000 UTC m=+2.125160249 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-13T14:31:22.159 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:21 smithi096 podman[88102]: 2024-09-13 14:31:21.931416991 +0000 UTC m=+2.248837401 container create b02b57424615009493d722b9392c17ec710c4ab0d10ec1135a5c97a42c7c6c0f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:22.218 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:31:22 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:31:22] "GET /metrics HTTP/1.1" 200 33857 "" "Prometheus/2.43.0" 2024-09-13T14:31:22.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-mon[39645]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:22.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-mon[39645]: from='client.16200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:22.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-mon[39645]: from='client.16206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:22.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:22.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:22.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-mon[39645]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:31:22.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-mon[29229]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:22.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-mon[29229]: from='client.16200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:22.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-mon[29229]: from='client.16206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:22.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:22.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:22.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-mon[29229]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:31:22.574 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 podman[88102]: 2024-09-13 14:31:22.158535776 +0000 UTC m=+2.475956186 container init b02b57424615009493d722b9392c17ec710c4ab0d10ec1135a5c97a42c7c6c0f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:22.574 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 podman[88102]: 2024-09-13 14:31:22.16173631 +0000 UTC m=+2.479156728 container start b02b57424615009493d722b9392c17ec710c4ab0d10ec1135a5c97a42c7c6c0f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:22.574 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.165Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-13T14:31:22.574 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.165Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-13T14:31:22.575 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.165Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:31:22.575 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.165Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:31:22.575 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:31:22.575 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:31:22.575 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:31:22.575 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:31:22.575 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:31:22.575 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:31:22.575 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:31:22.575 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:31:22.575 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:31:22.575 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:31:22.575 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:31:22.576 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:31:22.576 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:31:22.576 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:31:22.576 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:31:22.576 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:31:22.576 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:31:22.576 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:31:22.576 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:31:22.576 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:31:22.576 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:31:22.576 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:31:22.576 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:31:22.577 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:31:22.577 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:31:22.577 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:31:22.577 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:31:22.577 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:31:22.577 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:31:22.577 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:31:22.577 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:31:22.577 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:31:22.577 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:31:22.577 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:31:22.577 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:31:22.578 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:31:22.578 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:31:22.578 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:31:22.578 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:31:22.578 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:31:22.578 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:31:22.578 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:31:22.578 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:31:22.578 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:31:22.578 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:31:22.578 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:31:22.578 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:31:22.578 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:31:22.578 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a[88250]: ts=2024-09-13T14:31:22.166Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:31:22.579 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 bash[88102]: b02b57424615009493d722b9392c17ec710c4ab0d10ec1135a5c97a42c7c6c0f 2024-09-13T14:31:22.579 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 14:31:22 smithi096 systemd[1]: Started Ceph node-exporter.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:31:22.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:22 smithi204 ceph-mon[32546]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:22.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:22 smithi204 ceph-mon[32546]: from='client.16200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:22.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:22 smithi204 ceph-mon[32546]: from='client.16206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:22.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:22 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:22.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:22 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:22.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:22 smithi204 ceph-mon[32546]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:31:23.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:23 smithi096 ceph-mon[29229]: Deploying daemon prometheus.a on smithi204 2024-09-13T14:31:23.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:23 smithi096 ceph-mon[29229]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:23.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:23 smithi096 ceph-mon[39645]: Deploying daemon prometheus.a on smithi204 2024-09-13T14:31:23.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:23 smithi096 ceph-mon[39645]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:23.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:23 smithi204 ceph-mon[32546]: Deploying daemon prometheus.a on smithi204 2024-09-13T14:31:23.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:23 smithi204 ceph-mon[32546]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:24.476 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:24.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:24 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/573497745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:24.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:24 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/573497745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:24.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:24 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/573497745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:25.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:25 smithi096 ceph-mon[29229]: from='client.16218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:25.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:25 smithi096 ceph-mon[29229]: from='client.26135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:25.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:25 smithi096 ceph-mon[29229]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:25.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:25 smithi096 ceph-mon[39645]: from='client.16218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:25.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:25 smithi096 ceph-mon[39645]: from='client.26135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:25.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:25 smithi096 ceph-mon[39645]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:25.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:25 smithi204 ceph-mon[32546]: from='client.16218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:25.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:25 smithi204 ceph-mon[32546]: from='client.26135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:25.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:25 smithi204 ceph-mon[32546]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:27.072 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 14:31:26 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[87717]: ts=2024-09-13T14:31:26.687Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003931726s 2024-09-13T14:31:27.374 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:27.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:27 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2735581800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:27.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:27 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2735581800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:27.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:27 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2735581800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:28.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:28 smithi096 ceph-mon[29229]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:28.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:28 smithi096 ceph-mon[39645]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:28.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[29458]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:31:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:31:28.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:28 smithi204 ceph-mon[32546]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:29.395 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:29 smithi096 ceph-mon[29229]: from='client.16236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:29.395 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:29 smithi096 ceph-mon[29229]: from='client.26153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:29.395 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:29 smithi096 ceph-mon[39645]: from='client.16236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:29.396 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:29 smithi096 ceph-mon[39645]: from='client.26153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:29.405 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:29 smithi204 ceph-mon[32546]: from='client.16236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:29.405 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:29 smithi204 ceph-mon[32546]: from='client.26153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:30.261 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:30.409 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:30 smithi204 ceph-mon[32546]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:30.409 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:30 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/246649283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:30.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:30 smithi096 ceph-mon[39645]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:30.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:30 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/246649283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:30.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:30 smithi096 ceph-mon[29229]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:30.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:30 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/246649283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:31.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:31 smithi204 ceph-mon[32546]: from='client.16254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:31.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:31 smithi096 ceph-mon[29229]: from='client.16254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:31.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:31 smithi096 ceph-mon[39645]: from='client.16254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:32.390 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:32 smithi204 ceph-mon[32546]: from='client.16260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:32.390 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:32 smithi204 ceph-mon[32546]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:32.390 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:31:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:31:32] "GET /metrics HTTP/1.1" 200 33857 "" "Prometheus/2.43.0" 2024-09-13T14:31:32.391 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 systemd[1]: Stopping Ceph prometheus.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:31:32.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:32 smithi096 ceph-mon[29229]: from='client.16260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:32.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:32 smithi096 ceph-mon[29229]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:32.474 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:32 smithi096 ceph-mon[39645]: from='client.16260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:32.474 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:32 smithi096 ceph-mon[39645]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:32.707 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:31:32.389Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:31:32.708 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:31:32.389Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:31:32.708 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:31:32.389Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:31:32.708 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:31:32.389Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:31:32.708 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:31:32.389Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:31:32.708 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:31:32.389Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T14:31:32.708 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:31:32.390Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:31:32.708 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:31:32.390Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T14:31:32.708 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:31:32.390Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T14:31:32.708 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:31:32.390Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:31:32.708 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:31:32.390Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T14:31:32.708 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[62681]: ts=2024-09-13T14:31:32.390Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T14:31:32.709 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 podman[73842]: 2024-09-13 14:31:32.399858112 +0000 UTC m=+0.124138067 container died 420af5726f6f10b9e2ab86b8caf914b184ff25a8ae6e18c4ad7f1881243fefcb (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:32.967 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 podman[73842]: 2024-09-13 14:31:32.890263532 +0000 UTC m=+0.614543483 container cleanup 420af5726f6f10b9e2ab86b8caf914b184ff25a8ae6e18c4ad7f1881243fefcb (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:32.967 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:32 smithi204 bash[73842]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:31:33.187 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:33.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/251936905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:33.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 podman[73855]: 2024-09-13 14:31:33.150607214 +0000 UTC m=+0.752546665 container remove 420af5726f6f10b9e2ab86b8caf914b184ff25a8ae6e18c4ad7f1881243fefcb (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:33.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:33 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/251936905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:33.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:33 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/251936905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:33.718 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:31:33.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 systemd[1]: Stopped Ceph prometheus.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:31:33.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.620s CPU time. 2024-09-13T14:31:33.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 systemd[1]: Starting Ceph prometheus.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:31:33.982 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 podman[73983]: 2024-09-13 14:31:33.63447695 +0000 UTC m=+0.017496466 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-13T14:31:33.983 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 podman[73983]: 2024-09-13 14:31:33.752294496 +0000 UTC m=+0.135314043 container create e7fc3eeabfeed520c876f2d46b39048c36525e68736ae2c76d3fea8eb7a55941 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:33.983 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 podman[73983]: 2024-09-13 14:31:33.954305027 +0000 UTC m=+0.337324531 container init e7fc3eeabfeed520c876f2d46b39048c36525e68736ae2c76d3fea8eb7a55941 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:33.983 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 podman[73983]: 2024-09-13 14:31:33.957121525 +0000 UTC m=+0.340141027 container start e7fc3eeabfeed520c876f2d46b39048c36525e68736ae2c76d3fea8eb7a55941 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:31:33.983 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.975Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-13T14:31:33.983 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.975Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-13T14:31:33.983 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.975Z caller=main.go:623 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi204 (none))" 2024-09-13T14:31:33.983 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.975Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:31:33.983 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.975Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:31:33.983 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.977Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:31:33.983 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.978Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-13T14:31:33.983 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.980Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:31:33.983 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.980Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:31:33.984 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.981Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:31:33.984 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.981Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.719µs 2024-09-13T14:31:33.984 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.981Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:31:34.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:34 smithi204 ceph-mon[32546]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:34.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:34 smithi204 ceph-mon[32546]: from='client.26183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:34.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:34 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:34.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:34 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:34.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:34 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:34.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.982Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-13T14:31:34.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.993Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-13T14:31:34.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.993Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-13T14:31:34.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.993Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=23.264µs wal_replay_duration=11.407513ms wbl_replay_duration=113ns total_replay_duration=11.448567ms 2024-09-13T14:31:34.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.995Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:31:34.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.996Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-13T14:31:34.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:33 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:33.996Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:31:34.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:34 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:34.014Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.741942ms db_storage=1.005µs remote_storage=1.158µs web_handler=706ns query_engine=688ns scrape=403.366µs scrape_sd=96.607µs notify=10.916µs notify_sd=11.485µs rules=17.8545ms tracing=6.008µs 2024-09-13T14:31:34.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:34 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:34.014Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-13T14:31:34.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:34 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:34.014Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:31:34.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:34 smithi204 bash[73983]: e7fc3eeabfeed520c876f2d46b39048c36525e68736ae2c76d3fea8eb7a55941 2024-09-13T14:31:34.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:34 smithi204 systemd[1]: Started Ceph prometheus.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:31:34.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:34 smithi096 ceph-mon[29229]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:34.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:34 smithi096 ceph-mon[29229]: from='client.26183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:34.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:34 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:34.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:34 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:34.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:34 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:34.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:34 smithi096 ceph-mon[39645]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:34.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:34 smithi096 ceph-mon[39645]: from='client.26183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:34.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:34 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:34.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:34 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:34.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:34 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:35.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:35 smithi204 ceph-mon[32546]: from='client.16278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:35.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:35 smithi204 ceph-mon[32546]: Reconfiguring grafana.a (dependencies changed)... 2024-09-13T14:31:35.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:35 smithi204 ceph-mon[32546]: Reconfiguring daemon grafana.a on smithi204 2024-09-13T14:31:35.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:35 smithi096 ceph-mon[29229]: from='client.16278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:35.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:35 smithi096 ceph-mon[29229]: Reconfiguring grafana.a (dependencies changed)... 2024-09-13T14:31:35.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:35 smithi096 ceph-mon[29229]: Reconfiguring daemon grafana.a on smithi204 2024-09-13T14:31:35.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:35 smithi096 ceph-mon[39645]: from='client.16278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:35.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:35 smithi096 ceph-mon[39645]: Reconfiguring grafana.a (dependencies changed)... 2024-09-13T14:31:35.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:35 smithi096 ceph-mon[39645]: Reconfiguring daemon grafana.a on smithi204 2024-09-13T14:31:36.115 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:36.259 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:36 smithi204 ceph-mon[32546]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:36.259 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:36 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2465979097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:36.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:36 smithi096 ceph-mon[29229]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:36.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:36 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2465979097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:36.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:36 smithi096 ceph-mon[39645]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:36.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:36 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2465979097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:37.160 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:36 smithi204 systemd[1]: Stopping Ceph grafana.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:31:37.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:37 smithi204 ceph-mon[32546]: from='client.16290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:37.469 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:37 smithi204 podman[74216]: 2024-09-13 14:31:37.159428705 +0000 UTC m=+0.143623500 container died a693d43fc847340a2c069cd5589f86c2ab2a0a5590d5bba111eb05b52c4b78d9 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, vcs-type=git, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, description=Ceph Grafana Container, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, architecture=x86_64) 2024-09-13T14:31:37.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:37 smithi096 ceph-mon[29229]: from='client.16290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:37.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:37 smithi096 ceph-mon[39645]: from='client.16290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:37.969 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:37 smithi204 podman[74216]: 2024-09-13 14:31:37.76594397 +0000 UTC m=+0.750138764 container cleanup a693d43fc847340a2c069cd5589f86c2ab2a0a5590d5bba111eb05b52c4b78d9 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, io.openshift.expose-services=, architecture=x86_64, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, vcs-type=git, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, release=239.1651231664, name=ubi8) 2024-09-13T14:31:37.969 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:37 smithi204 bash[74216]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a 2024-09-13T14:31:38.323 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-mon[32546]: from='client.26207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:38.324 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-mon[32546]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:38.324 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:37 smithi204 podman[74228]: 2024-09-13 14:31:37.983665543 +0000 UTC m=+0.825896745 container remove a693d43fc847340a2c069cd5589f86c2ab2a0a5590d5bba111eb05b52c4b78d9 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, name=ubi8, distribution-scope=public, release=239.1651231664, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.openshift.expose-services=, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0) 2024-09-13T14:31:38.324 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:37 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:31:38.324 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-13T14:31:38.324 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 systemd[1]: Stopped Ceph grafana.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:31:38.324 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@grafana.a.service: Consumed 5.315s CPU time. 2024-09-13T14:31:38.324 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 systemd[1]: Starting Ceph grafana.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:31:38.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:38 smithi096 ceph-mon[39645]: from='client.26207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:38.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:38 smithi096 ceph-mon[39645]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:38.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:38 smithi096 ceph-mon[29229]: from='client.26207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:38.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:38 smithi096 ceph-mon[29229]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:38.579 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 podman[74321]: 2024-09-13 14:31:38.225074699 +0000 UTC m=+0.015518473 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-13T14:31:38.580 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 podman[74321]: 2024-09-13 14:31:38.359639592 +0000 UTC m=+0.150083357 container create 451523aebd73725094363334047a26fda8e2f23df04d92394ffac5bbd73e4ea6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, description=Ceph Grafana Container, vendor=Red Hat, Inc., name=ubi8, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5) 2024-09-13T14:31:38.850 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 podman[74321]: 2024-09-13 14:31:38.579466493 +0000 UTC m=+0.369910279 container init 451523aebd73725094363334047a26fda8e2f23df04d92394ffac5bbd73e4ea6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, vendor=Red Hat, Inc., io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, io.openshift.tags=base rhel8, version=8.5, distribution-scope=public, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-09-13T14:31:38.850 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 podman[74321]: 2024-09-13 14:31:38.582611235 +0000 UTC m=+0.373055008 container start 451523aebd73725094363334047a26fda8e2f23df04d92394ffac5bbd73e4ea6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a, description=Ceph Grafana Container, version=8.5, release=239.1651231664, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, distribution-scope=public, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-09-13T14:31:38.850 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 bash[74321]: 451523aebd73725094363334047a26fda8e2f23df04d92394ffac5bbd73e4ea6 2024-09-13T14:31:38.850 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 systemd[1]: Started Ceph grafana.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:31:38.850 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=settings t=2024-09-13T14:31:38.742381366Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-13T14:31:38.850 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=settings t=2024-09-13T14:31:38.742601406Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-13T14:31:38.850 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=settings t=2024-09-13T14:31:38.742629285Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T14:31:38.850 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=settings t=2024-09-13T14:31:38.742643922Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T14:31:38.851 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=settings t=2024-09-13T14:31:38.742655753Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T14:31:38.851 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=settings t=2024-09-13T14:31:38.742668443Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T14:31:38.851 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=settings t=2024-09-13T14:31:38.742679413Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T14:31:38.851 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=settings t=2024-09-13T14:31:38.74269032Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T14:31:38.851 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=settings t=2024-09-13T14:31:38.742701298Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T14:31:38.851 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=settings t=2024-09-13T14:31:38.742712049Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T14:31:38.851 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=settings t=2024-09-13T14:31:38.742723482Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T14:31:38.851 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=settings t=2024-09-13T14:31:38.742734378Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T14:31:38.851 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=settings t=2024-09-13T14:31:38.742745567Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T14:31:38.851 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=settings t=2024-09-13T14:31:38.742756748Z level=info msg="App mode production" 2024-09-13T14:31:38.851 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=sqlstore t=2024-09-13T14:31:38.742825657Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T14:31:38.851 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=sqlstore t=2024-09-13T14:31:38.742852431Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T14:31:38.851 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=migrator t=2024-09-13T14:31:38.757680863Z level=info msg="Starting DB migrations" 2024-09-13T14:31:38.852 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=migrator t=2024-09-13T14:31:38.76241842Z level=info msg="migrations completed" performed=0 skipped=481 duration=596.994µs 2024-09-13T14:31:38.852 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=sqlstore t=2024-09-13T14:31:38.762828807Z level=info msg="Created default organization" 2024-09-13T14:31:38.852 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=plugin.loader t=2024-09-13T14:31:38.821567054Z level=info msg="Plugin registered" pluginID=input 2024-09-13T14:31:38.852 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=plugin.loader t=2024-09-13T14:31:38.849380742Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-13T14:31:38.852 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=plugin.loader t=2024-09-13T14:31:38.849419329Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-13T14:31:38.852 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=secrets t=2024-09-13T14:31:38.849515695Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T14:31:39.022 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:39.218 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=query_data t=2024-09-13T14:31:38.85139726Z level=info msg="Query Service initialization" 2024-09-13T14:31:39.218 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=live.push_http t=2024-09-13T14:31:38.853812618Z level=info msg="Live Push Gateway initialization" 2024-09-13T14:31:39.604 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:39 smithi204 systemd[1]: Stopping Ceph node-exporter.b for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:31:39.857 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:39 smithi204 podman[74496]: 2024-09-13 14:31:39.603350709 +0000 UTC m=+0.131411410 container died 0fc850b0a1c6c26327d1e7050af9dbf25e0cec7faf3507ac7331b24cc6e22cda (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:31:39.858 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:39.858 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:39.858 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3053185001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:39.858 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-mon[32546]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T14:31:39.858 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-mon[32546]: Deploying daemon node-exporter.b on smithi204 2024-09-13T14:31:39.858 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-mon[32546]: from='client.16308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:39.858 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-mon[32546]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:39.858 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-mon[32546]: from='client.26225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:39.858 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=infra.usagestats.collector t=2024-09-13T14:31:39.66625569Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T14:31:39.859 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=provisioning.datasources t=2024-09-13T14:31:39.752136097Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-13T14:31:39.859 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=provisioning.datasources t=2024-09-13T14:31:39.752167268Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-13T14:31:39.859 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=provisioning.datasources t=2024-09-13T14:31:39.762539196Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T14:31:39.859 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=provisioning.alerting t=2024-09-13T14:31:39.846396844Z level=info msg="starting to provision alerting" 2024-09-13T14:31:39.859 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=provisioning.alerting t=2024-09-13T14:31:39.846425563Z level=info msg="finished to provision alerting" 2024-09-13T14:31:39.859 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=ngalert.state.manager t=2024-09-13T14:31:39.846552219Z level=info msg="Warming state cache for startup" 2024-09-13T14:31:39.859 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=grafanaStorageLogger t=2024-09-13T14:31:39.846593999Z level=info msg="storage starting" 2024-09-13T14:31:39.859 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=http.server t=2024-09-13T14:31:39.848412357Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T14:31:39.859 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=ngalert.state.manager t=2024-09-13T14:31:39.857104523Z level=info msg="State cache has been initialized" states=0 duration=10.551497ms 2024-09-13T14:31:39.859 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=ngalert.multiorg.alertmanager t=2024-09-13T14:31:39.85714043Z level=info msg="starting MultiOrg Alertmanager" 2024-09-13T14:31:39.859 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:31:39 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=ticker t=2024-09-13T14:31:39.85716879Z level=info msg=starting first_tick=2024-09-13T14:31:40Z 2024-09-13T14:31:40.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:40.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:40.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3053185001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:40.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[29229]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T14:31:40.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[29229]: Deploying daemon node-exporter.b on smithi204 2024-09-13T14:31:40.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[29229]: from='client.16308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:40.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[29229]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:40.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[29229]: from='client.26225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:40.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:40.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:40.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3053185001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:40.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[39645]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T14:31:40.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[39645]: Deploying daemon node-exporter.b on smithi204 2024-09-13T14:31:40.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[39645]: from='client.16308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:40.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[39645]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:40.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:39 smithi096 ceph-mon[39645]: from='client.26225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:40.218 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:40 smithi204 podman[74496]: 2024-09-13 14:31:40.077081977 +0000 UTC m=+0.605142667 container cleanup 0fc850b0a1c6c26327d1e7050af9dbf25e0cec7faf3507ac7331b24cc6e22cda (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:31:40.218 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:40 smithi204 bash[74496]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T14:31:40.716 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:40 smithi204 podman[74509]: 2024-09-13 14:31:40.353353644 +0000 UTC m=+0.753720795 container remove 0fc850b0a1c6c26327d1e7050af9dbf25e0cec7faf3507ac7331b24cc6e22cda (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:31:40.716 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:40 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:31:40.716 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:40 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T14:31:40.716 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:40 smithi204 systemd[1]: Stopped Ceph node-exporter.b for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:31:40.717 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:40 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 2.050s CPU time. 2024-09-13T14:31:40.717 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:40 smithi204 systemd[1]: Starting Ceph node-exporter.b for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:31:40.968 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:40 smithi204 bash[74638]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-13T14:31:41.718 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:41 smithi204 bash[74638]: Getting image source signatures 2024-09-13T14:31:41.718 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:41 smithi204 bash[74638]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-13T14:31:41.719 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:41 smithi204 bash[74638]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-13T14:31:41.719 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:41 smithi204 bash[74638]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-13T14:31:41.921 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:42.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:42 smithi096 ceph-mon[29229]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:42.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:42 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2525906348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:42.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:42 smithi096 ceph-mon[39645]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:42.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:42 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2525906348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:42.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:42 smithi204 ceph-mon[32546]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:42.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:42 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2525906348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:42.968 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:42 smithi204 bash[74638]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-13T14:31:42.969 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:42 smithi204 bash[74638]: Writing manifest to image destination 2024-09-13T14:31:43.458 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-mon[32546]: from='client.26237 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:43.458 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-mon[32546]: from='client.26243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:43.459 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:31:43] "GET /metrics HTTP/1.1" 200 33856 "" "Prometheus/2.51.0" 2024-09-13T14:31:43.459 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 podman[74638]: 2024-09-13 14:31:43.149104198 +0000 UTC m=+2.340088212 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-13T14:31:43.459 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 podman[74638]: 2024-09-13 14:31:43.247653264 +0000 UTC m=+2.438637277 container create c58ad3b83def6d730f4ee7c03b308cc87251da76923f9d53da5ebedc4725659e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:31:43.459 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 podman[74638]: 2024-09-13 14:31:43.45834887 +0000 UTC m=+2.649332897 container init c58ad3b83def6d730f4ee7c03b308cc87251da76923f9d53da5ebedc4725659e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:31:43.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:43 smithi096 ceph-mon[29229]: from='client.26237 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:43.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:43 smithi096 ceph-mon[29229]: from='client.26243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:43.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:43 smithi096 ceph-mon[39645]: from='client.26237 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:43.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:43 smithi096 ceph-mon[39645]: from='client.26243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:43.718 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 podman[74638]: 2024-09-13 14:31:43.461278505 +0000 UTC m=+2.652262526 container start c58ad3b83def6d730f4ee7c03b308cc87251da76923f9d53da5ebedc4725659e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:31:43.719 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.464Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-13T14:31:43.719 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.464Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-13T14:31:43.719 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:31:43.719 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:31:43.719 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:31:43.719 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:31:43.719 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:31:43.719 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:31:43.719 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:31:43.719 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:31:43.719 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:31:43.719 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:31:43.720 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:31:43.720 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:31:43.720 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:31:43.720 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:31:43.720 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:31:43.720 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:31:43.721 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:31:43.721 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:31:43.724 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:31:43.724 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:31:43.724 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:31:43.724 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:31:43.724 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:31:43.724 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:31:43.725 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:31:43.725 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:31:43.725 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:31:43.725 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:31:43.725 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:31:43.725 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:31:43.725 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:31:43.725 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:31:43.725 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:31:43.725 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:31:43.725 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:31:43.725 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.465Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:31:43.725 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.466Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:31:43.726 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.466Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:31:43.726 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.466Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:31:43.726 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.466Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:31:43.726 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.466Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:31:43.726 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.466Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:31:43.726 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.466Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:31:43.726 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.466Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:31:43.726 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.466Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:31:43.726 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.466Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:31:43.726 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.466Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:31:43.726 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.466Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:31:43.726 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.466Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:31:43.726 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.466Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:31:43.727 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b[74713]: ts=2024-09-13T14:31:43.466Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:31:43.727 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 bash[74638]: c58ad3b83def6d730f4ee7c03b308cc87251da76923f9d53da5ebedc4725659e 2024-09-13T14:31:43.727 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 14:31:43 smithi204 systemd[1]: Started Ceph node-exporter.b for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:31:44.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:44 smithi204 ceph-mon[32546]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:44.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:44 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:44.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:44 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:44.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:44 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:31:44.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:44 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:31:44.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:44 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:31:44.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:44 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:44.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:44.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:44 smithi096 ceph-mon[29229]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:44.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:44 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:44.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:44 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:44.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:44 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:31:44.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:44 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:31:44.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:44 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:31:44.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:44 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:44.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:44 smithi096 ceph-mon[39645]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:44.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:44 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:44.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:44 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:44.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:44 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:31:44.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:44 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:31:44.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:44 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:31:44.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:44 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:44.816 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:45.393 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[29229]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:31:45.394 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[29229]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:31:45.394 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[29229]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:31:45.394 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[29229]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:31:45.394 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[29229]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:31:45.394 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/73047133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:45.394 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:45.394 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:31:45.394 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:31:45.394 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:31:45.394 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:45.395 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[39645]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:31:45.395 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[39645]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:31:45.395 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[39645]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:31:45.395 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[39645]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:31:45.395 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[39645]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:31:45.395 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/73047133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:45.395 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:45.395 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:31:45.395 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:31:45.395 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:31:45.395 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:45 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:45.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:45 smithi204 ceph-mon[32546]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:31:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:45 smithi204 ceph-mon[32546]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:31:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:45 smithi204 ceph-mon[32546]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:31:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:45 smithi204 ceph-mon[32546]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:31:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:45 smithi204 ceph-mon[32546]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:31:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:45 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/73047133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:45 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:45 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:31:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:45 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:31:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:45 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:31:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:45 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:46.140 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:45 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:46.140 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:45 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:46 smithi096 ceph-mon[29229]: from='client.26255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:46 smithi096 ceph-mon[29229]: Upgrade: Updating mgr.y 2024-09-13T14:31:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:46 smithi096 ceph-mon[29229]: Deploying daemon mgr.y on smithi096 2024-09-13T14:31:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:46 smithi096 ceph-mon[29229]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:46 smithi096 ceph-mon[29229]: from='client.16350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:46 smithi096 ceph-mon[39645]: from='client.26255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:46 smithi096 ceph-mon[39645]: Upgrade: Updating mgr.y 2024-09-13T14:31:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:46 smithi096 ceph-mon[39645]: Deploying daemon mgr.y on smithi096 2024-09-13T14:31:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:46 smithi096 ceph-mon[39645]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:46 smithi096 ceph-mon[39645]: from='client.16350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:46.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:46 smithi204 ceph-mon[32546]: from='client.26255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:46.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:46 smithi204 ceph-mon[32546]: Upgrade: Updating mgr.y 2024-09-13T14:31:46.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:46 smithi204 ceph-mon[32546]: Deploying daemon mgr.y on smithi096 2024-09-13T14:31:46.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:46 smithi204 ceph-mon[32546]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:46.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:46 smithi204 ceph-mon[32546]: from='client.16350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:46.698 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:46 smithi096 systemd[1]: Stopping Ceph mgr.y for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:31:47.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:46.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:47.264 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:46 smithi096 podman[89156]: 2024-09-13 14:31:46.976871212 +0000 UTC m=+0.135926437 container died 7acb3784c92cd2665fee91086513160b5ab7ac81cce043e46641f77cab1b2412 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS) 2024-09-13T14:31:47.572 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:47 smithi096 podman[89156]: 2024-09-13 14:31:47.345853072 +0000 UTC m=+0.504908291 container cleanup 7acb3784c92cd2665fee91086513160b5ab7ac81cce043e46641f77cab1b2412 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-13T14:31:47.572 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:47 smithi096 bash[89156]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y 2024-09-13T14:31:47.757 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:48.065 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:47 smithi096 podman[89168]: 2024-09-13 14:31:47.614542842 +0000 UTC m=+0.635193078 container remove 7acb3784c92cd2665fee91086513160b5ab7ac81cce043e46641f77cab1b2412 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-09-13T14:31:48.065 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:47 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:31:48.065 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:47 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-09-13T14:31:48.065 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:47 smithi096 systemd[1]: Stopped Ceph mgr.y for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:31:48.065 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:47 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.y.service: Consumed 39.889s CPU time. 2024-09-13T14:31:48.065 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:48 smithi096 systemd[1]: Starting Ceph mgr.y for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:31:48.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:48 smithi096 ceph-mon[39645]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:48.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:48 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3032885792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:48.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:48 smithi096 ceph-mon[29229]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:48.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:48 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3032885792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:48.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:48 smithi096 podman[89365]: 2024-09-13 14:31:48.175333164 +0000 UTC m=+0.015629877 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:31:48.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:48 smithi096 podman[89365]: 2024-09-13 14:31:48.282484469 +0000 UTC m=+0.122781216 container create a4bc51171f6e7979db2dd1de560dbf97688832d73e8b630979eba72f017e0e47 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-13T14:31:48.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:48 smithi204 ceph-mon[32546]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:48.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:48 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3032885792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:48.822 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:48 smithi096 podman[89365]: 2024-09-13 14:31:48.518696359 +0000 UTC m=+0.358993071 container init a4bc51171f6e7979db2dd1de560dbf97688832d73e8b630979eba72f017e0e47 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-13T14:31:48.823 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:48 smithi096 podman[89365]: 2024-09-13 14:31:48.52137465 +0000 UTC m=+0.361671364 container start a4bc51171f6e7979db2dd1de560dbf97688832d73e8b630979eba72f017e0e47 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-13T14:31:48.823 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:48 smithi096 ceph-mgr[89385]: -- 172.21.15.96:0/577314085 <== mon.0 v2:172.21.15.96:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557c2e7a50e0 con 0x557c2e782800 2024-09-13T14:31:48.823 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:48 smithi096 bash[89365]: a4bc51171f6e7979db2dd1de560dbf97688832d73e8b630979eba72f017e0e47 2024-09-13T14:31:48.823 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:48 smithi096 systemd[1]: Started Ceph mgr.y for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:31:49.441 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:49 smithi096 ceph-mon[29229]: from='client.26273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:49.442 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:49 smithi096 ceph-mon[29229]: from='client.26279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:49.442 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:49 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:49.442 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:49 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:49.442 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:49 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:49.442 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:49 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:49.442 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:49 smithi096 ceph-mon[39645]: from='client.26273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:49.442 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:49 smithi096 ceph-mon[39645]: from='client.26279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:49.442 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:49 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:49.442 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:49 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:49.442 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:49 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:49.443 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:49 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:49.443 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:49 smithi204 ceph-mon[32546]: from='client.26273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:49.443 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:49 smithi204 ceph-mon[32546]: from='client.26279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:49.443 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:49 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:49.443 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:49 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:49.443 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:49 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:49.443 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:49 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:50.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:50 smithi096 ceph-mon[29229]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:50.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:50 smithi096 ceph-mon[39645]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:50.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:50 smithi204 ceph-mon[32546]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:50.734 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:51 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2626421933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:51.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:51 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2626421933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:51 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2626421933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:52.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:52 smithi204 ceph-mon[32546]: from='client.26297 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:52 smithi204 ceph-mon[32546]: from='client.16392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:52 smithi204 ceph-mon[32546]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:52.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:52 smithi096 ceph-mon[29229]: from='client.26297 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:52.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:52 smithi096 ceph-mon[29229]: from='client.16392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:52.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:52 smithi096 ceph-mon[29229]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:52.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:52 smithi096 ceph-mon[39645]: from='client.26297 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:52.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:52 smithi096 ceph-mon[39645]: from='client.16392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:52.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:52 smithi096 ceph-mon[39645]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:53.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:31:53 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:31:53] "GET /metrics HTTP/1.1" 200 33847 "" "Prometheus/2.51.0" 2024-09-13T14:31:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:53 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:53 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:53 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:53 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:53 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:53 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:53 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:53 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:53.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:53 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:53.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:53 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:53.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:53 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:53.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:53 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:53.681 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:54.234 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[29229]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:54.234 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/2904944876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:54.234 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[29229]: Standby manager daemon y restarted 2024-09-13T14:31:54.234 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[29229]: Standby manager daemon y started 2024-09-13T14:31:54.234 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:31:54.234 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:31:54.234 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:31:54.234 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:31:54.235 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[39645]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:54.235 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2904944876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:54.235 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[39645]: Standby manager daemon y restarted 2024-09-13T14:31:54.235 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[39645]: Standby manager daemon y started 2024-09-13T14:31:54.235 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:31:54.235 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:31:54.235 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:31:54.235 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:31:54.235 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:53 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:31:53] ENGINE Bus STARTING 2024-09-13T14:31:54.235 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:53 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: CherryPy Checker: 2024-09-13T14:31:54.236 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:53 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: The Application mounted at '' has an empty config. 2024-09-13T14:31:54.236 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:53 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:31:54.236 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:31:54] ENGINE Serving on http://:::9283 2024-09-13T14:31:54.236 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:54 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:31:54] ENGINE Bus STARTED 2024-09-13T14:31:54.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:54 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:54 smithi204 ceph-mon[32546]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:54 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2904944876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:54 smithi204 ceph-mon[32546]: Standby manager daemon y restarted 2024-09-13T14:31:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:54 smithi204 ceph-mon[32546]: Standby manager daemon y started 2024-09-13T14:31:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:54 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:31:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:54 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:31:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:54 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:31:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:54 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:31:55.418 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:55 smithi096 ceph-mon[29229]: from='client.16404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:55.418 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:55 smithi096 ceph-mon[29229]: from='client.26321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:55.418 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:55 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:55.418 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:55 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:55.418 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:55 smithi096 ceph-mon[29229]: mgrmap e26: x(active, since 67s), standbys: y 2024-09-13T14:31:55.419 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:55 smithi096 ceph-mon[39645]: from='client.16404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:55.419 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:55 smithi096 ceph-mon[39645]: from='client.26321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:55.419 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:55 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:55.419 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:55 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:55.419 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:55 smithi096 ceph-mon[39645]: mgrmap e26: x(active, since 67s), standbys: y 2024-09-13T14:31:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:55 smithi204 ceph-mon[32546]: from='client.16404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:55 smithi204 ceph-mon[32546]: from='client.26321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:55.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:55 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:55.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:55 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:55.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:55 smithi204 ceph-mon[32546]: mgrmap e26: x(active, since 67s), standbys: y 2024-09-13T14:31:56.162 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:56.162 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:56.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:56 smithi204 ceph-mon[32546]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:56.469 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:31:56 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: 2024-09-13T14:31:56.314+0000 7f6b15d53640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-13T14:31:56.469 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:31:56 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: ignoring --setuser ceph since I am not root 2024-09-13T14:31:56.469 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:31:56 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: ignoring --setgroup ceph since I am not root 2024-09-13T14:31:56.469 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:31:56 smithi204 ceph-mgr[65452]: -- 172.21.15.204:0/1896842154 <== mon.1 v2:172.21.15.204:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5633062ff0e0 con 0x5633062dd000 2024-09-13T14:31:56.539 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:56 smithi096 ceph-mon[29229]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:56.539 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:56 smithi096 ceph-mon[39645]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:56.539 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:56 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:31:56] ENGINE Bus STOPPING 2024-09-13T14:31:56.825 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:56 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:31:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:31:56.825 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:56 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:31:56] ENGINE Bus STOPPED 2024-09-13T14:31:57.175 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:56 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:31:56.923+0000 7f54e4011640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:57.175 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:56 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:31:56.923+0000 7f54e4011640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:57.175 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:56 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:31:56.923+0000 7f54e4011640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:57.175 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:56 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:31:56.923+0000 7f54e4011640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:57.175 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:56 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:31:56.923+0000 7f54e4011640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:57.175 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:56 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:31:56.927+0000 7f54e7017640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:57.175 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:56 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:31:56.927+0000 7f54e7017640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:57.175 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:56 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:31:56.927+0000 7f54e7017640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:57.176 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:56 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:31:56.928+0000 7f54e7017640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:57.176 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:56 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:31:56.928+0000 7f54e7017640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:57.176 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:57.176 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:57.176 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:57.176 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:31:57.176 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:57.176 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:57.176 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:31:57.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:31:57.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: Failing over to other MGR 2024-09-13T14:31:57.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:31:57.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:31:57.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: Activating manager daemon y 2024-09-13T14:31:57.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:31:57.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.25378 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T14:31:57.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: mgrmap e27: y(active, starting, since 0.0105967s) 2024-09-13T14:31:57.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3963487386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:57.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:31:57.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:31:57.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:31:57.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi204.hbwbot"}]: dispatch 2024-09-13T14:31:57.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi096.dqiqpz"}]: dispatch 2024-09-13T14:31:57.178 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:31:57.178 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:31:57.178 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:31:57.178 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:31:57.178 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:31:57.178 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:31:57.178 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:31:57.178 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:31:57.178 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:31:57.178 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:31:57.178 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:57.179 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:57.179 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:57.179 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:31:57.179 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:57.179 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:57.179 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:31:57.179 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:31:57.179 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: Failing over to other MGR 2024-09-13T14:31:57.179 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:31:57.179 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:31:57.179 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: Activating manager daemon y 2024-09-13T14:31:57.180 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:31:57.180 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.25378 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T14:31:57.180 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: mgrmap e27: y(active, starting, since 0.0105967s) 2024-09-13T14:31:57.180 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3963487386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:57.180 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:31:57.180 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:31:57.180 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:31:57.180 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi204.hbwbot"}]: dispatch 2024-09-13T14:31:57.180 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi096.dqiqpz"}]: dispatch 2024-09-13T14:31:57.180 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:31:57.181 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:31:57.181 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:31:57.181 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:31:57.181 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:31:57.181 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:31:57.182 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:31:57.182 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:31:57.182 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:31:57.182 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:31:57.182 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:31:57.182 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:31:57.182 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: Manager daemon y is now available 2024-09-13T14:31:57.182 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:57.182 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:57.183 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:31:57.183 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:31:57.183 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:31:57.183 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:31:57.183 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:31:57.183 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: Manager daemon y is now available 2024-09-13T14:31:57.183 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:57.183 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:57.183 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:31:57.183 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:31:57.183 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:31:57.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:31:56 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:31:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:31:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' 2024-09-13T14:31:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:31:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:31:57.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: Failing over to other MGR 2024-09-13T14:31:57.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.25378 172.21.15.204:0/4147725533' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:31:57.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:31:57.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: Activating manager daemon y 2024-09-13T14:31:57.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:31:57.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.25378 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T14:31:57.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: mgrmap e27: y(active, starting, since 0.0105967s) 2024-09-13T14:31:57.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3963487386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:57.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:31:57.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:31:57.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:31:57.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi204.hbwbot"}]: dispatch 2024-09-13T14:31:57.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi096.dqiqpz"}]: dispatch 2024-09-13T14:31:57.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:31:57.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:31:57.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:31:57.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:31:57.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:31:57.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:31:57.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:31:57.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:31:57.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:31:57.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:31:57.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:31:57.222 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:31:57.222 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: Manager daemon y is now available 2024-09-13T14:31:57.222 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:57.222 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:57.222 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:31:57.222 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:31:57.222 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:57 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:31:57.336 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:31:57.433 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:31:57] ENGINE Bus STARTING 2024-09-13T14:31:57.433 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:31:57] ENGINE Serving on http://:::9283 2024-09-13T14:31:57.433 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:31:57] ENGINE Bus STARTED 2024-09-13T14:31:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:31:57.987+0000 7f54c4d58640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:31:57] ENGINE Error in HTTPServer.serve 2024-09-13T14:31:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: Traceback (most recent call last): 2024-09-13T14:31:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:31:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: self._connections.run(self.expiration_interval) 2024-09-13T14:31:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:31:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: self._run(expiration_interval) 2024-09-13T14:31:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:31:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:31:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:31:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:31:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:31:58.074 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: s = self.context.wrap_socket( 2024-09-13T14:31:58.074 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:31:58.074 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: return self.sslsocket_class._create( 2024-09-13T14:31:58.074 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:31:58.074 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: self.do_handshake() 2024-09-13T14:31:58.074 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:31:58.074 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: self._sslobj.do_handshake() 2024-09-13T14:31:58.074 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:31:58.074 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:31:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:31:58.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:58 smithi096 ceph-mon[29229]: mgrmap e28: y(active, since 1.01192s) 2024-09-13T14:31:58.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:58 smithi096 ceph-mon[29229]: from='client.16428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:58.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:58 smithi096 ceph-mon[29229]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:31:58.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:58 smithi096 ceph-mon[29229]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:58.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:58 smithi096 ceph-mon[39645]: mgrmap e28: y(active, since 1.01192s) 2024-09-13T14:31:58.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:58 smithi096 ceph-mon[39645]: from='client.16428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:58.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:58 smithi096 ceph-mon[39645]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:31:58.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:58 smithi096 ceph-mon[39645]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:58.657 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:58 smithi204 ceph-mon[32546]: mgrmap e28: y(active, since 1.01192s) 2024-09-13T14:31:58.657 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:58 smithi204 ceph-mon[32546]: from='client.16428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:58.657 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:58 smithi204 ceph-mon[32546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:31:58.657 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:58 smithi204 ceph-mon[32546]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:59 smithi096 ceph-mon[29229]: [13/Sep/2024:14:31:57] ENGINE Bus STARTING 2024-09-13T14:31:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:59 smithi096 ceph-mon[29229]: [13/Sep/2024:14:31:57] ENGINE Serving on http://172.21.15.96:8765 2024-09-13T14:31:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:59 smithi096 ceph-mon[29229]: [13/Sep/2024:14:31:57] ENGINE Serving on https://172.21.15.96:7150 2024-09-13T14:31:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:59 smithi096 ceph-mon[29229]: [13/Sep/2024:14:31:57] ENGINE Bus STARTED 2024-09-13T14:31:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:59 smithi096 ceph-mon[29229]: [13/Sep/2024:14:31:57] ENGINE Error in HTTPServer.serve 2024-09-13T14:31:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout: Traceback (most recent call last): 2024-09-13T14:31:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:31:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:31:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:31:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._run(expiration_interval) 2024-09-13T14:31:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:31:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:31:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:31:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:31:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:31:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout: s = self.context.wrap_socket( 2024-09-13T14:31:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:31:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout: return self.sslsocket_class._create( 2024-09-13T14:31:59.574 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:31:59.574 INFO:journalctl@ceph.mon.a.smithi096.stdout: self.do_handshake() 2024-09-13T14:31:59.574 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:31:59.574 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._sslobj.do_handshake() 2024-09-13T14:31:59.574 INFO:journalctl@ceph.mon.a.smithi096.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:31:59.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:59 smithi096 ceph-mon[29229]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:59.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:31:59 smithi096 ceph-mon[29229]: mgrmap e29: y(active, since 2s) 2024-09-13T14:31:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:59 smithi096 ceph-mon[39645]: [13/Sep/2024:14:31:57] ENGINE Bus STARTING 2024-09-13T14:31:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:59 smithi096 ceph-mon[39645]: [13/Sep/2024:14:31:57] ENGINE Serving on http://172.21.15.96:8765 2024-09-13T14:31:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:59 smithi096 ceph-mon[39645]: [13/Sep/2024:14:31:57] ENGINE Serving on https://172.21.15.96:7150 2024-09-13T14:31:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:59 smithi096 ceph-mon[39645]: [13/Sep/2024:14:31:57] ENGINE Bus STARTED 2024-09-13T14:31:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:59 smithi096 ceph-mon[39645]: [13/Sep/2024:14:31:57] ENGINE Error in HTTPServer.serve 2024-09-13T14:31:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout: Traceback (most recent call last): 2024-09-13T14:31:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:31:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:31:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:31:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout: self._run(expiration_interval) 2024-09-13T14:31:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:31:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:31:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:31:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:31:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:31:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout: s = self.context.wrap_socket( 2024-09-13T14:31:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:31:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout: return self.sslsocket_class._create( 2024-09-13T14:31:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:31:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout: self.do_handshake() 2024-09-13T14:31:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:31:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout: self._sslobj.do_handshake() 2024-09-13T14:31:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:31:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:59 smithi096 ceph-mon[39645]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:31:59 smithi096 ceph-mon[39645]: mgrmap e29: y(active, since 2s) 2024-09-13T14:31:59.695 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:59 smithi204 ceph-mon[32546]: [13/Sep/2024:14:31:57] ENGINE Bus STARTING 2024-09-13T14:31:59.695 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:59 smithi204 ceph-mon[32546]: [13/Sep/2024:14:31:57] ENGINE Serving on http://172.21.15.96:8765 2024-09-13T14:31:59.695 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:59 smithi204 ceph-mon[32546]: [13/Sep/2024:14:31:57] ENGINE Serving on https://172.21.15.96:7150 2024-09-13T14:31:59.695 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:59 smithi204 ceph-mon[32546]: [13/Sep/2024:14:31:57] ENGINE Bus STARTED 2024-09-13T14:31:59.695 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:59 smithi204 ceph-mon[32546]: [13/Sep/2024:14:31:57] ENGINE Error in HTTPServer.serve 2024-09-13T14:31:59.695 INFO:journalctl@ceph.mon.b.smithi204.stdout: Traceback (most recent call last): 2024-09-13T14:31:59.695 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:31:59.695 INFO:journalctl@ceph.mon.b.smithi204.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:31:59.696 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:31:59.696 INFO:journalctl@ceph.mon.b.smithi204.stdout: self._run(expiration_interval) 2024-09-13T14:31:59.696 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:31:59.696 INFO:journalctl@ceph.mon.b.smithi204.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:31:59.696 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:31:59.696 INFO:journalctl@ceph.mon.b.smithi204.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:31:59.696 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:31:59.696 INFO:journalctl@ceph.mon.b.smithi204.stdout: s = self.context.wrap_socket( 2024-09-13T14:31:59.696 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:31:59.696 INFO:journalctl@ceph.mon.b.smithi204.stdout: return self.sslsocket_class._create( 2024-09-13T14:31:59.696 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:31:59.697 INFO:journalctl@ceph.mon.b.smithi204.stdout: self.do_handshake() 2024-09-13T14:31:59.697 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:31:59.697 INFO:journalctl@ceph.mon.b.smithi204.stdout: self._sslobj.do_handshake() 2024-09-13T14:31:59.697 INFO:journalctl@ceph.mon.b.smithi204.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:31:59.697 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:59 smithi204 ceph-mon[32546]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:31:59.697 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:31:59 smithi204 ceph-mon[32546]: mgrmap e29: y(active, since 2s) 2024-09-13T14:32:00.265 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:32:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:00 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/172840626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:00.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:32:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:00.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:00 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/172840626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:00.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:00 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/172840626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:01.579 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:01 smithi204 ceph-mon[32546]: from='client.16479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:01.579 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:01 smithi204 ceph-mon[32546]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:01.580 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:01 smithi204 ceph-mon[32546]: mgrmap e30: y(active, since 4s) 2024-09-13T14:32:01.580 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:01 smithi204 ceph-mon[32546]: from='client.26393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:01.580 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:01 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:01.580 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:01 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:01.580 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:01 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:01.580 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:01 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:01.600 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[29229]: from='client.16479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:01.600 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[29229]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:01.600 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[29229]: mgrmap e30: y(active, since 4s) 2024-09-13T14:32:01.600 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[29229]: from='client.26393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:01.600 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:01.600 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:01.600 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:01.600 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:01.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[39645]: from='client.16479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:01.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[39645]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:01.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[39645]: mgrmap e30: y(active, since 4s) 2024-09-13T14:32:01.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[39645]: from='client.26393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:01.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:01.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:01.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:01.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:01 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:01.968 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:01 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: [13/Sep/2024:14:32:01] ENGINE Bus STARTING 2024-09-13T14:32:01.969 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:01 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: CherryPy Checker: 2024-09-13T14:32:01.969 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:01 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: The Application mounted at '' has an empty config. 2024-09-13T14:32:01.969 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:01 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: 2024-09-13T14:32:01.969 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:01 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: [13/Sep/2024:14:32:01] ENGINE Serving on http://:::9283 2024-09-13T14:32:01.969 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:01 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: [13/Sep/2024:14:32:01] ENGINE Bus STARTED 2024-09-13T14:32:02.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:02.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:02.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[39645]: Standby manager daemon x started 2024-09-13T14:32:02.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/4006895423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:32:02.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/4006895423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:32:02.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/4006895423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:32:02.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/4006895423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:32:02.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:02.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:02.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:02.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:02.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[29229]: Standby manager daemon x started 2024-09-13T14:32:02.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/4006895423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:32:02.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/4006895423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:32:02.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/4006895423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:32:02.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/4006895423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:32:02.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:02.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:02 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:02.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:02 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:02.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:02 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:02 smithi204 ceph-mon[32546]: Standby manager daemon x started 2024-09-13T14:32:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:02 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/4006895423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:32:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:02 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/4006895423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:32:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:02 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/4006895423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:32:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:02 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/4006895423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:32:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:02 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:02 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:03.180 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:32:03.718 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:03 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[65448]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:32:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.51.0" 2024-09-13T14:32:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:03 smithi204 ceph-mon[32546]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:03 smithi204 ceph-mon[32546]: mgrmap e31: y(active, since 6s), standbys: x 2024-09-13T14:32:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:03 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:32:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:03 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3258813025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:03.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:03 smithi096 ceph-mon[29229]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:03.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:03 smithi096 ceph-mon[29229]: mgrmap e31: y(active, since 6s), standbys: x 2024-09-13T14:32:03.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:03 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:32:03.813 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:03 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3258813025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:03.813 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:03 smithi096 ceph-mon[39645]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:03.813 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:03 smithi096 ceph-mon[39645]: mgrmap e31: y(active, since 6s), standbys: x 2024-09-13T14:32:03.813 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:03 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:32:03.813 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:03 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3258813025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:04.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:04 smithi204 ceph-mon[32546]: from='client.26405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:04.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:04 smithi204 ceph-mon[32546]: from='client.16503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:04.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:04 smithi096 ceph-mon[29229]: from='client.26405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:04.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:04 smithi096 ceph-mon[29229]: from='client.16503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:04.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:04 smithi096 ceph-mon[39645]: from='client.26405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:04.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:04 smithi096 ceph-mon[39645]: from='client.16503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:05.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[29229]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T14:32:05.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:05.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:32:05.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[29229]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:32:05.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[29229]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:32:05.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[39645]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T14:32:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:32:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[39645]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:32:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[39645]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:32:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:05 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:05 smithi204 ceph-mon[32546]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T14:32:05.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:05 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:05 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:05 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:05 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:32:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:05 smithi204 ceph-mon[32546]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:32:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:05 smithi204 ceph-mon[32546]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:32:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:05 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:05 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:05 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:05 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:05 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:06.116 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:32:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:06 smithi204 ceph-mon[32546]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:06 smithi204 ceph-mon[32546]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:06 smithi204 ceph-mon[32546]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:06 smithi204 ceph-mon[32546]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:06 smithi204 ceph-mon[32546]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:06 smithi204 ceph-mon[32546]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:06 smithi204 ceph-mon[32546]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:32:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:06 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/4276775754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:06.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[39645]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:06.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[39645]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:06.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[39645]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:06.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[39645]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:06.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[39645]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:06.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[39645]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:06.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[39645]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:32:06.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/4276775754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:06.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[29229]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:06.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[29229]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:06.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[29229]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:06.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[29229]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:06.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[29229]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:06.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[29229]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:06.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[29229]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:32:06.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:06 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/4276775754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:07.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:32:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:07.812 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:07 smithi204 ceph-mon[32546]: Reconfiguring daemon prometheus.a on smithi204 2024-09-13T14:32:07.812 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:07 smithi204 ceph-mon[32546]: from='client.16515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:07.812 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:07 smithi204 ceph-mon[32546]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:32:07.812 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:07 smithi204 ceph-mon[32546]: from='client.26429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:07.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:07 smithi096 ceph-mon[39645]: Reconfiguring daemon prometheus.a on smithi204 2024-09-13T14:32:07.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:07 smithi096 ceph-mon[39645]: from='client.16515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:07.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:07 smithi096 ceph-mon[39645]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:32:07.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:07 smithi096 ceph-mon[39645]: from='client.26429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:07.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:07 smithi096 ceph-mon[29229]: Reconfiguring daemon prometheus.a on smithi204 2024-09-13T14:32:07.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:07 smithi096 ceph-mon[29229]: from='client.16515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:07.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:07 smithi096 ceph-mon[29229]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:32:07.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:07 smithi096 ceph-mon[29229]: from='client.26429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:08.691 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:08 smithi204 ceph-mon[32546]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T14:32:08.968 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:08 smithi204 systemd[1]: Stopping Ceph prometheus.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:32:09.014 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:32:09.014 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:08 smithi096 ceph-mon[29229]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T14:32:09.014 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:08 smithi096 ceph-mon[39645]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T14:32:09.370 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:32:09.069Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:32:09.371 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:32:09.069Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:32:09.371 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:32:09.069Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:32:09.371 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:32:09.069Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:32:09.371 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:32:09.069Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:32:09.371 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:32:09.069Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-13T14:32:09.371 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:32:09.069Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:32:09.371 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:32:09.069Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-13T14:32:09.371 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:32:09.069Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-13T14:32:09.371 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:32:09.071Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:32:09.371 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:32:09.071Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-13T14:32:09.372 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[73998]: ts=2024-09-13T14:32:09.071Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-13T14:32:09.372 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 podman[77143]: 2024-09-13 14:32:09.07986416 +0000 UTC m=+0.116052629 container died e7fc3eeabfeed520c876f2d46b39048c36525e68736ae2c76d3fea8eb7a55941 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:32:09.679 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:09 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/4168929072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:09.679 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:09 smithi204 ceph-mon[32546]: from='client.26441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:09.679 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:09 smithi204 ceph-mon[32546]: from='client.16539 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:09.679 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 podman[77143]: 2024-09-13 14:32:09.46909252 +0000 UTC m=+0.505280992 container cleanup e7fc3eeabfeed520c876f2d46b39048c36525e68736ae2c76d3fea8eb7a55941 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:32:09.679 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 bash[77143]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:32:09.968 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 podman[77156]: 2024-09-13 14:32:09.678788761 +0000 UTC m=+0.600525766 container remove e7fc3eeabfeed520c876f2d46b39048c36525e68736ae2c76d3fea8eb7a55941 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:32:09.968 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:32:09.968 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 systemd[1]: Stopped Ceph prometheus.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:32:09.969 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.056s CPU time. 2024-09-13T14:32:09.969 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:09 smithi204 systemd[1]: Starting Ceph prometheus.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:32:10.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:09 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/4168929072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:10.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:09 smithi096 ceph-mon[29229]: from='client.26441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:10.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:09 smithi096 ceph-mon[29229]: from='client.16539 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:10.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:09 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/4168929072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:10.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:09 smithi096 ceph-mon[39645]: from='client.26441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:10.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:09 smithi096 ceph-mon[39645]: from='client.16539 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:10.355 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 podman[77247]: 2024-09-13 14:32:10.004133294 +0000 UTC m=+0.016936010 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-13T14:32:10.356 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 podman[77247]: 2024-09-13 14:32:10.104830127 +0000 UTC m=+0.117632873 container create c1d744b6a96b23ffee75907c4e30283f63be4ba6613fc93534f5eb071315d6d7 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:32:10.356 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 podman[77247]: 2024-09-13 14:32:10.31539647 +0000 UTC m=+0.328199170 container init c1d744b6a96b23ffee75907c4e30283f63be4ba6613fc93534f5eb071315d6d7 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:32:10.356 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 podman[77247]: 2024-09-13 14:32:10.318170689 +0000 UTC m=+0.330973389 container start c1d744b6a96b23ffee75907c4e30283f63be4ba6613fc93534f5eb071315d6d7 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:32:10.356 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.337Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-13T14:32:10.356 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.337Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-13T14:32:10.356 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.337Z caller=main.go:623 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi204 (none))" 2024-09-13T14:32:10.356 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.337Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:32:10.356 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.337Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:32:10.356 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.339Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:32:10.356 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.340Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-13T14:32:10.356 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.341Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:32:10.357 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.341Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:32:10.357 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.343Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:32:10.357 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.343Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.695µs 2024-09-13T14:32:10.357 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.343Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:32:10.357 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.343Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-13T14:32:10.357 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.354Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-13T14:32:10.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.360Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-13T14:32:10.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.360Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-13T14:32:10.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.360Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=23.807µs wal_replay_duration=17.115015ms wbl_replay_duration=117ns total_replay_duration=17.157051ms 2024-09-13T14:32:10.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.365Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:32:10.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.365Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-13T14:32:10.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.365Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:32:10.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.379Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.980056ms db_storage=938ns remote_storage=1.092µs web_handler=588ns query_engine=929ns scrape=439.577µs scrape_sd=89.359µs notify=9.793µs notify_sd=7.953µs rules=12.795896ms tracing=4.815µs 2024-09-13T14:32:10.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.379Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-13T14:32:10.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:10.379Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:32:10.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 bash[77247]: c1d744b6a96b23ffee75907c4e30283f63be4ba6613fc93534f5eb071315d6d7 2024-09-13T14:32:10.720 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:10 smithi204 systemd[1]: Started Ceph prometheus.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:32:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:11 smithi204 ceph-mon[32546]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:11 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:11 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:11 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:32:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:11 smithi204 ceph-mon[32546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:32:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:11 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:32:11.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:11 smithi096 ceph-mon[29229]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:11.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:11 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:11.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:11 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:11.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:11 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:32:11.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:11 smithi096 ceph-mon[29229]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:32:11.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:11 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:32:11.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:11 smithi096 ceph-mon[39645]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:11.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:11 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:11.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:11 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:11.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:11 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:32:11.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:11 smithi096 ceph-mon[39645]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:32:11.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:11 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:32:11.919 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:32:12.560 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:12 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/4214251454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:12.560 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:12 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:32:12.561 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:12 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:32:12.561 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:12 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:32:12.561 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:12 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:12.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:12 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/4214251454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:12.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:12 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:32:12.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:12 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:32:12.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:12 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:32:12.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:12 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:12.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:12 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/4214251454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:12.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:12 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:32:12.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:12 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:32:12.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:12 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:32:12.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:12 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:13.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:13 smithi204 ceph-mon[32546]: from='client.26459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:13.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:13 smithi204 ceph-mon[32546]: Upgrade: Updating mgr.x 2024-09-13T14:32:13.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:13 smithi204 ceph-mon[32546]: Deploying daemon mgr.x on smithi204 2024-09-13T14:32:13.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:13 smithi204 ceph-mon[32546]: from='client.26465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:13.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:13 smithi204 ceph-mon[32546]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:13.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:13 smithi096 ceph-mon[29229]: from='client.26459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:13.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:13 smithi096 ceph-mon[29229]: Upgrade: Updating mgr.x 2024-09-13T14:32:13.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:13 smithi096 ceph-mon[29229]: Deploying daemon mgr.x on smithi204 2024-09-13T14:32:13.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:13 smithi096 ceph-mon[29229]: from='client.26465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:13.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:13 smithi096 ceph-mon[29229]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:13.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:13 smithi096 ceph-mon[39645]: from='client.26459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:13.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:13 smithi096 ceph-mon[39645]: Upgrade: Updating mgr.x 2024-09-13T14:32:13.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:13 smithi096 ceph-mon[39645]: Deploying daemon mgr.x on smithi204 2024-09-13T14:32:13.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:13 smithi096 ceph-mon[39645]: from='client.26465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:13.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:13 smithi096 ceph-mon[39645]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:13.903 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:13 smithi204 systemd[1]: Stopping Ceph mgr.x for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:32:14.172 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:13 smithi204 podman[77596]: 2024-09-13 14:32:13.902224085 +0000 UTC m=+0.144280368 container died 2a60f8e84e6f375ec5cf16394db9e90ed8166ec0bf38aeddd4266f1bfece6dd8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-13T14:32:14.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:14 smithi204 podman[77596]: 2024-09-13 14:32:14.262427672 +0000 UTC m=+0.504483996 container cleanup 2a60f8e84e6f375ec5cf16394db9e90ed8166ec0bf38aeddd4266f1bfece6dd8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-09-13T14:32:14.468 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:14 smithi204 bash[77596]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x 2024-09-13T14:32:14.824 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:32:14.840 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:14 smithi204 podman[77608]: 2024-09-13 14:32:14.505088014 +0000 UTC m=+0.604394268 container remove 2a60f8e84e6f375ec5cf16394db9e90ed8166ec0bf38aeddd4266f1bfece6dd8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:32:14.840 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:14 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:32:14.840 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:14 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T14:32:14.840 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:14 smithi204 systemd[1]: Stopped Ceph mgr.x for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:32:14.840 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:14 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.x.service: Consumed 18.478s CPU time. 2024-09-13T14:32:15.185 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:14 smithi204 systemd[1]: Starting Ceph mgr.x for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:32:15.185 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:15 smithi204 podman[77735]: 2024-09-13 14:32:14.951048902 +0000 UTC m=+0.016325383 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:32:15.185 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:15 smithi204 podman[77735]: 2024-09-13 14:32:15.073695719 +0000 UTC m=+0.138972201 container create 04572bea1e6fa024cb0842bda1f6b4e197695cca853dc44d9db2769b0895e14a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T14:32:15.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:15 smithi204 ceph-mon[32546]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:15.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:15 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/610596611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:15.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:15 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:15.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:15 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:15.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:15 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:32:15.719 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:15 smithi204 podman[77735]: 2024-09-13 14:32:15.276478963 +0000 UTC m=+0.341755445 container init 04572bea1e6fa024cb0842bda1f6b4e197695cca853dc44d9db2769b0895e14a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-13T14:32:15.719 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:15 smithi204 podman[77735]: 2024-09-13 14:32:15.27912713 +0000 UTC m=+0.344403620 container start 04572bea1e6fa024cb0842bda1f6b4e197695cca853dc44d9db2769b0895e14a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:32:15.719 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:15 smithi204 ceph-mgr[77754]: -- 172.21.15.204:0/3470803092 <== mon.2 v2:172.21.15.96:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561b46a4d0e0 con 0x561b46a2a800 2024-09-13T14:32:15.719 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:15 smithi204 bash[77735]: 04572bea1e6fa024cb0842bda1f6b4e197695cca853dc44d9db2769b0895e14a 2024-09-13T14:32:15.719 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:15 smithi204 systemd[1]: Started Ceph mgr.x for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:32:15.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:15 smithi096 ceph-mon[39645]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:15.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:15 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/610596611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:15.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:15 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:15.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:15 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:15.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:15 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:32:15.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:15 smithi096 ceph-mon[29229]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:15.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:15 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/610596611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:15.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:15 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:15.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:15 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:15.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:15 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:32:16.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:16 smithi096 ceph-mon[39645]: from='client.16569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:16.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:16 smithi096 ceph-mon[39645]: from='client.16575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:16.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:16 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:16.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:16 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:16 smithi096 ceph-mon[29229]: from='client.16569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:16 smithi096 ceph-mon[29229]: from='client.16575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:16 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:16 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:16.829 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:16 smithi204 ceph-mon[32546]: from='client.16569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:16.829 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:16 smithi204 ceph-mon[32546]: from='client.16575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:16.829 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:16 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:16.829 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:16 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:17.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:16 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:17.723 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:32:17.768 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:17 smithi204 ceph-mon[32546]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:17.768 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:17 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/3196152483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:17.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:17 smithi096 ceph-mon[39645]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:17.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:17 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/3196152483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:17.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:17 smithi096 ceph-mon[29229]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:17.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:17 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/3196152483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:18.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:32:17] "GET /metrics HTTP/1.1" 200 33858 "" "Prometheus/2.51.0" 2024-09-13T14:32:19.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:19 smithi204 ceph-mon[32546]: from='client.16590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:19.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:19 smithi204 ceph-mon[32546]: from='client.26504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:19.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:19 smithi204 ceph-mon[32546]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:19.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:19 smithi096 ceph-mon[29229]: from='client.16590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:19.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:19 smithi096 ceph-mon[29229]: from='client.26504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:19.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:19 smithi096 ceph-mon[29229]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:19.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:19 smithi096 ceph-mon[39645]: from='client.16590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:19.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:19 smithi096 ceph-mon[39645]: from='client.26504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:19.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:19 smithi096 ceph-mon[39645]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:20.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:20.622 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:32:20.727 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[77750]: [13/Sep/2024:14:32:20] ENGINE Bus STARTING 2024-09-13T14:32:20.727 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[77750]: CherryPy Checker: 2024-09-13T14:32:20.728 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[77750]: The Application mounted at '' has an empty config. 2024-09-13T14:32:20.728 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[77750]: 2024-09-13T14:32:20.728 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[77750]: [13/Sep/2024:14:32:20] ENGINE Serving on http://:::9283 2024-09-13T14:32:20.728 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[77750]: [13/Sep/2024:14:32:20] ENGINE Bus STARTED 2024-09-13T14:32:20.728 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:20.728 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:20.728 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:20.728 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:20.728 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-mon[32546]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:20.728 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1992189628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:20.729 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-mon[32546]: Standby manager daemon x restarted 2024-09-13T14:32:20.729 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-mon[32546]: Standby manager daemon x started 2024-09-13T14:32:20.729 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/3304791982' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:32:20.729 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/3304791982' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:32:20.729 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/3304791982' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:32:20.729 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:20 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/3304791982' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:32:21.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:21.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:21.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:21.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[29229]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/1992189628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[29229]: Standby manager daemon x restarted 2024-09-13T14:32:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[29229]: Standby manager daemon x started 2024-09-13T14:32:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/3304791982' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:32:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/3304791982' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:32:21.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/3304791982' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:32:21.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[29229]: from='mgr.? 172.21.15.204:0/3304791982' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:32:21.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:21.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:21.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:21.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:21.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[39645]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:21.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1992189628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:21.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[39645]: Standby manager daemon x restarted 2024-09-13T14:32:21.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[39645]: Standby manager daemon x started 2024-09-13T14:32:21.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/3304791982' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:32:21.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/3304791982' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:32:21.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/3304791982' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:32:21.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:20 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/3304791982' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:32:22.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:22 smithi204 ceph-mon[32546]: from='client.26516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:22.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:22 smithi204 ceph-mon[32546]: from='client.16614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:22.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:22 smithi204 ceph-mon[32546]: mgrmap e32: y(active, since 24s), standbys: x 2024-09-13T14:32:22.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:22 smithi096 ceph-mon[29229]: from='client.26516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:22.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:22 smithi096 ceph-mon[29229]: from='client.16614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:22.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:22 smithi096 ceph-mon[29229]: mgrmap e32: y(active, since 24s), standbys: x 2024-09-13T14:32:22.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:22 smithi096 ceph-mon[39645]: from='client.26516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:22.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:22 smithi096 ceph-mon[39645]: from='client.16614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:22.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:22 smithi096 ceph-mon[39645]: mgrmap e32: y(active, since 24s), standbys: x 2024-09-13T14:32:23.389 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:23 smithi204 ceph-mon[32546]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:23.389 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:23 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/320923583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:23.534 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:32:23.534 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:23 smithi096 ceph-mon[29229]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:23.534 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:23 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/320923583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:23.534 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:23 smithi096 ceph-mon[39645]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:23.534 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:23 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/320923583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:24.425 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:24.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:24.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:24.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:24.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:24.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:32:24.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:24.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: from='client.26534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:24.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:32:24.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:32:24.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: Upgrade: Setting container_image for all mgr 2024-09-13T14:32:24.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:24.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:32:24.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T14:32:24.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:32:24.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T14:32:24.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:32:24.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T14:32:24.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:24.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:24.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:24.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:32:24.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:24.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: from='client.26534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:24.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:32:24.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:32:24.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: Upgrade: Setting container_image for all mgr 2024-09-13T14:32:24.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:24.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:32:24.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T14:32:24.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:32:24.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T14:32:24.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:32:24.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:24 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T14:32:24.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:24.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:24.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:24.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:32:24.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:24.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: from='client.26534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:24.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:32:24.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:32:24.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: Upgrade: Setting container_image for all mgr 2024-09-13T14:32:24.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:24.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:32:24.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T14:32:24.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:32:24.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T14:32:24.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:32:24.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:24 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T14:32:25.677 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:25 smithi096 ceph-mon[29229]: Upgrade: It appears safe to stop mon.a 2024-09-13T14:32:25.678 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:25 smithi096 ceph-mon[29229]: from='client.26540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:25.678 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:25 smithi096 ceph-mon[29229]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:25.678 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:25 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:25.678 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:25 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:32:25.678 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:25 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:32:25.678 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:25 smithi096 ceph-mon[29229]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:25.678 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:25 smithi096 ceph-mon[39645]: Upgrade: It appears safe to stop mon.a 2024-09-13T14:32:25.678 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:25 smithi096 ceph-mon[39645]: from='client.26540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:25.679 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:25 smithi096 ceph-mon[39645]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:25.679 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:25 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:25.679 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:25 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:32:25.679 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:25 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:32:25.679 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:25 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:25.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:25 smithi204 ceph-mon[32546]: Upgrade: It appears safe to stop mon.a 2024-09-13T14:32:25.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:25 smithi204 ceph-mon[32546]: from='client.26540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:25.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:25 smithi204 ceph-mon[32546]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:25.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:25 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' 2024-09-13T14:32:25.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:25 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:32:25.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:25 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:32:25.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:25 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:26.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:25 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:26.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:25 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:26.465 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:32:26.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:26 smithi204 ceph-mon[32546]: Upgrade: Updating mon.a 2024-09-13T14:32:26.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:26 smithi204 ceph-mon[32546]: Deploying daemon mon.a on smithi096 2024-09-13T14:32:26.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:26 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/4156440332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:26.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:26 smithi096 ceph-mon[29229]: Upgrade: Updating mon.a 2024-09-13T14:32:26.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:26 smithi096 ceph-mon[29229]: Deploying daemon mon.a on smithi096 2024-09-13T14:32:26.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:26 smithi096 ceph-mon[29229]: from='client.? 172.21.15.96:0/4156440332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:26.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:26 smithi096 ceph-mon[39645]: Upgrade: Updating mon.a 2024-09-13T14:32:26.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:26 smithi096 ceph-mon[39645]: Deploying daemon mon.a on smithi096 2024-09-13T14:32:26.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:26 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/4156440332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:27.112 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:26 smithi096 systemd[1]: Stopping Ceph mon.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:32:27.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:26 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:27.463 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a[29225]: 2024-09-13T14:32:27.111+0000 7f9e2082e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:32:27.463 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a[29225]: 2024-09-13T14:32:27.111+0000 7f9e2082e640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-13T14:32:27.463 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:27 smithi096 podman[98710]: 2024-09-13 14:32:27.315654995 +0000 UTC m=+0.312938141 container died 96e95ed67f926384baf6702dc5fbb3e4686072ff9aa3c2f5ebce42533fb138d1 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2) 2024-09-13T14:32:27.463 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:32:27] ENGINE Bus STOPPING 2024-09-13T14:32:27.464 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:32:27] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:32:27.770 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:32:27] ENGINE Bus STOPPED 2024-09-13T14:32:27.770 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:32:27] ENGINE Bus STARTING 2024-09-13T14:32:27.770 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:32:27] ENGINE Serving on http://:::9283 2024-09-13T14:32:27.771 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:32:27] ENGINE Bus STARTED 2024-09-13T14:32:28.047 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:27 smithi096 podman[98710]: 2024-09-13 14:32:27.7702151 +0000 UTC m=+0.767498244 container cleanup 96e95ed67f926384baf6702dc5fbb3e4686072ff9aa3c2f5ebce42533fb138d1 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:32:28.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:27 smithi096 bash[98710]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a 2024-09-13T14:32:28.048 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:32:27] "GET /metrics HTTP/1.1" 200 33858 "" "Prometheus/2.51.0" 2024-09-13T14:32:28.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 podman[98722]: 2024-09-13 14:32:28.047092186 +0000 UTC m=+0.729929360 container remove 96e95ed67f926384baf6702dc5fbb3e4686072ff9aa3c2f5ebce42533fb138d1 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:32:28.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T14:32:28.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 systemd[1]: Stopped Ceph mon.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:32:28.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.a.service: Consumed 12.273s CPU time. 2024-09-13T14:32:28.726 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 systemd[1]: Starting Ceph mon.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:32:28.726 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 podman[98861]: 2024-09-13 14:32:28.516928712 +0000 UTC m=+0.015641030 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:32:28.726 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 podman[98861]: 2024-09-13 14:32:28.632105694 +0000 UTC m=+0.130818042 container create 97a94128e048ed7eb609e53a28bf4a29c94b24310260d4b5b69fa0fcdae33191 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:32:29.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 podman[98861]: 2024-09-13 14:32:28.851261746 +0000 UTC m=+0.349974052 container init 97a94128e048ed7eb609e53a28bf4a29c94b24310260d4b5b69fa0fcdae33191 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:32:29.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 podman[98861]: 2024-09-13 14:32:28.854593641 +0000 UTC m=+0.353305945 container start 97a94128e048ed7eb609e53a28bf4a29c94b24310260d4b5b69fa0fcdae33191 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-09-13T14:32:29.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:32:29.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev), process ceph-mon, pid 2 2024-09-13T14:32:29.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: pidfile_write: ignore empty --pid-file 2024-09-13T14:32:29.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: load: jerasure load: lrc 2024-09-13T14:32:29.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: RocksDB version: 7.9.2 2024-09-13T14:32:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Git sha 0 2024-09-13T14:32:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Compile date 2024-09-12 20:46:14 2024-09-13T14:32:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: DB SUMMARY 2024-09-13T14:32:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: DB Session ID: FJTI0IJEQJNDO5YCB3HP 2024-09-13T14:32:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:32:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:32:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1161 Bytes 2024-09-13T14:32:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-09-13T14:32:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 50569 ; 2024-09-13T14:32:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:32:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:32:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:32:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:32:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:32:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:32:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.env: 0x55e51493f160 2024-09-13T14:32:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:32:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.info_log: 0x55e515dff880 2024-09-13T14:32:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:32:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.statistics: (nil) 2024-09-13T14:32:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.use_fsync: 0 2024-09-13T14:32:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:32:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:32:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:32:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:32:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:32:29.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:32:29.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:32:29.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:32:29.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:32:29.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:32:29.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:32:29.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.db_log_dir: 2024-09-13T14:32:29.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.wal_dir: 2024-09-13T14:32:29.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:32:29.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:32:29.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:32:29.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:32:29.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:32:29.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:32:29.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:32:29.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:32:29.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.write_buffer_manager: 0x55e515e02960 2024-09-13T14:32:29.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:32:29.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:32:29.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:32:29.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:32:29.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:32:29.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:32:29.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:32:29.077 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:32:29.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.unordered_write: 0 2024-09-13T14:32:29.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:32:29.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:32:29.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:32:29.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:32:29.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.row_cache: None 2024-09-13T14:32:29.078 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.wal_filter: None 2024-09-13T14:32:29.080 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:32:29.080 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:32:29.080 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:32:29.080 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:32:29.080 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.wal_compression: 0 2024-09-13T14:32:29.080 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:32:29.080 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:32:29.080 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:32:29.080 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:32:29.080 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:32:29.080 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:32:29.080 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:32:29.081 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:32:29.081 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:32:29.081 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:32:29.081 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:32:29.081 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:32:29.081 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:32:29.081 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:32:29.081 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:32:29.081 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:32:29.081 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:32:29.081 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:32:29.081 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:32:29.081 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:32:29.082 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:32:29.082 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:32:29.082 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:32:29.082 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_open_files: -1 2024-09-13T14:32:29.082 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:32:29.082 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:32:29.082 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:32:29.082 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:32:29.082 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:32:29.082 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Compression algorithms supported: 2024-09-13T14:32:29.082 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: kZSTD supported: 0 2024-09-13T14:32:29.082 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:32:29.082 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:32:29.082 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:32:29.083 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:32:29.083 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:32:29.083 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:32:29.083 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:32:29.083 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:32:29.083 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:32:29.083 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-13T14:32:29.083 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:32:29.083 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:32:29.083 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.merge_operator: 2024-09-13T14:32:29.083 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compaction_filter: None 2024-09-13T14:32:29.083 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:32:29.083 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:32:29.084 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:32:29.084 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:32:29.084 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e515dff4e0) 2024-09-13T14:32:29.084 INFO:journalctl@ceph.mon.a.smithi096.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:32:29.084 INFO:journalctl@ceph.mon.a.smithi096.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:32:29.084 INFO:journalctl@ceph.mon.a.smithi096.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:32:29.084 INFO:journalctl@ceph.mon.a.smithi096.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:32:29.084 INFO:journalctl@ceph.mon.a.smithi096.stdout: index_type: 0 2024-09-13T14:32:29.084 INFO:journalctl@ceph.mon.a.smithi096.stdout: data_block_index_type: 0 2024-09-13T14:32:29.084 INFO:journalctl@ceph.mon.a.smithi096.stdout: index_shortening: 1 2024-09-13T14:32:29.084 INFO:journalctl@ceph.mon.a.smithi096.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:32:29.084 INFO:journalctl@ceph.mon.a.smithi096.stdout: checksum: 4 2024-09-13T14:32:29.084 INFO:journalctl@ceph.mon.a.smithi096.stdout: no_block_cache: 0 2024-09-13T14:32:29.084 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_cache: 0x55e515df2850 2024-09-13T14:32:29.085 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:32:29.085 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_cache_options: 2024-09-13T14:32:29.085 INFO:journalctl@ceph.mon.a.smithi096.stdout: capacity : 536870912 2024-09-13T14:32:29.085 INFO:journalctl@ceph.mon.a.smithi096.stdout: num_shard_bits : 4 2024-09-13T14:32:29.085 INFO:journalctl@ceph.mon.a.smithi096.stdout: strict_capacity_limit : 0 2024-09-13T14:32:29.085 INFO:journalctl@ceph.mon.a.smithi096.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:32:29.085 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_cache_compressed: (nil) 2024-09-13T14:32:29.085 INFO:journalctl@ceph.mon.a.smithi096.stdout: persistent_cache: (nil) 2024-09-13T14:32:29.085 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_size: 4096 2024-09-13T14:32:29.085 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_size_deviation: 10 2024-09-13T14:32:29.085 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_restart_interval: 16 2024-09-13T14:32:29.085 INFO:journalctl@ceph.mon.a.smithi096.stdout: index_block_restart_interval: 1 2024-09-13T14:32:29.085 INFO:journalctl@ceph.mon.a.smithi096.stdout: metadata_block_size: 4096 2024-09-13T14:32:29.085 INFO:journalctl@ceph.mon.a.smithi096.stdout: partition_filters: 0 2024-09-13T14:32:29.086 INFO:journalctl@ceph.mon.a.smithi096.stdout: use_delta_encoding: 1 2024-09-13T14:32:29.086 INFO:journalctl@ceph.mon.a.smithi096.stdout: filter_policy: bloomfilter 2024-09-13T14:32:29.086 INFO:journalctl@ceph.mon.a.smithi096.stdout: whole_key_filtering: 1 2024-09-13T14:32:29.086 INFO:journalctl@ceph.mon.a.smithi096.stdout: verify_compression: 0 2024-09-13T14:32:29.086 INFO:journalctl@ceph.mon.a.smithi096.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:32:29.086 INFO:journalctl@ceph.mon.a.smithi096.stdout: format_version: 5 2024-09-13T14:32:29.086 INFO:journalctl@ceph.mon.a.smithi096.stdout: enable_index_compression: 1 2024-09-13T14:32:29.086 INFO:journalctl@ceph.mon.a.smithi096.stdout: block_align: 0 2024-09-13T14:32:29.086 INFO:journalctl@ceph.mon.a.smithi096.stdout: max_auto_readahead_size: 262144 2024-09-13T14:32:29.086 INFO:journalctl@ceph.mon.a.smithi096.stdout: prepopulate_block_cache: 0 2024-09-13T14:32:29.086 INFO:journalctl@ceph.mon.a.smithi096.stdout: initial_auto_readahead_size: 8192 2024-09-13T14:32:29.086 INFO:journalctl@ceph.mon.a.smithi096.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T14:32:29.086 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:32:29.086 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:32:29.087 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compression: NoCompression 2024-09-13T14:32:29.087 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:32:29.087 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:32:29.087 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:32:29.087 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.num_levels: 7 2024-09-13T14:32:29.087 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:32:29.087 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:32:29.087 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:32:29.087 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:32:29.087 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:32:29.087 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:32:29.087 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:32:29.087 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:32:29.087 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:32:29.087 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:32:29.088 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:32:29.088 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:32:29.088 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:32:29.088 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:32:29.088 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:32:29.088 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:32:29.088 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:32:29.088 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:32:29.088 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:32:29.088 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:32:29.088 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:32:29.088 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:32:29.088 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:32:29.089 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:32:29.089 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:32:29.089 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:32:29.089 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:32:29.089 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:32:29.089 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:32:29.089 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:32:29.089 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:32:29.089 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:32:29.089 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:32:29.089 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:32:29.089 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:32:29.090 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:32:29.090 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:32:29.090 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:32:29.090 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:32:29.090 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:32:29.090 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:32:29.090 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:32:29.090 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:32:29.090 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:32:29.090 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:32:29.090 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:32:29.090 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:32:29.091 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:32:29.091 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:32:29.091 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:32:29.091 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:32:29.091 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:32:29.091 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:32:29.091 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:32:29.091 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:32:29.091 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:32:29.091 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:32:29.091 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:32:29.091 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:32:29.092 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:32:29.092 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:32:29.092 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:32:29.092 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:32:29.092 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:32:29.092 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:32:29.092 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.ttl: 2592000 2024-09-13T14:32:29.092 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:32:29.092 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:32:29.092 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:32:29.092 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:32:29.092 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:32:29.093 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:32:29.093 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:32:29.093 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:32:29.093 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:32:29.093 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:32:29.093 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:32:29.093 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:32:29.093 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:32:29.093 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-09-13T14:32:29.093 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:32:29.093 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12308, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:32:29.093 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-09-13T14:32:29.094 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 41eb8f6f-7217-4b59-8d92-ef940ef74c68 2024-09-13T14:32:29.094 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237948881470, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-09-13T14:32:29.094 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-09-13T14:32:29.094 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237948882105, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 51196, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12309, "largest_seqno": 12377, "table_properties": {"data_size": 49809, "index_size": 221, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 133, "raw_key_size": 1318, "raw_average_key_size": 26, "raw_value_size": 48669, "raw_average_value_size": 973, "num_data_blocks": 10, "num_entries": 50, "num_filter_entries": 50, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726237948, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "41eb8f6f-7217-4b59-8d92-ef940ef74c68", "db_session_id": "FJTI0IJEQJNDO5YCB3HP", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:32:29.094 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237948882198, "job": 1, "event": "recovery_finished"} 2024-09-13T14:32:29.094 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-09-13T14:32:29.094 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:32:29.094 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:32:29.094 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e515e22e00 2024-09-13T14:32:29.094 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: DB pointer 0x55e515f22000 2024-09-13T14:32:29.094 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:32:29.094 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:32:29.095 INFO:journalctl@ceph.mon.a.smithi096.stdout: ** DB Stats ** 2024-09-13T14:32:29.095 INFO:journalctl@ceph.mon.a.smithi096.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:32:29.095 INFO:journalctl@ceph.mon.a.smithi096.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:32:29.095 INFO:journalctl@ceph.mon.a.smithi096.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:32:29.095 INFO:journalctl@ceph.mon.a.smithi096.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:32:29.095 INFO:journalctl@ceph.mon.a.smithi096.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:32:29.095 INFO:journalctl@ceph.mon.a.smithi096.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:32:29.095 INFO:journalctl@ceph.mon.a.smithi096.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:32:29.095 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:32:29.095 INFO:journalctl@ceph.mon.a.smithi096.stdout: ** Compaction Stats [default] ** 2024-09-13T14:32:29.095 INFO:journalctl@ceph.mon.a.smithi096.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:32:29.095 INFO:journalctl@ceph.mon.a.smithi096.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:32:29.096 INFO:journalctl@ceph.mon.a.smithi096.stdout: L0 1/0 50.00 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 111.7 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-13T14:32:29.096 INFO:journalctl@ceph.mon.a.smithi096.stdout: L6 1/0 7.41 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T14:32:29.096 INFO:journalctl@ceph.mon.a.smithi096.stdout: Sum 2/0 7.46 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 111.7 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-13T14:32:29.096 INFO:journalctl@ceph.mon.a.smithi096.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 111.7 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-13T14:32:29.096 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:32:29.096 INFO:journalctl@ceph.mon.a.smithi096.stdout: ** Compaction Stats [default] ** 2024-09-13T14:32:29.096 INFO:journalctl@ceph.mon.a.smithi096.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:32:29.096 INFO:journalctl@ceph.mon.a.smithi096.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:32:29.096 INFO:journalctl@ceph.mon.a.smithi096.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 111.7 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-13T14:32:29.096 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:32:29.096 INFO:journalctl@ceph.mon.a.smithi096.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:32:29.096 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:32:29.096 INFO:journalctl@ceph.mon.a.smithi096.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:32:29.097 INFO:journalctl@ceph.mon.a.smithi096.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:32:29.097 INFO:journalctl@ceph.mon.a.smithi096.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:32:29.097 INFO:journalctl@ceph.mon.a.smithi096.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:32:29.097 INFO:journalctl@ceph.mon.a.smithi096.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:32:29.097 INFO:journalctl@ceph.mon.a.smithi096.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:32:29.097 INFO:journalctl@ceph.mon.a.smithi096.stdout: Cumulative compaction: 0.00 GB write, 17.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:32:29.097 INFO:journalctl@ceph.mon.a.smithi096.stdout: Interval compaction: 0.00 GB write, 17.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:32:29.097 INFO:journalctl@ceph.mon.a.smithi096.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:32:29.097 INFO:journalctl@ceph.mon.a.smithi096.stdout: Block cache BinnedLRUCache@0x55e515df2850#2 capacity: 512.00 MB usage: 0.47 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-09-13T14:32:29.097 INFO:journalctl@ceph.mon.a.smithi096.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.17 KB,3.27826e-05%) IndexBlock(1,0.30 KB,5.66244e-05%) Misc(1,0.00 KB,0%) 2024-09-13T14:32:29.097 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:32:29.097 INFO:journalctl@ceph.mon.a.smithi096.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:32:29.097 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: starting mon.a rank 0 at public addrs [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] at bind addrs [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:32:29.098 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: mon.a@-1(???) e3 preinit fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:32:29.098 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: mon.a@-1(???).mds e8 new map 2024-09-13T14:32:29.098 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: mon.a@-1(???).mds e8 print_map 2024-09-13T14:32:29.098 INFO:journalctl@ceph.mon.a.smithi096.stdout: e8 2024-09-13T14:32:29.098 INFO:journalctl@ceph.mon.a.smithi096.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T14:32:29.098 INFO:journalctl@ceph.mon.a.smithi096.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:32:29.098 INFO:journalctl@ceph.mon.a.smithi096.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:32:29.098 INFO:journalctl@ceph.mon.a.smithi096.stdout: legacy client fscid: 1 2024-09-13T14:32:29.098 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:32:29.098 INFO:journalctl@ceph.mon.a.smithi096.stdout: Filesystem 'foo' (1) 2024-09-13T14:32:29.098 INFO:journalctl@ceph.mon.a.smithi096.stdout: fs_name foo 2024-09-13T14:32:29.098 INFO:journalctl@ceph.mon.a.smithi096.stdout: epoch 7 2024-09-13T14:32:29.098 INFO:journalctl@ceph.mon.a.smithi096.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T14:32:29.098 INFO:journalctl@ceph.mon.a.smithi096.stdout: created 2024-09-13T14:28:10.489785+0000 2024-09-13T14:32:29.099 INFO:journalctl@ceph.mon.a.smithi096.stdout: modified 2024-09-13T14:28:20.810288+0000 2024-09-13T14:32:29.099 INFO:journalctl@ceph.mon.a.smithi096.stdout: tableserver 0 2024-09-13T14:32:29.099 INFO:journalctl@ceph.mon.a.smithi096.stdout: root 0 2024-09-13T14:32:29.099 INFO:journalctl@ceph.mon.a.smithi096.stdout: session_timeout 60 2024-09-13T14:32:29.099 INFO:journalctl@ceph.mon.a.smithi096.stdout: session_autoclose 300 2024-09-13T14:32:29.099 INFO:journalctl@ceph.mon.a.smithi096.stdout: max_file_size 1099511627776 2024-09-13T14:32:29.099 INFO:journalctl@ceph.mon.a.smithi096.stdout: max_xattr_size 65536 2024-09-13T14:32:29.099 INFO:journalctl@ceph.mon.a.smithi096.stdout: required_client_features {} 2024-09-13T14:32:29.099 INFO:journalctl@ceph.mon.a.smithi096.stdout: last_failure 0 2024-09-13T14:32:29.099 INFO:journalctl@ceph.mon.a.smithi096.stdout: last_failure_osd_epoch 0 2024-09-13T14:32:29.099 INFO:journalctl@ceph.mon.a.smithi096.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:32:29.099 INFO:journalctl@ceph.mon.a.smithi096.stdout: max_mds 1 2024-09-13T14:32:29.100 INFO:journalctl@ceph.mon.a.smithi096.stdout: in 0 2024-09-13T14:32:29.100 INFO:journalctl@ceph.mon.a.smithi096.stdout: up {0=24539} 2024-09-13T14:32:29.100 INFO:journalctl@ceph.mon.a.smithi096.stdout: failed 2024-09-13T14:32:29.100 INFO:journalctl@ceph.mon.a.smithi096.stdout: damaged 2024-09-13T14:32:29.100 INFO:journalctl@ceph.mon.a.smithi096.stdout: stopped 2024-09-13T14:32:29.100 INFO:journalctl@ceph.mon.a.smithi096.stdout: data_pools [4] 2024-09-13T14:32:29.100 INFO:journalctl@ceph.mon.a.smithi096.stdout: metadata_pool 3 2024-09-13T14:32:29.100 INFO:journalctl@ceph.mon.a.smithi096.stdout: inline_data disabled 2024-09-13T14:32:29.100 INFO:journalctl@ceph.mon.a.smithi096.stdout: balancer 2024-09-13T14:32:29.100 INFO:journalctl@ceph.mon.a.smithi096.stdout: bal_rank_mask -1 2024-09-13T14:32:29.100 INFO:journalctl@ceph.mon.a.smithi096.stdout: standby_count_wanted 1 2024-09-13T14:32:29.100 INFO:journalctl@ceph.mon.a.smithi096.stdout: qdb_cluster leader: 0 members: 2024-09-13T14:32:29.100 INFO:journalctl@ceph.mon.a.smithi096.stdout: [mds.foo.smithi096.dqiqpz{0:24539} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.96:6834/1494616691,v1:172.21.15.96:6835/1494616691] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:32:29.101 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:32:29.101 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:32:29.101 INFO:journalctl@ceph.mon.a.smithi096.stdout: Standby daemons: 2024-09-13T14:32:29.101 INFO:journalctl@ceph.mon.a.smithi096.stdout: 2024-09-13T14:32:29.101 INFO:journalctl@ceph.mon.a.smithi096.stdout: [mds.foo.smithi204.hbwbot{-1:24484} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.204:6832/291805447,v1:172.21.15.204:6833/291805447] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:32:29.101 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: mon.a@-1(???).osd e81 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T14:32:29.101 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: mon.a@-1(???).osd e81 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:32:29.101 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: mon.a@-1(???).osd e81 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:32:29.101 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: mon.a@-1(???).osd e81 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:32:29.101 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 ceph-mon[98901]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2024-09-13T14:32:29.101 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 bash[98861]: 97a94128e048ed7eb609e53a28bf4a29c94b24310260d4b5b69fa0fcdae33191 2024-09-13T14:32:29.101 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:28 smithi096 systemd[1]: Started Ceph mon.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:32:29.382 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:32:30.469 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[77750]: ignoring --setuser ceph since I am not root 2024-09-13T14:32:30.469 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[77750]: ignoring --setgroup ceph since I am not root 2024-09-13T14:32:30.469 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mgr[77754]: -- 172.21.15.204:0/3739452664 <== mon.1 v2:172.21.15.204:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557502c310e0 con 0x557502c0f400 2024-09-13T14:32:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:30.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:32:30.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:32:30.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:32:30.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:32:30.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:32:30.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:30.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/1826059705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:30.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: mon.a calling monitor election 2024-09-13T14:32:30.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: from='client.26570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:30.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:32:30.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: monmap epoch 3 2024-09-13T14:32:30.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:32:30.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: last_changed 2024-09-13T14:22:50.469402+0000 2024-09-13T14:32:30.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: created 2024-09-13T14:21:13.478657+0000 2024-09-13T14:32:30.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: min_mon_release 17 (quincy) 2024-09-13T14:32:30.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: election_strategy: 1 2024-09-13T14:32:30.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: 0: [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon.a 2024-09-13T14:32:30.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: 1: [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] mon.b 2024-09-13T14:32:30.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: 2: [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0] mon.c 2024-09-13T14:32:30.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:32:30.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:32:30.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: mgrmap e32: y(active, since 33s), standbys: x 2024-09-13T14:32:30.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: overall HEALTH_OK 2024-09-13T14:32:30.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: from='mgr.26285 ' entity='' 2024-09-13T14:32:30.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:30 smithi204 ceph-mon[32546]: mgrmap e33: y(active, since 34s), standbys: x 2024-09-13T14:32:30.618 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ignoring --setuser ceph since I am not root 2024-09-13T14:32:30.618 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ignoring --setgroup ceph since I am not root 2024-09-13T14:32:30.619 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mgr[89385]: -- 172.21.15.96:0/2209483344 <== mon.2 v2:172.21.15.96:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5581b8f4b0e0 con 0x5581b8f29400 2024-09-13T14:32:30.619 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:32:30.619 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:32:30.619 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:32:30.619 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:32:30.619 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:32:30.619 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:30.619 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/1826059705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:30.619 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: mon.a calling monitor election 2024-09-13T14:32:30.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: from='client.26570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:30.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:32:30.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: monmap epoch 3 2024-09-13T14:32:30.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:32:30.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: last_changed 2024-09-13T14:22:50.469402+0000 2024-09-13T14:32:30.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: created 2024-09-13T14:21:13.478657+0000 2024-09-13T14:32:30.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: min_mon_release 17 (quincy) 2024-09-13T14:32:30.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: election_strategy: 1 2024-09-13T14:32:30.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: 0: [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon.a 2024-09-13T14:32:30.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: 1: [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] mon.b 2024-09-13T14:32:30.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: 2: [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0] mon.c 2024-09-13T14:32:30.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:32:30.620 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:32:30.621 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: mgrmap e32: y(active, since 33s), standbys: x 2024-09-13T14:32:30.621 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: overall HEALTH_OK 2024-09-13T14:32:30.621 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: from='mgr.26285 ' entity='' 2024-09-13T14:32:30.621 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[39645]: mgrmap e33: y(active, since 34s), standbys: x 2024-09-13T14:32:30.621 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:32:30.621 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:32:30.621 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:32:30.621 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: from='mgr.26285 172.21.15.96:0/963254820' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:32:30.621 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:32:30.621 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:30.621 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1826059705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:32:30.622 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: mon.a calling monitor election 2024-09-13T14:32:30.622 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: from='client.26570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:30.622 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:32:30.622 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: monmap epoch 3 2024-09-13T14:32:30.622 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:32:30.622 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: last_changed 2024-09-13T14:22:50.469402+0000 2024-09-13T14:32:30.622 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: created 2024-09-13T14:21:13.478657+0000 2024-09-13T14:32:30.622 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: min_mon_release 17 (quincy) 2024-09-13T14:32:30.622 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: election_strategy: 1 2024-09-13T14:32:30.622 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: 0: [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon.a 2024-09-13T14:32:30.622 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: 1: [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] mon.b 2024-09-13T14:32:30.623 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: 2: [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0] mon.c 2024-09-13T14:32:30.623 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:32:30.623 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:32:30.623 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: mgrmap e32: y(active, since 33s), standbys: x 2024-09-13T14:32:30.623 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: overall HEALTH_OK 2024-09-13T14:32:30.623 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: from='mgr.26285 ' entity='' 2024-09-13T14:32:30.623 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:30 smithi096 ceph-mon[98901]: mgrmap e33: y(active, since 34s), standbys: x 2024-09-13T14:32:32.007 INFO:teuthology.orchestra.run.smithi096.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:32:32.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:32 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/296189588' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:32:32.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:32 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/296189588' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:32:32.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:32 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/296189588' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:32:32.747 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:32:33.011 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:34.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:34 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:34.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:35.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:35 smithi204 ceph-mon[32546]: Standby manager daemon x restarted 2024-09-13T14:32:35.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:35 smithi204 ceph-mon[32546]: Standby manager daemon x started 2024-09-13T14:32:35.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:35 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/1509974749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:32:35.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:35 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/1509974749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:32:35.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:35 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/1509974749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:32:35.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:35 smithi204 ceph-mon[32546]: from='mgr.? 172.21.15.204:0/1509974749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:32:35.719 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:35 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[77750]: [13/Sep/2024:14:32:35] ENGINE Bus STARTING 2024-09-13T14:32:35.719 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:35 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[77750]: CherryPy Checker: 2024-09-13T14:32:35.719 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:35 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[77750]: The Application mounted at '' has an empty config. 2024-09-13T14:32:35.719 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:35 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[77750]: 2024-09-13T14:32:35.719 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:35 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[77750]: [13/Sep/2024:14:32:35] ENGINE Serving on http://:::9283 2024-09-13T14:32:35.720 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 14:32:35 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x[77750]: [13/Sep/2024:14:32:35] ENGINE Bus STARTED 2024-09-13T14:32:35.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-mon[98901]: Standby manager daemon x restarted 2024-09-13T14:32:35.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-mon[98901]: Standby manager daemon x started 2024-09-13T14:32:35.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-mon[98901]: from='mgr.? 172.21.15.204:0/1509974749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:32:35.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-mon[98901]: from='mgr.? 172.21.15.204:0/1509974749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:32:35.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-mon[98901]: from='mgr.? 172.21.15.204:0/1509974749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:32:35.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-mon[98901]: from='mgr.? 172.21.15.204:0/1509974749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:32:35.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-mon[39645]: Standby manager daemon x restarted 2024-09-13T14:32:35.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-mon[39645]: Standby manager daemon x started 2024-09-13T14:32:35.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/1509974749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:32:35.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/1509974749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:32:35.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/1509974749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:32:35.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-mon[39645]: from='mgr.? 172.21.15.204:0/1509974749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:32:36.048 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:35 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:36.048 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:35 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:36.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:32:35.850+0000 7ff017ee4640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:32:36.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:32:35.850+0000 7ff017ee4640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:32:36.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:32:35.850+0000 7ff017ee4640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:32:36.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:32:35.850+0000 7ff017ee4640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:32:36.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:32:35.850+0000 7ff017ee4640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:32:36.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:32:35.851+0000 7ff01aeea640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:32:36.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:32:35.851+0000 7ff01aeea640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:32:36.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:32:35.851+0000 7ff01aeea640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:32:36.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:32:35.851+0000 7ff01aeea640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:32:36.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:35 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:32:35.851+0000 7ff01aeea640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:32:36.539 INFO:teuthology.orchestra.run.smithi096.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:32:36.539 INFO:teuthology.orchestra.run.smithi096.stdout:alertmanager.a smithi096 *:9093,9094 running (80s) 35s ago 5m 18.9M - 0.27.0 11f11916f8cd e04c10a0dca2 2024-09-13T14:32:36.539 INFO:teuthology.orchestra.run.smithi096.stdout:grafana.a smithi204 *:3000 running (58s) 16s ago 5m 79.9M - 9.4.7 954c08fa6188 451523aebd73 2024-09-13T14:32:36.539 INFO:teuthology.orchestra.run.smithi096.stdout:mds.foo.smithi096.dqiqpz smithi096 running (4m) 35s ago 4m 18.6M - 17.2.7-1627-gbbdadfa3 a994b00deb3a ab8d33048bb7 2024-09-13T14:32:36.539 INFO:teuthology.orchestra.run.smithi096.stdout:mds.foo.smithi204.hbwbot smithi204 running (4m) 16s ago 4m 15.3M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 97a3ff635de2 2024-09-13T14:32:36.539 INFO:teuthology.orchestra.run.smithi096.stdout:mgr.x smithi204 *:8443,9283,8765 running (21s) 16s ago 9m 66.3M - 19.3.0-4697-gd58f7306 fec07016a2b0 04572bea1e6f 2024-09-13T14:32:36.539 INFO:teuthology.orchestra.run.smithi096.stdout:mgr.y smithi096 *:8443,9283,8765 running (48s) 35s ago 11m 488M - 19.3.0-4697-gd58f7306 fec07016a2b0 a4bc51171f6e 2024-09-13T14:32:36.539 INFO:teuthology.orchestra.run.smithi096.stdout:mon.a smithi096 running (11m) 35s ago 11m 57.2M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 96e95ed67f92 2024-09-13T14:32:36.539 INFO:teuthology.orchestra.run.smithi096.stdout:mon.b smithi204 running (9m) 16s ago 9m 45.2M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a a37b0845979d 2024-09-13T14:32:36.539 INFO:teuthology.orchestra.run.smithi096.stdout:mon.c smithi096 running (9m) 35s ago 9m 43.3M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 44a18794779c 2024-09-13T14:32:36.540 INFO:teuthology.orchestra.run.smithi096.stdout:node-exporter.a smithi096 *:9100 running (74s) 35s ago 5m 9835k - 1.7.0 72c9c2088986 b02b57424615 2024-09-13T14:32:36.540 INFO:teuthology.orchestra.run.smithi096.stdout:node-exporter.b smithi204 *:9100 running (53s) 16s ago 5m 7658k - 1.7.0 72c9c2088986 c58ad3b83def 2024-09-13T14:32:36.540 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0 smithi096 running (9m) 35s ago 9m 46.7M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a a35fe012b65d 2024-09-13T14:32:36.540 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1 smithi096 running (8m) 35s ago 8m 50.4M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a edbcf0f2ed44 2024-09-13T14:32:36.540 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2 smithi096 running (8m) 35s ago 8m 47.4M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 565505fd2e48 2024-09-13T14:32:36.540 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3 smithi096 running (7m) 35s ago 7m 44.0M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a afd4dcb32d2d 2024-09-13T14:32:36.540 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4 smithi204 running (7m) 16s ago 7m 44.1M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a b48920b2b9e7 2024-09-13T14:32:36.540 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5 smithi204 running (7m) 16s ago 7m 46.0M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 0ed6e334d2ea 2024-09-13T14:32:36.540 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6 smithi204 running (6m) 16s ago 6m 47.2M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8f12f56d751c 2024-09-13T14:32:36.540 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7 smithi204 running (6m) 16s ago 6m 50.6M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a ffd75c8be38a 2024-09-13T14:32:36.540 INFO:teuthology.orchestra.run.smithi096.stdout:prometheus.a smithi204 *:9095 running (26s) 16s ago 5m 37.2M - 2.51.0 1d3b7f56885b c1d744b6a96b 2024-09-13T14:32:36.644 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:32:36.519+0000 7ff04b218640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-13T14:32:36.645 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:32:36] ENGINE Bus STARTING 2024-09-13T14:32:36.645 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: CherryPy Checker: 2024-09-13T14:32:36.645 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: The Application mounted at '' has an empty config. 2024-09-13T14:32:36.645 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:32:36.645 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:32:36] ENGINE Serving on http://:::9283 2024-09-13T14:32:36.645 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: mgrmap e34: y(active, since 39s), standbys: x 2024-09-13T14:32:36.645 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: Active manager daemon y restarted 2024-09-13T14:32:36.645 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: Activating manager daemon y 2024-09-13T14:32:36.645 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:32:36.646 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: mgrmap e35: y(active, starting, since 0.00515693s), standbys: x 2024-09-13T14:32:36.646 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:32:36.646 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:32:36.646 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:32:36.646 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi204.hbwbot"}]: dispatch 2024-09-13T14:32:36.646 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi096.dqiqpz"}]: dispatch 2024-09-13T14:32:36.646 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:32:36.646 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:32:36.646 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:32:36.646 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:32:36.646 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:32:36.646 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:32:36.646 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:32:36.646 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:32:36.647 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:32:36.647 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:32:36.647 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:32:36.647 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:32:36.647 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:32:36.647 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: Manager daemon y is now available 2024-09-13T14:32:36.647 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:32:36.647 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:32:36.647 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:32:36.647 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:32:36.647 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:32:36.647 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:32:36.647 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:32:36.648 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: mgrmap e34: y(active, since 39s), standbys: x 2024-09-13T14:32:36.648 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: Active manager daemon y restarted 2024-09-13T14:32:36.648 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: Activating manager daemon y 2024-09-13T14:32:36.648 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:32:36.648 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: mgrmap e35: y(active, starting, since 0.00515693s), standbys: x 2024-09-13T14:32:36.648 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:32:36.648 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:32:36.648 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:32:36.648 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi204.hbwbot"}]: dispatch 2024-09-13T14:32:36.648 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi096.dqiqpz"}]: dispatch 2024-09-13T14:32:36.648 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:32:36.649 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:32:36.649 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:32:36.649 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:32:36.649 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:32:36.649 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:32:36.649 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:32:36.649 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:32:36.649 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:32:36.649 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:32:36.649 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:32:36.649 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:32:36.649 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:32:36.649 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: Manager daemon y is now available 2024-09-13T14:32:36.649 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:32:36.650 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:32:36.650 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:32:36.650 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:32:36.650 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:32:36.650 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:32:36.650 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:32:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: mgrmap e34: y(active, since 39s), standbys: x 2024-09-13T14:32:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: Active manager daemon y restarted 2024-09-13T14:32:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: Activating manager daemon y 2024-09-13T14:32:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:32:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: mgrmap e35: y(active, starting, since 0.00515693s), standbys: x 2024-09-13T14:32:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:32:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:32:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:32:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi204.hbwbot"}]: dispatch 2024-09-13T14:32:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi096.dqiqpz"}]: dispatch 2024-09-13T14:32:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:32:36.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:32:36.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:32:36.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:32:36.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:32:36.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:32:36.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:32:36.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:32:36.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:32:36.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:32:36.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:32:36.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:32:36.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:32:36.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: Manager daemon y is now available 2024-09-13T14:32:36.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:32:36.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:32:36.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:32:36.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:32:36.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:32:36.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:32:36.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:32:36.924 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:32:36] ENGINE Bus STARTED 2024-09-13T14:32:36.924 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:32:36.729+0000 7feffed38640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:32:36] ENGINE Error in HTTPServer.serve 2024-09-13T14:32:36.924 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: Traceback (most recent call last): 2024-09-13T14:32:36.924 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:32:36.924 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: self._connections.run(self.expiration_interval) 2024-09-13T14:32:36.924 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:32:36.924 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: self._run(expiration_interval) 2024-09-13T14:32:36.924 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:32:36.925 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:32:36.925 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:32:36.925 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:32:36.925 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:32:36.925 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: s = self.context.wrap_socket( 2024-09-13T14:32:36.925 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:32:36.925 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: return self.sslsocket_class._create( 2024-09-13T14:32:36.925 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:32:36.925 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: self.do_handshake() 2024-09-13T14:32:36.925 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:32:36.925 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: self._sslobj.do_handshake() 2024-09-13T14:32:36.925 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:32:36.925 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:36 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:32:37.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:36 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:37.307 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade pause' 2024-09-13T14:32:37.553 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:37.553 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:37.553 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[98901]: mgrmap e36: y(active, since 1.01419s), standbys: x 2024-09-13T14:32:37.553 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[98901]: [13/Sep/2024:14:32:36] ENGINE Bus STARTING 2024-09-13T14:32:37.553 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[98901]: [13/Sep/2024:14:32:36] ENGINE Serving on https://172.21.15.96:7150 2024-09-13T14:32:37.553 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[98901]: [13/Sep/2024:14:32:36] ENGINE Error in HTTPServer.serve 2024-09-13T14:32:37.553 INFO:journalctl@ceph.mon.a.smithi096.stdout: Traceback (most recent call last): 2024-09-13T14:32:37.553 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:32:37.553 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:32:37.554 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:32:37.554 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._run(expiration_interval) 2024-09-13T14:32:37.554 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:32:37.554 INFO:journalctl@ceph.mon.a.smithi096.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:32:37.554 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:32:37.554 INFO:journalctl@ceph.mon.a.smithi096.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:32:37.554 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:32:37.554 INFO:journalctl@ceph.mon.a.smithi096.stdout: s = self.context.wrap_socket( 2024-09-13T14:32:37.554 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:32:37.554 INFO:journalctl@ceph.mon.a.smithi096.stdout: return self.sslsocket_class._create( 2024-09-13T14:32:37.554 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:32:37.554 INFO:journalctl@ceph.mon.a.smithi096.stdout: self.do_handshake() 2024-09-13T14:32:37.554 INFO:journalctl@ceph.mon.a.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:32:37.554 INFO:journalctl@ceph.mon.a.smithi096.stdout: self._sslobj.do_handshake() 2024-09-13T14:32:37.555 INFO:journalctl@ceph.mon.a.smithi096.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:32:37.555 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[98901]: [13/Sep/2024:14:32:36] ENGINE Serving on http://172.21.15.96:8765 2024-09-13T14:32:37.555 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[98901]: [13/Sep/2024:14:32:36] ENGINE Bus STARTED 2024-09-13T14:32:37.555 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:37.555 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:37.555 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[39645]: mgrmap e36: y(active, since 1.01419s), standbys: x 2024-09-13T14:32:37.555 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[39645]: [13/Sep/2024:14:32:36] ENGINE Bus STARTING 2024-09-13T14:32:37.555 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[39645]: [13/Sep/2024:14:32:36] ENGINE Serving on https://172.21.15.96:7150 2024-09-13T14:32:37.555 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[39645]: [13/Sep/2024:14:32:36] ENGINE Error in HTTPServer.serve 2024-09-13T14:32:37.555 INFO:journalctl@ceph.mon.c.smithi096.stdout: Traceback (most recent call last): 2024-09-13T14:32:37.555 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:32:37.556 INFO:journalctl@ceph.mon.c.smithi096.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:32:37.556 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:32:37.556 INFO:journalctl@ceph.mon.c.smithi096.stdout: self._run(expiration_interval) 2024-09-13T14:32:37.556 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:32:37.556 INFO:journalctl@ceph.mon.c.smithi096.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:32:37.556 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:32:37.556 INFO:journalctl@ceph.mon.c.smithi096.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:32:37.556 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:32:37.556 INFO:journalctl@ceph.mon.c.smithi096.stdout: s = self.context.wrap_socket( 2024-09-13T14:32:37.556 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:32:37.556 INFO:journalctl@ceph.mon.c.smithi096.stdout: return self.sslsocket_class._create( 2024-09-13T14:32:37.556 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:32:37.556 INFO:journalctl@ceph.mon.c.smithi096.stdout: self.do_handshake() 2024-09-13T14:32:37.556 INFO:journalctl@ceph.mon.c.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:32:37.557 INFO:journalctl@ceph.mon.c.smithi096.stdout: self._sslobj.do_handshake() 2024-09-13T14:32:37.557 INFO:journalctl@ceph.mon.c.smithi096.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:32:37.557 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[39645]: [13/Sep/2024:14:32:36] ENGINE Serving on http://172.21.15.96:8765 2024-09-13T14:32:37.557 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-mon[39645]: [13/Sep/2024:14:32:36] ENGINE Bus STARTED 2024-09-13T14:32:37.831 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:37 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:37.831 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:37 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:37.831 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:37 smithi204 ceph-mon[32546]: mgrmap e36: y(active, since 1.01419s), standbys: x 2024-09-13T14:32:37.831 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:37 smithi204 ceph-mon[32546]: [13/Sep/2024:14:32:36] ENGINE Bus STARTING 2024-09-13T14:32:37.831 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:37 smithi204 ceph-mon[32546]: [13/Sep/2024:14:32:36] ENGINE Serving on https://172.21.15.96:7150 2024-09-13T14:32:37.831 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:37 smithi204 ceph-mon[32546]: [13/Sep/2024:14:32:36] ENGINE Error in HTTPServer.serve 2024-09-13T14:32:37.831 INFO:journalctl@ceph.mon.b.smithi204.stdout: Traceback (most recent call last): 2024-09-13T14:32:37.832 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:32:37.832 INFO:journalctl@ceph.mon.b.smithi204.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:32:37.832 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:32:37.832 INFO:journalctl@ceph.mon.b.smithi204.stdout: self._run(expiration_interval) 2024-09-13T14:32:37.832 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:32:37.832 INFO:journalctl@ceph.mon.b.smithi204.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:32:37.832 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:32:37.832 INFO:journalctl@ceph.mon.b.smithi204.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:32:37.832 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:32:37.832 INFO:journalctl@ceph.mon.b.smithi204.stdout: s = self.context.wrap_socket( 2024-09-13T14:32:37.832 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:32:37.832 INFO:journalctl@ceph.mon.b.smithi204.stdout: return self.sslsocket_class._create( 2024-09-13T14:32:37.832 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:32:37.833 INFO:journalctl@ceph.mon.b.smithi204.stdout: self.do_handshake() 2024-09-13T14:32:37.833 INFO:journalctl@ceph.mon.b.smithi204.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:32:37.833 INFO:journalctl@ceph.mon.b.smithi204.stdout: self._sslobj.do_handshake() 2024-09-13T14:32:37.833 INFO:journalctl@ceph.mon.b.smithi204.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:32:37.833 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:37 smithi204 ceph-mon[32546]: [13/Sep/2024:14:32:36] ENGINE Serving on http://172.21.15.96:8765 2024-09-13T14:32:37.833 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:37 smithi204 ceph-mon[32546]: [13/Sep/2024:14:32:36] ENGINE Bus STARTED 2024-09-13T14:32:37.985 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:38.223 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:32:37] "GET /metrics HTTP/1.1" 200 31160 "" "Prometheus/2.51.0" 2024-09-13T14:32:38.793 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:38 smithi096 ceph-mon[98901]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:38.793 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:38 smithi096 ceph-mon[98901]: mgrmap e37: y(active, since 2s), standbys: x 2024-09-13T14:32:38.793 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:38 smithi096 ceph-mon[39645]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:38.794 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:38 smithi096 ceph-mon[39645]: mgrmap e37: y(active, since 2s), standbys: x 2024-09-13T14:32:38.818 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:38 smithi204 ceph-mon[32546]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:38.818 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:38 smithi204 ceph-mon[32546]: mgrmap e37: y(active, since 2s), standbys: x 2024-09-13T14:32:40.396 INFO:teuthology.orchestra.run.smithi096.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:32:40.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:40.697 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:40 smithi096 ceph-mon[98901]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:40.697 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:40 smithi096 ceph-mon[98901]: mgrmap e38: y(active, since 4s), standbys: x 2024-09-13T14:32:40.697 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:40 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:40.697 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:40 smithi096 ceph-mon[39645]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:40.698 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:40 smithi096 ceph-mon[39645]: mgrmap e38: y(active, since 4s), standbys: x 2024-09-13T14:32:40.698 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:40 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:40.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:40 smithi204 ceph-mon[32546]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:40.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:40 smithi204 ceph-mon[32546]: mgrmap e38: y(active, since 4s), standbys: x 2024-09-13T14:32:40.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:40 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:41.742 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'sleep 60' 2024-09-13T14:32:41.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:41 smithi204 ceph-mon[32546]: from='client.26642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:41.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:41 smithi204 ceph-mon[32546]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:32:41.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:41 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:41.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:41 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:41.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:41 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:41.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:41 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:42.005 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:41 smithi096 ceph-mon[98901]: from='client.26642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:42.005 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:41 smithi096 ceph-mon[98901]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:32:42.005 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:41 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:42.005 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:41 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:42.005 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:41 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:42.005 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:41 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:42.005 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:41 smithi096 ceph-mon[39645]: from='client.26642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:42.006 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:41 smithi096 ceph-mon[39645]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:32:42.006 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:41 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:42.006 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:41 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:42.006 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:41 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:42.006 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:41 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:42.515 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:43.538 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:43 smithi204 ceph-mon[32546]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:43.538 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:43 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:43.538 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:43 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:43.538 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:43 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:43.538 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:43 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:43.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:43 smithi096 ceph-mon[98901]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:43.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:43 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:43.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:43 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:43.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:43 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:43.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:43 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:43.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:43 smithi096 ceph-mon[39645]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:43.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:43 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:43.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:43 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:43.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:43 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:43.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:43 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:44.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:45.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:45 smithi096 ceph-mon[98901]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T14:32:45.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:45 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:45.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:45 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:45 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:45 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:32:45.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:45 smithi096 ceph-mon[39645]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T14:32:45.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:45 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:45.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:45 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:45.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:45 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:45.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:45 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:32:45.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:45 smithi204 ceph-mon[32546]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T14:32:45.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:45 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:45.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:45 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:45.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:45 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:45.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:45 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:32:46.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:45 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:46.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:45 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:46.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:32:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:32:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:32:46.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:46.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:32:46.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:32:46.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:32:46.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:46.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:46.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:46.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:46.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:46.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:46.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:32:46.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:46.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:32:46.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:46 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-13T14:32:46.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: Updating smithi204:/etc/ceph/ceph.conf 2024-09-13T14:32:46.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:46.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:46.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:46.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:46.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: Updating smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:46.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: Updating smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:46.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:46.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:32:46.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:32:46.724 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:32:46.724 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:32:47.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:47.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:47 smithi096 ceph-mon[98901]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:32:47.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:47 smithi096 ceph-mon[39645]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:32:47.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:47 smithi204 ceph-mon[32546]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:32:48.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:32:47] "GET /metrics HTTP/1.1" 200 31160 "" "Prometheus/2.51.0" 2024-09-13T14:32:49.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:49 smithi096 ceph-mon[98901]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T14:32:49.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:49 smithi096 ceph-mon[39645]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T14:32:49.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:49 smithi204 ceph-mon[32546]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T14:32:50.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:51.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:51 smithi096 ceph-mon[98901]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:51.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:32:51.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:51 smithi096 ceph-mon[39645]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:51.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:51 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:32:51.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:51 smithi204 ceph-mon[32546]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:51.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:51 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:32:53.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:53 smithi096 ceph-mon[98901]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:53.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:53 smithi096 ceph-mon[39645]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:53.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:53 smithi204 ceph-mon[32546]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:54.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:54 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:54.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:55.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:55 smithi096 ceph-mon[98901]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:55.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:55 smithi096 ceph-mon[39645]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:55.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:55 smithi204 ceph-mon[32546]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:32:56.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:56.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:57.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:32:56 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:32:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:57.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:57 smithi096 ceph-mon[98901]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:57.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:57 smithi096 ceph-mon[39645]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:57.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:57 smithi204 ceph-mon[32546]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:58.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:32:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:32:57] "GET /metrics HTTP/1.1" 200 33851 "" "Prometheus/2.51.0" 2024-09-13T14:32:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:32:59 smithi096 ceph-mon[98901]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:59.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:32:59 smithi096 ceph-mon[39645]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:59.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:32:59 smithi204 ceph-mon[32546]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:00.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:01.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:01 smithi096 ceph-mon[98901]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:01.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:01 smithi096 ceph-mon[39645]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:01.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:01 smithi204 ceph-mon[32546]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:03.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:03 smithi096 ceph-mon[98901]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:03.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:03 smithi096 ceph-mon[39645]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:03.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:03 smithi204 ceph-mon[32546]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:04.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:04 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:05.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:05 smithi096 ceph-mon[98901]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:05.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:05 smithi096 ceph-mon[39645]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:05 smithi204 ceph-mon[32546]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:06.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:05 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:06.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:05 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:06.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:33:06.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:06 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:33:06.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:06 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:33:07.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:07.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:07 smithi096 ceph-mon[98901]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:07.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:07 smithi096 ceph-mon[39645]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:07.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:07 smithi204 ceph-mon[32546]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:08.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:33:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:33:07] "GET /metrics HTTP/1.1" 200 33852 "" "Prometheus/2.51.0" 2024-09-13T14:33:09.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:09 smithi096 ceph-mon[98901]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:09.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:09 smithi096 ceph-mon[39645]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:09.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:09 smithi204 ceph-mon[32546]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:11.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:11 smithi096 ceph-mon[98901]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:11.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:11 smithi096 ceph-mon[39645]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:11 smithi204 ceph-mon[32546]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:13.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:13 smithi096 ceph-mon[98901]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:13.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:13 smithi096 ceph-mon[39645]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:13.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:13 smithi204 ceph-mon[32546]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:14.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:14 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:15.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:15 smithi096 ceph-mon[98901]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:15.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:15 smithi096 ceph-mon[39645]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:15.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:15 smithi204 ceph-mon[32546]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:16.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:15 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:16.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:15 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:17.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:16 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:17.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:17 smithi096 ceph-mon[98901]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:17.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:17 smithi096 ceph-mon[39645]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:17.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:17 smithi204 ceph-mon[32546]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:18.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:33:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:33:17] "GET /metrics HTTP/1.1" 200 33852 "" "Prometheus/2.51.0" 2024-09-13T14:33:19.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:19 smithi096 ceph-mon[98901]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:19.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:19 smithi096 ceph-mon[39645]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:19.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:19 smithi204 ceph-mon[32546]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:20.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:21.689 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:21 smithi204 ceph-mon[32546]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:21.689 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:21 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:33:21.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:21 smithi096 ceph-mon[39645]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:21.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:21 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:33:21.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:21 smithi096 ceph-mon[98901]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:21.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:33:23.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:23 smithi204 ceph-mon[32546]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:23.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:23 smithi096 ceph-mon[98901]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:23.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:23 smithi096 ceph-mon[39645]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:24.474 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:24 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:24.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:25.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:25 smithi204 ceph-mon[32546]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:25.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:25 smithi096 ceph-mon[98901]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:25.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:25 smithi096 ceph-mon[39645]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:26.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:25 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:26.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:25 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:27.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:26 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:27.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:27 smithi204 ceph-mon[32546]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:27.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:27 smithi096 ceph-mon[98901]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:27.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:27 smithi096 ceph-mon[39645]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:28.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:33:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:33:27] "GET /metrics HTTP/1.1" 200 33848 "" "Prometheus/2.51.0" 2024-09-13T14:33:29.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:29 smithi204 ceph-mon[32546]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:29.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:29 smithi096 ceph-mon[98901]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:29.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:29 smithi096 ceph-mon[39645]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:30.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:31.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:31 smithi204 ceph-mon[32546]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:31.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:31 smithi096 ceph-mon[98901]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:31.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:31 smithi096 ceph-mon[39645]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:33.624 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:33 smithi096 ceph-mon[98901]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:33.625 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:33 smithi096 ceph-mon[39645]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:33.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:33 smithi204 ceph-mon[32546]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:34.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:34 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:35.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:35 smithi204 ceph-mon[32546]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:35.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:35 smithi096 ceph-mon[98901]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:35.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:35 smithi096 ceph-mon[39645]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:36.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:35 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:36.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:35 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:36.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:36 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:33:36.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:33:36.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:36 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:33:37.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:36 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:37.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:37 smithi204 ceph-mon[32546]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:37.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:37 smithi096 ceph-mon[98901]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:37.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:37 smithi096 ceph-mon[39645]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:38.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:33:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:33:37] "GET /metrics HTTP/1.1" 200 33848 "" "Prometheus/2.51.0" 2024-09-13T14:33:39.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:39 smithi204 ceph-mon[32546]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:39.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:39 smithi096 ceph-mon[98901]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:39.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:39 smithi096 ceph-mon[39645]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:40.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:41.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:41 smithi204 ceph-mon[32546]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:41.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:41 smithi096 ceph-mon[98901]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:41.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:41 smithi096 ceph-mon[39645]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:43.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:43 smithi204 ceph-mon[32546]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:43.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:43 smithi096 ceph-mon[98901]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:43.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:43 smithi096 ceph-mon[39645]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:44.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:44.775 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade resume' 2024-09-13T14:33:45.027 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:45.456 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:45 smithi096 ceph-mon[98901]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:45.456 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:45 smithi096 ceph-mon[39645]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:45.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:45 smithi204 ceph-mon[32546]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:46.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:45 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:46.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:45 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:46.622 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:46 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:33:46.622 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:46 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:33:46.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:46 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:33:46.801 INFO:teuthology.orchestra.run.smithi096.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:33:47.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:47.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:47 smithi096 ceph-mon[98901]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:47.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:47 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:47.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:47 smithi096 ceph-mon[39645]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:47.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:47 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:47.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:47 smithi204 ceph-mon[32546]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:47.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:47 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:47.996 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for majority of mons to upgrade' 2024-09-13T14:33:48.316 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:33:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:33:47] "GET /metrics HTTP/1.1" 200 33848 "" "Prometheus/2.51.0" 2024-09-13T14:33:48.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:48 smithi096 ceph-mon[98901]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:33:48.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:48 smithi096 ceph-mon[98901]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:33:48.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:48 smithi096 ceph-mon[39645]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:33:48.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:48 smithi096 ceph-mon[39645]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:33:48.626 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:48 smithi204 ceph-mon[32546]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:33:48.626 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:48 smithi204 ceph-mon[32546]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:33:48.787 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:49.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:49 smithi096 ceph-mon[98901]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:49.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:49 smithi096 ceph-mon[39645]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:49.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:49 smithi204 ceph-mon[32546]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:50.362 INFO:teuthology.orchestra.run.smithi096.stdout:wait for majority of mons to upgrade 2024-09-13T14:33:50.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:51.016 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-13T14:33:51.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:50 smithi204 ceph-mon[32546]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:51.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:50 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:51.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:50 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:51.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:50 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:33:51.219 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:50 smithi096 ceph-mon[98901]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:51.219 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:50 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:51.220 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:50 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:51.220 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:50 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:33:51.220 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:50 smithi096 ceph-mon[39645]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:51.220 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:50 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:51.220 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:50 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:51.220 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:50 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:33:51.660 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:53.036 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:52 smithi096 ceph-mon[98901]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:53.036 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:52 smithi096 ceph-mon[39645]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:53.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:52 smithi204 ceph-mon[32546]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:54.004 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:33:54.148 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:53 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/2260971845' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:33:54.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:53 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2260971845' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:33:54.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:53 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/2260971845' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:33:54.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:54 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:55.047 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[98901]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:55.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:55.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:55.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[98901]: from='client.26660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:33:55.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:33:55.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:33:55.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:55.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[98901]: from='client.26666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:33:55.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:33:55.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:33:55.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:33:55.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-13T14:33:55.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[39645]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:55.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:55.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:55.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[39645]: from='client.26660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:33:55.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:33:55.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:33:55.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:55.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[39645]: from='client.26666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:33:55.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:33:55.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:33:55.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:33:55.050 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:54 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-13T14:33:55.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:54 smithi204 ceph-mon[32546]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:54 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:54 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:54 smithi204 ceph-mon[32546]: from='client.26660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:33:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:54 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:33:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:54 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:33:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:54 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:54 smithi204 ceph-mon[32546]: from='client.26666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:33:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:54 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:33:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:54 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:33:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:54 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:33:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:54 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-13T14:33:56.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:55 smithi096 ceph-mon[98901]: Upgrade: It appears safe to stop mon.c 2024-09-13T14:33:56.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:55 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:56.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:55 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:33:56.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:55 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:33:56.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:55 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:33:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:55 smithi096 ceph-mon[39645]: Upgrade: It appears safe to stop mon.c 2024-09-13T14:33:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:55 smithi096 ceph-mon[39645]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:55 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:33:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:55 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:33:56.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:55 smithi096 ceph-mon[39645]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:33:56.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:55 smithi204 ceph-mon[32546]: Upgrade: It appears safe to stop mon.c 2024-09-13T14:33:56.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:55 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:33:56.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:55 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:33:56.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:55 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:33:56.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:55 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:33:56.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:56.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:56.956 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:33:57.051 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:56 smithi096 ceph-mon[98901]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:57.051 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:56 smithi096 ceph-mon[98901]: Upgrade: Updating mon.c 2024-09-13T14:33:57.051 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:56 smithi096 ceph-mon[98901]: Deploying daemon mon.c on smithi096 2024-09-13T14:33:57.051 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:33:56 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/674456330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:33:57.051 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:56 smithi096 ceph-mon[39645]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:57.052 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:56 smithi096 ceph-mon[39645]: Upgrade: Updating mon.c 2024-09-13T14:33:57.052 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:56 smithi096 ceph-mon[39645]: Deploying daemon mon.c on smithi096 2024-09-13T14:33:57.052 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:56 smithi096 ceph-mon[39645]: from='client.? 172.21.15.96:0/674456330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:33:57.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:56 smithi204 ceph-mon[32546]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:57.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:56 smithi204 ceph-mon[32546]: Upgrade: Updating mon.c 2024-09-13T14:33:57.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:56 smithi204 ceph-mon[32546]: Deploying daemon mon.c on smithi096 2024-09-13T14:33:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:33:56 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/674456330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:33:57.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:33:56 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:33:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:57.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:57 smithi096 systemd[1]: Stopping Ceph mon.c for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:33:57.946 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:33:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:33:57] ENGINE Bus STOPPING 2024-09-13T14:33:57.946 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c[39641]: 2024-09-13T14:33:57.588+0000 7f39722e0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:33:57.946 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c[39641]: 2024-09-13T14:33:57.588+0000 7f39722e0640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-13T14:33:57.946 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:57 smithi096 podman[102767]: 2024-09-13 14:33:57.711510522 +0000 UTC m=+0.244198420 container died 44a18794779ce339a72b7a128c5a8fe5eb09b65f67484a9ab4a3cbb4c6d52f64 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731) 2024-09-13T14:33:58.214 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:33:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:33:57] "GET /metrics HTTP/1.1" 200 33849 "" "Prometheus/2.51.0" 2024-09-13T14:33:58.215 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:33:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:33:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:33:58.215 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:33:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:33:57] ENGINE Bus STOPPED 2024-09-13T14:33:58.466 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:58 smithi096 podman[102767]: 2024-09-13 14:33:58.214537647 +0000 UTC m=+0.747225543 container cleanup 44a18794779ce339a72b7a128c5a8fe5eb09b65f67484a9ab4a3cbb4c6d52f64 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2) 2024-09-13T14:33:58.467 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:58 smithi096 bash[102767]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c 2024-09-13T14:33:58.788 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:58 smithi096 podman[102779]: 2024-09-13 14:33:58.466296309 +0000 UTC m=+0.759275019 container remove 44a18794779ce339a72b7a128c5a8fe5eb09b65f67484a9ab4a3cbb4c6d52f64 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-13T14:33:58.788 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:58 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-13T14:33:58.788 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:58 smithi096 systemd[1]: Stopped Ceph mon.c for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:33:58.788 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:58 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.c.service: Consumed 7.657s CPU time. 2024-09-13T14:33:59.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:58 smithi096 systemd[1]: Starting Ceph mon.c for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:33:59.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:58 smithi096 podman[102908]: 2024-09-13 14:33:58.896573374 +0000 UTC m=+0.015801554 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:33:59.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 podman[102908]: 2024-09-13 14:33:59.008907961 +0000 UTC m=+0.128136132 container create babdd691d9b6f9db01e49db29e8b8f6650954acad4c80ce77f47b922f1e27943 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-13T14:33:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 podman[102908]: 2024-09-13 14:33:59.253186124 +0000 UTC m=+0.372414294 container init babdd691d9b6f9db01e49db29e8b8f6650954acad4c80ce77f47b922f1e27943 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-13T14:33:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 podman[102908]: 2024-09-13 14:33:59.256404349 +0000 UTC m=+0.375632521 container start babdd691d9b6f9db01e49db29e8b8f6650954acad4c80ce77f47b922f1e27943 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-13T14:33:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:33:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev), process ceph-mon, pid 2 2024-09-13T14:33:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: pidfile_write: ignore empty --pid-file 2024-09-13T14:33:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: load: jerasure load: lrc 2024-09-13T14:33:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: RocksDB version: 7.9.2 2024-09-13T14:33:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Git sha 0 2024-09-13T14:33:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Compile date 2024-09-12 20:46:14 2024-09-13T14:33:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: DB SUMMARY 2024-09-13T14:33:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: DB Session ID: EZIYF4RM3PBOCQ9UALIW 2024-09-13T14:33:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:33:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:33:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1085 Bytes 2024-09-13T14:33:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-09-13T14:33:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 4036913 ; 2024-09-13T14:33:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:33:59.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:33:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:33:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:33:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:33:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:33:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.env: 0x561055cbf160 2024-09-13T14:33:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:33:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.info_log: 0x561057e57880 2024-09-13T14:33:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:33:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.statistics: (nil) 2024-09-13T14:33:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.use_fsync: 0 2024-09-13T14:33:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:33:59.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:33:59.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:33:59.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:33:59.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:33:59.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:33:59.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:33:59.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:33:59.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:33:59.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:33:59.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:33:59.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.db_log_dir: 2024-09-13T14:33:59.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.wal_dir: 2024-09-13T14:33:59.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:33:59.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:33:59.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:33:59.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:33:59.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:33:59.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:33:59.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:33:59.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:33:59.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.write_buffer_manager: 0x561057e5a960 2024-09-13T14:33:59.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:33:59.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:33:59.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:33:59.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:33:59.578 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:33:59.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:33:59.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:33:59.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:33:59.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.unordered_write: 0 2024-09-13T14:33:59.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:33:59.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:33:59.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:33:59.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:33:59.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.row_cache: None 2024-09-13T14:33:59.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.wal_filter: None 2024-09-13T14:33:59.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:33:59.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:33:59.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:33:59.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:33:59.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.wal_compression: 0 2024-09-13T14:33:59.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:33:59.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:33:59.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:33:59.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:33:59.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:33:59.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:33:59.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:33:59.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:33:59.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:33:59.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:33:59.580 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:33:59.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:33:59.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:33:59.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:33:59.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:33:59.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:33:59.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:33:59.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:33:59.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:33:59.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:33:59.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:33:59.581 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:33:59.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:33:59.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_open_files: -1 2024-09-13T14:33:59.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:33:59.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:33:59.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:33:59.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:33:59.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:33:59.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Compression algorithms supported: 2024-09-13T14:33:59.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: kZSTD supported: 0 2024-09-13T14:33:59.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:33:59.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:33:59.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:33:59.582 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:33:59.583 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:33:59.583 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:33:59.583 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:33:59.583 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:33:59.583 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:33:59.583 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-09-13T14:33:59.583 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:33:59.583 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:33:59.583 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.merge_operator: 2024-09-13T14:33:59.583 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compaction_filter: None 2024-09-13T14:33:59.583 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:33:59.583 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:33:59.583 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:33:59.583 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:33:59.584 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561057e574e0) 2024-09-13T14:33:59.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:33:59.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:33:59.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:33:59.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:33:59.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: index_type: 0 2024-09-13T14:33:59.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: data_block_index_type: 0 2024-09-13T14:33:59.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: index_shortening: 1 2024-09-13T14:33:59.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:33:59.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: checksum: 4 2024-09-13T14:33:59.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: no_block_cache: 0 2024-09-13T14:33:59.584 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_cache: 0x561057e4a850 2024-09-13T14:33:59.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:33:59.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_cache_options: 2024-09-13T14:33:59.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: capacity : 536870912 2024-09-13T14:33:59.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: num_shard_bits : 4 2024-09-13T14:33:59.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: strict_capacity_limit : 0 2024-09-13T14:33:59.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:33:59.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_cache_compressed: (nil) 2024-09-13T14:33:59.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: persistent_cache: (nil) 2024-09-13T14:33:59.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_size: 4096 2024-09-13T14:33:59.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_size_deviation: 10 2024-09-13T14:33:59.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_restart_interval: 16 2024-09-13T14:33:59.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: index_block_restart_interval: 1 2024-09-13T14:33:59.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: metadata_block_size: 4096 2024-09-13T14:33:59.585 INFO:journalctl@ceph.mon.c.smithi096.stdout: partition_filters: 0 2024-09-13T14:33:59.586 INFO:journalctl@ceph.mon.c.smithi096.stdout: use_delta_encoding: 1 2024-09-13T14:33:59.586 INFO:journalctl@ceph.mon.c.smithi096.stdout: filter_policy: bloomfilter 2024-09-13T14:33:59.586 INFO:journalctl@ceph.mon.c.smithi096.stdout: whole_key_filtering: 1 2024-09-13T14:33:59.586 INFO:journalctl@ceph.mon.c.smithi096.stdout: verify_compression: 0 2024-09-13T14:33:59.586 INFO:journalctl@ceph.mon.c.smithi096.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:33:59.586 INFO:journalctl@ceph.mon.c.smithi096.stdout: format_version: 5 2024-09-13T14:33:59.586 INFO:journalctl@ceph.mon.c.smithi096.stdout: enable_index_compression: 1 2024-09-13T14:33:59.586 INFO:journalctl@ceph.mon.c.smithi096.stdout: block_align: 0 2024-09-13T14:33:59.586 INFO:journalctl@ceph.mon.c.smithi096.stdout: max_auto_readahead_size: 262144 2024-09-13T14:33:59.586 INFO:journalctl@ceph.mon.c.smithi096.stdout: prepopulate_block_cache: 0 2024-09-13T14:33:59.586 INFO:journalctl@ceph.mon.c.smithi096.stdout: initial_auto_readahead_size: 8192 2024-09-13T14:33:59.586 INFO:journalctl@ceph.mon.c.smithi096.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T14:33:59.586 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:33:59.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:33:59.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compression: NoCompression 2024-09-13T14:33:59.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:33:59.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:33:59.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:33:59.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.num_levels: 7 2024-09-13T14:33:59.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:33:59.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:33:59.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:33:59.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:33:59.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:33:59.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:33:59.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:33:59.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:33:59.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:33:59.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:33:59.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:33:59.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:33:59.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:33:59.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:33:59.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:33:59.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:33:59.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:33:59.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:33:59.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:33:59.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:33:59.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:33:59.589 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:33:59.589 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:33:59.589 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:33:59.589 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:33:59.589 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:33:59.589 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:33:59.589 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:33:59.589 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:33:59.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:33:59.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:33:59.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:33:59.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:33:59.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:33:59.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:33:59.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:33:59.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:33:59.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:33:59.590 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:33:59.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:33:59.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:33:59.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:33:59.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:33:59.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:33:59.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:33:59.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:33:59.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:33:59.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:33:59.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:33:59.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:33:59.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:33:59.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:33:59.591 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:33:59.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:33:59.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:33:59.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:33:59.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:33:59.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:33:59.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:33:59.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:33:59.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:33:59.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:33:59.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:33:59.592 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:33:59.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:33:59.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.ttl: 2592000 2024-09-13T14:33:59.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:33:59.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:33:59.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:33:59.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:33:59.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:33:59.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:33:59.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:33:59.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:33:59.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:33:59.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:33:59.594 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:33:59.594 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:33:59.594 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:33:59.594 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-13T14:33:59.594 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:33:59.594 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12928, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:33:59.594 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-13T14:33:59.594 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 23a2321b-de2b-4a7a-8e5d-cecc562acfad 2024-09-13T14:33:59.594 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726238039284350, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-13T14:33:59.594 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-13T14:33:59.594 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726238039295667, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 2573895, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12933, "largest_seqno": 14200, "table_properties": {"data_size": 2567837, "index_size": 3669, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1349, "raw_key_size": 11753, "raw_average_key_size": 22, "raw_value_size": 2556555, "raw_average_value_size": 4944, "num_data_blocks": 166, "num_entries": 517, "num_filter_entries": 517, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726238039, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "23a2321b-de2b-4a7a-8e5d-cecc562acfad", "db_session_id": "EZIYF4RM3PBOCQ9UALIW", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:33:59.594 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726238039295812, "job": 1, "event": "recovery_finished"} 2024-09-13T14:33:59.594 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-13T14:33:59.595 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:33:59.595 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:33:59.595 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x561057e7ae00 2024-09-13T14:33:59.595 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: DB pointer 0x561057f7a000 2024-09-13T14:33:59.595 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:33:59.595 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:33:59.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: ** DB Stats ** 2024-09-13T14:33:59.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:33:59.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:33:59.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:33:59.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:33:59.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:33:59.595 INFO:journalctl@ceph.mon.c.smithi096.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:33:59.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:33:59.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:33:59.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: ** Compaction Stats [default] ** 2024-09-13T14:33:59.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:33:59.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:33:59.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: L0 1/0 2.45 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 629.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:33:59.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: L6 1/0 7.41 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T14:33:59.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: Sum 2/0 9.87 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 629.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:33:59.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 629.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:33:59.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:33:59.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: ** Compaction Stats [default] ** 2024-09-13T14:33:59.596 INFO:journalctl@ceph.mon.c.smithi096.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:33:59.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:33:59.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 629.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:33:59.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:33:59.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:33:59.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:33:59.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:33:59.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-13T14:33:59.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:33:59.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:33:59.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:33:59.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:33:59.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: Cumulative compaction: 0.00 GB write, 172.27 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:33:59.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: Interval compaction: 0.00 GB write, 172.27 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:33:59.597 INFO:journalctl@ceph.mon.c.smithi096.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:33:59.598 INFO:journalctl@ceph.mon.c.smithi096.stdout: Block cache BinnedLRUCache@0x561057e4a850#2 capacity: 512.00 MB usage: 5.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-13T14:33:59.598 INFO:journalctl@ceph.mon.c.smithi096.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.41 KB,0.000268221%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-09-13T14:33:59.598 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:33:59.598 INFO:journalctl@ceph.mon.c.smithi096.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:33:59.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: starting mon.c rank 2 at public addrs [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0] at bind addrs [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:33:59.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: mon.c@-1(???) e3 preinit fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:33:59.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: mon.c@-1(???).mds e8 new map 2024-09-13T14:33:59.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: mon.c@-1(???).mds e8 print_map 2024-09-13T14:33:59.598 INFO:journalctl@ceph.mon.c.smithi096.stdout: e8 2024-09-13T14:33:59.598 INFO:journalctl@ceph.mon.c.smithi096.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T14:33:59.598 INFO:journalctl@ceph.mon.c.smithi096.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:33:59.598 INFO:journalctl@ceph.mon.c.smithi096.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:33:59.598 INFO:journalctl@ceph.mon.c.smithi096.stdout: legacy client fscid: 1 2024-09-13T14:33:59.599 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:33:59.599 INFO:journalctl@ceph.mon.c.smithi096.stdout: Filesystem 'foo' (1) 2024-09-13T14:33:59.599 INFO:journalctl@ceph.mon.c.smithi096.stdout: fs_name foo 2024-09-13T14:33:59.599 INFO:journalctl@ceph.mon.c.smithi096.stdout: epoch 7 2024-09-13T14:33:59.599 INFO:journalctl@ceph.mon.c.smithi096.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T14:33:59.599 INFO:journalctl@ceph.mon.c.smithi096.stdout: created 2024-09-13T14:28:10.489785+0000 2024-09-13T14:33:59.599 INFO:journalctl@ceph.mon.c.smithi096.stdout: modified 2024-09-13T14:28:20.810288+0000 2024-09-13T14:33:59.599 INFO:journalctl@ceph.mon.c.smithi096.stdout: tableserver 0 2024-09-13T14:33:59.599 INFO:journalctl@ceph.mon.c.smithi096.stdout: root 0 2024-09-13T14:33:59.599 INFO:journalctl@ceph.mon.c.smithi096.stdout: session_timeout 60 2024-09-13T14:33:59.599 INFO:journalctl@ceph.mon.c.smithi096.stdout: session_autoclose 300 2024-09-13T14:33:59.599 INFO:journalctl@ceph.mon.c.smithi096.stdout: max_file_size 1099511627776 2024-09-13T14:33:59.599 INFO:journalctl@ceph.mon.c.smithi096.stdout: max_xattr_size 65536 2024-09-13T14:33:59.599 INFO:journalctl@ceph.mon.c.smithi096.stdout: required_client_features {} 2024-09-13T14:33:59.600 INFO:journalctl@ceph.mon.c.smithi096.stdout: last_failure 0 2024-09-13T14:33:59.600 INFO:journalctl@ceph.mon.c.smithi096.stdout: last_failure_osd_epoch 0 2024-09-13T14:33:59.600 INFO:journalctl@ceph.mon.c.smithi096.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:33:59.600 INFO:journalctl@ceph.mon.c.smithi096.stdout: max_mds 1 2024-09-13T14:33:59.600 INFO:journalctl@ceph.mon.c.smithi096.stdout: in 0 2024-09-13T14:33:59.600 INFO:journalctl@ceph.mon.c.smithi096.stdout: up {0=24539} 2024-09-13T14:33:59.600 INFO:journalctl@ceph.mon.c.smithi096.stdout: failed 2024-09-13T14:33:59.600 INFO:journalctl@ceph.mon.c.smithi096.stdout: damaged 2024-09-13T14:33:59.600 INFO:journalctl@ceph.mon.c.smithi096.stdout: stopped 2024-09-13T14:33:59.600 INFO:journalctl@ceph.mon.c.smithi096.stdout: data_pools [4] 2024-09-13T14:33:59.600 INFO:journalctl@ceph.mon.c.smithi096.stdout: metadata_pool 3 2024-09-13T14:33:59.600 INFO:journalctl@ceph.mon.c.smithi096.stdout: inline_data disabled 2024-09-13T14:33:59.600 INFO:journalctl@ceph.mon.c.smithi096.stdout: balancer 2024-09-13T14:33:59.600 INFO:journalctl@ceph.mon.c.smithi096.stdout: bal_rank_mask -1 2024-09-13T14:33:59.601 INFO:journalctl@ceph.mon.c.smithi096.stdout: standby_count_wanted 1 2024-09-13T14:33:59.601 INFO:journalctl@ceph.mon.c.smithi096.stdout: qdb_cluster leader: 0 members: 2024-09-13T14:33:59.601 INFO:journalctl@ceph.mon.c.smithi096.stdout: [mds.foo.smithi096.dqiqpz{0:24539} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.96:6834/1494616691,v1:172.21.15.96:6835/1494616691] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:33:59.601 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:33:59.601 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:33:59.601 INFO:journalctl@ceph.mon.c.smithi096.stdout: Standby daemons: 2024-09-13T14:33:59.601 INFO:journalctl@ceph.mon.c.smithi096.stdout: 2024-09-13T14:33:59.601 INFO:journalctl@ceph.mon.c.smithi096.stdout: [mds.foo.smithi204.hbwbot{-1:24484} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.204:6832/291805447,v1:172.21.15.204:6833/291805447] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:33:59.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: mon.c@-1(???).osd e82 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T14:33:59.601 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: mon.c@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:33:59.602 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: mon.c@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:33:59.602 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: mon.c@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:33:59.602 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 ceph-mon[102928]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-09-13T14:33:59.602 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 bash[102908]: babdd691d9b6f9db01e49db29e8b8f6650954acad4c80ce77f47b922f1e27943 2024-09-13T14:33:59.602 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:33:59 smithi096 systemd[1]: Started Ceph mon.c for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:34:00.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:04.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:04 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:04.822 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:34:04] ENGINE Bus STARTING 2024-09-13T14:34:04.822 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:34:04] ENGINE Serving on http://:::9283 2024-09-13T14:34:04.822 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:34:04] ENGINE Bus STARTED 2024-09-13T14:34:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: mon.c calling monitor election 2024-09-13T14:34:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: mon.a calling monitor election 2024-09-13T14:34:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: mon.b calling monitor election 2024-09-13T14:34:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:34:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: monmap epoch 3 2024-09-13T14:34:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:34:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: last_changed 2024-09-13T14:22:50.469402+0000 2024-09-13T14:34:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: created 2024-09-13T14:21:13.478657+0000 2024-09-13T14:34:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: min_mon_release 17 (quincy) 2024-09-13T14:34:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: election_strategy: 1 2024-09-13T14:34:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: 0: [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon.a 2024-09-13T14:34:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: 1: [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] mon.b 2024-09-13T14:34:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: 2: [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0] mon.c 2024-09-13T14:34:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:34:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:34:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: mgrmap e38: y(active, since 90s), standbys: x 2024-09-13T14:34:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T14:34:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:34:05.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[102928]: overall HEALTH_OK 2024-09-13T14:34:05.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: mon.c calling monitor election 2024-09-13T14:34:05.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: mon.a calling monitor election 2024-09-13T14:34:05.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: mon.b calling monitor election 2024-09-13T14:34:05.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:34:05.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: monmap epoch 3 2024-09-13T14:34:05.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:34:05.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: last_changed 2024-09-13T14:22:50.469402+0000 2024-09-13T14:34:05.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: created 2024-09-13T14:21:13.478657+0000 2024-09-13T14:34:05.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: min_mon_release 17 (quincy) 2024-09-13T14:34:05.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: election_strategy: 1 2024-09-13T14:34:05.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: 0: [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon.a 2024-09-13T14:34:05.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: 1: [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] mon.b 2024-09-13T14:34:05.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: 2: [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0] mon.c 2024-09-13T14:34:05.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:34:05.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:34:05.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: mgrmap e38: y(active, since 90s), standbys: x 2024-09-13T14:34:05.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T14:34:05.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:34:05.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:05 smithi096 ceph-mon[98901]: overall HEALTH_OK 2024-09-13T14:34:05.917 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:34:05.968 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:05.969 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: mon.c calling monitor election 2024-09-13T14:34:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: mon.a calling monitor election 2024-09-13T14:34:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: mon.b calling monitor election 2024-09-13T14:34:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:34:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: monmap epoch 3 2024-09-13T14:34:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:34:05.972 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: last_changed 2024-09-13T14:22:50.469402+0000 2024-09-13T14:34:05.972 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: created 2024-09-13T14:21:13.478657+0000 2024-09-13T14:34:05.972 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: min_mon_release 17 (quincy) 2024-09-13T14:34:05.972 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: election_strategy: 1 2024-09-13T14:34:05.972 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: 0: [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon.a 2024-09-13T14:34:05.972 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: 1: [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] mon.b 2024-09-13T14:34:05.972 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: 2: [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0] mon.c 2024-09-13T14:34:05.973 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:34:05.973 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:34:05.973 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: mgrmap e38: y(active, since 90s), standbys: x 2024-09-13T14:34:05.973 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T14:34:05.973 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: Cluster is now healthy 2024-09-13T14:34:05.973 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:05 smithi204 ceph-mon[32546]: overall HEALTH_OK 2024-09-13T14:34:06.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:06 smithi096 ceph-mon[98901]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:06.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:34:06.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:06 smithi096 ceph-mon[98901]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:06.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:06 smithi096 ceph-mon[98901]: from='client.25999 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:06.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:06 smithi096 ceph-mon[102928]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:06.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:34:06.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:06 smithi096 ceph-mon[102928]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:06.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:06 smithi096 ceph-mon[102928]: from='client.25999 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:06.949 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:06 smithi204 ceph-mon[32546]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:06.949 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:06 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:34:06.949 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:06 smithi204 ceph-mon[32546]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:06.949 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:06 smithi204 ceph-mon[32546]: from='client.25999 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:07.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:08.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:34:07] "GET /metrics HTTP/1.1" 200 33846 "" "Prometheus/2.51.0" 2024-09-13T14:34:08.569 INFO:teuthology.orchestra.run.smithi096.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:34:08.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:08 smithi096 ceph-mon[98901]: mgrmap e39: y(active, since 92s), standbys: x 2024-09-13T14:34:08.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:08 smithi096 ceph-mon[98901]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:34:08.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:08 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:08.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:08 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:08.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:08 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:08.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:08 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:08.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:08 smithi096 ceph-mon[102928]: mgrmap e39: y(active, since 92s), standbys: x 2024-09-13T14:34:08.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:08 smithi096 ceph-mon[102928]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:34:08.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:08 smithi096 ceph-mon[102928]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:08.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:08 smithi096 ceph-mon[102928]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:08.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:08 smithi096 ceph-mon[102928]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:08.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:08 smithi096 ceph-mon[102928]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:08.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:08 smithi204 ceph-mon[32546]: mgrmap e39: y(active, since 92s), standbys: x 2024-09-13T14:34:08.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:08 smithi204 ceph-mon[32546]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:34:08.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:08 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:08.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:08 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:08.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:08 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:08.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:08 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:09.288 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:34:09.544 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:09 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:34:09.306+0000 7ff04b218640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-13T14:34:09.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:09 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/295025681' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:34:09.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:09 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/295025681' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:34:09.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:09 smithi204 ceph-mon[32546]: from='client.? 172.21.15.96:0/295025681' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:34:10.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:10.481 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:10.702 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:10 smithi096 ceph-mon[98901]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:34:10.702 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:10 smithi096 ceph-mon[102928]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:34:10.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:10 smithi204 ceph-mon[32546]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:34:12.814 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:12 smithi096 ceph-mon[98901]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:34:12.815 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:12 smithi096 ceph-mon[102928]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:34:12.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:12 smithi204 ceph-mon[32546]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:34:13.640 INFO:teuthology.orchestra.run.smithi096.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:34:13.640 INFO:teuthology.orchestra.run.smithi096.stdout:alertmanager.a smithi096 *:9093,9094 running (2m) 5s ago 7m 18.9M - 0.27.0 11f11916f8cd e04c10a0dca2 2024-09-13T14:34:13.640 INFO:teuthology.orchestra.run.smithi096.stdout:grafana.a smithi204 *:3000 running (2m) 23s ago 6m 79.5M - 9.4.7 954c08fa6188 451523aebd73 2024-09-13T14:34:13.640 INFO:teuthology.orchestra.run.smithi096.stdout:mds.foo.smithi096.dqiqpz smithi096 running (5m) 5s ago 5m 19.7M - 17.2.7-1627-gbbdadfa3 a994b00deb3a ab8d33048bb7 2024-09-13T14:34:13.640 INFO:teuthology.orchestra.run.smithi096.stdout:mds.foo.smithi204.hbwbot smithi204 running (5m) 23s ago 5m 15.5M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 97a3ff635de2 2024-09-13T14:34:13.640 INFO:teuthology.orchestra.run.smithi096.stdout:mgr.x smithi204 *:8443,9283,8765 running (118s) 23s ago 11m 453M - 19.3.0-4697-gd58f7306 fec07016a2b0 04572bea1e6f 2024-09-13T14:34:13.640 INFO:teuthology.orchestra.run.smithi096.stdout:mgr.y smithi096 *:8443,9283,8765 running (2m) 5s ago 12m 507M - 19.3.0-4697-gd58f7306 fec07016a2b0 a4bc51171f6e 2024-09-13T14:34:13.640 INFO:teuthology.orchestra.run.smithi096.stdout:mon.a smithi096 running (105s) 5s ago 12m 43.7M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 97a94128e048 2024-09-13T14:34:13.640 INFO:teuthology.orchestra.run.smithi096.stdout:mon.b smithi204 running (11m) 23s ago 11m 48.6M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a a37b0845979d 2024-09-13T14:34:13.640 INFO:teuthology.orchestra.run.smithi096.stdout:mon.c smithi096 running (14s) 5s ago 11m 20.7M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 babdd691d9b6 2024-09-13T14:34:13.641 INFO:teuthology.orchestra.run.smithi096.stdout:node-exporter.a smithi096 *:9100 running (2m) 5s ago 6m 9.79M - 1.7.0 72c9c2088986 b02b57424615 2024-09-13T14:34:13.641 INFO:teuthology.orchestra.run.smithi096.stdout:node-exporter.b smithi204 *:9100 running (2m) 23s ago 6m 10.0M - 1.7.0 72c9c2088986 c58ad3b83def 2024-09-13T14:34:13.641 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0 smithi096 running (10m) 5s ago 10m 47.8M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a a35fe012b65d 2024-09-13T14:34:13.641 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1 smithi096 running (10m) 5s ago 10m 51.3M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a edbcf0f2ed44 2024-09-13T14:34:13.641 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2 smithi096 running (10m) 5s ago 10m 48.0M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 565505fd2e48 2024-09-13T14:34:13.641 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3 smithi096 running (9m) 5s ago 9m 44.7M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a afd4dcb32d2d 2024-09-13T14:34:13.641 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4 smithi204 running (9m) 23s ago 9m 45.1M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a b48920b2b9e7 2024-09-13T14:34:13.641 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5 smithi204 running (8m) 23s ago 8m 46.7M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 0ed6e334d2ea 2024-09-13T14:34:13.641 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6 smithi204 running (8m) 23s ago 8m 47.8M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8f12f56d751c 2024-09-13T14:34:13.641 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7 smithi204 running (8m) 23s ago 8m 50.6M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a ffd75c8be38a 2024-09-13T14:34:13.641 INFO:teuthology.orchestra.run.smithi096.stdout:prometheus.a smithi204 *:9095 running (2m) 23s ago 7m 44.1M - 2.51.0 1d3b7f56885b c1d744b6a96b 2024-09-13T14:34:14.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:14.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:14.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:34:14.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:34:14.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:14.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:34:14.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:34:14.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:34:14.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-13T14:34:14.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[98901]: Upgrade: It appears safe to stop mon.b 2024-09-13T14:34:14.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[102928]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:14.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[102928]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:14.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:34:14.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:34:14.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[102928]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:14.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:34:14.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:34:14.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:34:14.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-13T14:34:14.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:14 smithi096 ceph-mon[102928]: Upgrade: It appears safe to stop mon.b 2024-09-13T14:34:14.393 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade pause' 2024-09-13T14:34:14.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:14 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:14.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:14.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:14 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:14.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:14 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:14.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:14 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:34:14.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:14 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:34:14.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:14 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:14.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:14 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:34:14.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:14 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:34:14.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:14 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:34:14.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:14 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-13T14:34:14.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:14 smithi204 ceph-mon[32546]: Upgrade: It appears safe to stop mon.b 2024-09-13T14:34:14.642 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:15.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:15 smithi096 ceph-mon[98901]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:34:15.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:15 smithi096 ceph-mon[98901]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:15.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:15 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:15.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:15 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:34:15.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:15 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:34:15.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:15 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:34:15.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:15 smithi096 ceph-mon[102928]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:34:15.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:15 smithi096 ceph-mon[102928]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:15.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:15 smithi096 ceph-mon[102928]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:15.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:15 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:34:15.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:15 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:34:15.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:15 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:34:15.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:15 smithi204 ceph-mon[32546]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:34:15.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:15 smithi204 ceph-mon[32546]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:15.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:15 smithi204 ceph-mon[32546]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:15.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:15 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:34:15.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:15 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:34:15.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:15 smithi204 ceph-mon[32546]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:34:16.215 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:15 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:16.215 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:15 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:16.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:16 smithi096 ceph-mon[98901]: Upgrade: Updating mon.b 2024-09-13T14:34:16.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:16 smithi096 ceph-mon[98901]: Deploying daemon mon.b on smithi204 2024-09-13T14:34:16.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:16 smithi096 ceph-mon[102928]: Upgrade: Updating mon.b 2024-09-13T14:34:16.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:16 smithi096 ceph-mon[102928]: Deploying daemon mon.b on smithi204 2024-09-13T14:34:16.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:16 smithi204 ceph-mon[32546]: Upgrade: Updating mon.b 2024-09-13T14:34:16.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:16 smithi204 ceph-mon[32546]: Deploying daemon mon.b on smithi204 2024-09-13T14:34:16.486 INFO:teuthology.orchestra.run.smithi096.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:34:16.950 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:16 smithi204 systemd[1]: Stopping Ceph mon.b for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:34:16.950 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:16 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b[32532]: 2024-09-13T14:34:16.743+0000 7f1383fc2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:34:16.950 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:16 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b[32532]: 2024-09-13T14:34:16.743+0000 7f1383fc2640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-13T14:34:17.062 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:16 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:34:16] ENGINE Bus STOPPING 2024-09-13T14:34:17.062 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:16 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:34:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:34:17.062 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:16 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:34:16] ENGINE Bus STOPPED 2024-09-13T14:34:17.063 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:16 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:34:16] ENGINE Bus STARTING 2024-09-13T14:34:17.063 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:34:17] ENGINE Serving on http://:::9283 2024-09-13T14:34:17.063 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:14:34:17] ENGINE Bus STARTED 2024-09-13T14:34:17.205 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'sleep 60' 2024-09-13T14:34:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:16 smithi204 podman[86908]: 2024-09-13 14:34:16.989552598 +0000 UTC m=+0.348513923 container died a37b0845979d4d4a3406a0b68c169aed006fdda0ae2ea5c448ebe7f228884387 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731) 2024-09-13T14:34:17.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:16 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:17.459 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:17.628 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:17 smithi204 podman[86908]: 2024-09-13 14:34:17.360486911 +0000 UTC m=+0.719448236 container cleanup a37b0845979d4d4a3406a0b68c169aed006fdda0ae2ea5c448ebe7f228884387 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-13T14:34:17.629 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:17 smithi204 bash[86908]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b 2024-09-13T14:34:17.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:17 smithi204 podman[86919]: 2024-09-13 14:34:17.628498356 +0000 UTC m=+0.630396539 container remove a37b0845979d4d4a3406a0b68c169aed006fdda0ae2ea5c448ebe7f228884387 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731) 2024-09-13T14:34:17.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:17 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-13T14:34:17.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:17 smithi204 systemd[1]: Stopped Ceph mon.b for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:34:17.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:17 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.b.service: Consumed 8.513s CPU time. 2024-09-13T14:34:18.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:34:17] "GET /metrics HTTP/1.1" 200 33846 "" "Prometheus/2.51.0" 2024-09-13T14:34:18.321 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:17 smithi204 systemd[1]: Starting Ceph mon.b for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:34:18.321 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 podman[87052]: 2024-09-13 14:34:18.096544877 +0000 UTC m=+0.016135507 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:34:18.321 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 podman[87052]: 2024-09-13 14:34:18.213598155 +0000 UTC m=+0.133188778 container create 4cc02299264a7d91ea242d2c1a08e54ed47cae79108e57893ff26235ae235224 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:34:18.485 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 podman[87052]: 2024-09-13 14:34:18.458012074 +0000 UTC m=+0.377602705 container init 4cc02299264a7d91ea242d2c1a08e54ed47cae79108e57893ff26235ae235224 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-13T14:34:18.485 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 podman[87052]: 2024-09-13 14:34:18.461668047 +0000 UTC m=+0.381258668 container start 4cc02299264a7d91ea242d2c1a08e54ed47cae79108e57893ff26235ae235224 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:34:18.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:34:18.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev), process ceph-mon, pid 2 2024-09-13T14:34:18.486 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: pidfile_write: ignore empty --pid-file 2024-09-13T14:34:18.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[98901]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:34:18.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[98901]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:18.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[98901]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:34:18.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[98901]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:18.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:34:18.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:34:18.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:34:18.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:34:18.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[102928]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:34:18.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[102928]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:18.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[102928]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:34:18.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[102928]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:18.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:34:18.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:34:18.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:34:18.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:18 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:34:18.593 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: load: jerasure load: lrc 2024-09-13T14:34:18.593 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: RocksDB version: 7.9.2 2024-09-13T14:34:18.593 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Git sha 0 2024-09-13T14:34:18.593 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Compile date 2024-09-12 20:46:14 2024-09-13T14:34:18.593 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: DB SUMMARY 2024-09-13T14:34:18.593 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: DB Session ID: RTWFCBNKU6KY7OUSPUM1 2024-09-13T14:34:18.593 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:34:18.593 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:34:18.594 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1085 Bytes 2024-09-13T14:34:18.594 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-09-13T14:34:18.594 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 4870512 ; 2024-09-13T14:34:18.594 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:34:18.594 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:34:18.594 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:34:18.594 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:34:18.594 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:34:18.594 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:34:18.595 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.env: 0x5568edaef160 2024-09-13T14:34:18.595 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:34:18.595 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.info_log: 0x5568efd83880 2024-09-13T14:34:18.595 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:34:18.595 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.statistics: (nil) 2024-09-13T14:34:18.595 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.use_fsync: 0 2024-09-13T14:34:18.595 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:34:18.595 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:34:18.595 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:34:18.595 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:34:18.595 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:34:18.595 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:34:18.595 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:34:18.596 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:34:18.596 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:34:18.596 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:34:18.596 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:34:18.596 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.db_log_dir: 2024-09-13T14:34:18.596 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.wal_dir: 2024-09-13T14:34:18.596 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:34:18.596 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:34:18.596 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:34:18.596 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:34:18.598 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:34:18.598 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:34:18.598 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:34:18.598 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:34:18.598 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.write_buffer_manager: 0x5568efd86960 2024-09-13T14:34:18.598 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:34:18.598 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:34:18.598 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:34:18.599 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:34:18.599 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:34:18.599 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:34:18.599 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:34:18.599 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:34:18.599 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.unordered_write: 0 2024-09-13T14:34:18.599 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:34:18.599 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:34:18.599 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:34:18.599 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:34:18.599 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.row_cache: None 2024-09-13T14:34:18.599 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.wal_filter: None 2024-09-13T14:34:18.599 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:34:18.599 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:34:18.600 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:34:18.600 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:34:18.600 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.wal_compression: 0 2024-09-13T14:34:18.600 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:34:18.600 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:34:18.600 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:34:18.600 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:34:18.600 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:34:18.600 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:34:18.600 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:34:18.600 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:34:18.600 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:34:18.600 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:34:18.601 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:34:18.601 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:34:18.601 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:34:18.601 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:34:18.601 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:34:18.601 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:34:18.601 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:34:18.601 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:34:18.601 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:34:18.601 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:34:18.601 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:34:18.601 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:34:18.601 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:34:18.601 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_open_files: -1 2024-09-13T14:34:18.602 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:34:18.602 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:34:18.602 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:34:18.602 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:34:18.602 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:34:18.602 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Compression algorithms supported: 2024-09-13T14:34:18.602 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: kZSTD supported: 0 2024-09-13T14:34:18.602 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:34:18.602 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:34:18.602 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:34:18.602 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:34:18.602 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:34:18.602 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:34:18.602 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:34:18.603 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:34:18.603 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:34:18.603 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-09-13T14:34:18.603 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:34:18.603 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:34:18.603 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.merge_operator: 2024-09-13T14:34:18.603 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compaction_filter: None 2024-09-13T14:34:18.603 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:34:18.603 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:34:18.603 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:34:18.603 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:34:18.603 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5568efd834e0) 2024-09-13T14:34:18.604 INFO:journalctl@ceph.mon.b.smithi204.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:34:18.604 INFO:journalctl@ceph.mon.b.smithi204.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:34:18.604 INFO:journalctl@ceph.mon.b.smithi204.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:34:18.604 INFO:journalctl@ceph.mon.b.smithi204.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:34:18.604 INFO:journalctl@ceph.mon.b.smithi204.stdout: index_type: 0 2024-09-13T14:34:18.604 INFO:journalctl@ceph.mon.b.smithi204.stdout: data_block_index_type: 0 2024-09-13T14:34:18.604 INFO:journalctl@ceph.mon.b.smithi204.stdout: index_shortening: 1 2024-09-13T14:34:18.604 INFO:journalctl@ceph.mon.b.smithi204.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:34:18.605 INFO:journalctl@ceph.mon.b.smithi204.stdout: checksum: 4 2024-09-13T14:34:18.605 INFO:journalctl@ceph.mon.b.smithi204.stdout: no_block_cache: 0 2024-09-13T14:34:18.605 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_cache: 0x5568efd76850 2024-09-13T14:34:18.605 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:34:18.605 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_cache_options: 2024-09-13T14:34:18.605 INFO:journalctl@ceph.mon.b.smithi204.stdout: capacity : 536870912 2024-09-13T14:34:18.605 INFO:journalctl@ceph.mon.b.smithi204.stdout: num_shard_bits : 4 2024-09-13T14:34:18.605 INFO:journalctl@ceph.mon.b.smithi204.stdout: strict_capacity_limit : 0 2024-09-13T14:34:18.605 INFO:journalctl@ceph.mon.b.smithi204.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:34:18.605 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_cache_compressed: (nil) 2024-09-13T14:34:18.605 INFO:journalctl@ceph.mon.b.smithi204.stdout: persistent_cache: (nil) 2024-09-13T14:34:18.605 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_size: 4096 2024-09-13T14:34:18.605 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_size_deviation: 10 2024-09-13T14:34:18.606 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_restart_interval: 16 2024-09-13T14:34:18.606 INFO:journalctl@ceph.mon.b.smithi204.stdout: index_block_restart_interval: 1 2024-09-13T14:34:18.606 INFO:journalctl@ceph.mon.b.smithi204.stdout: metadata_block_size: 4096 2024-09-13T14:34:18.606 INFO:journalctl@ceph.mon.b.smithi204.stdout: partition_filters: 0 2024-09-13T14:34:18.606 INFO:journalctl@ceph.mon.b.smithi204.stdout: use_delta_encoding: 1 2024-09-13T14:34:18.606 INFO:journalctl@ceph.mon.b.smithi204.stdout: filter_policy: bloomfilter 2024-09-13T14:34:18.606 INFO:journalctl@ceph.mon.b.smithi204.stdout: whole_key_filtering: 1 2024-09-13T14:34:18.606 INFO:journalctl@ceph.mon.b.smithi204.stdout: verify_compression: 0 2024-09-13T14:34:18.606 INFO:journalctl@ceph.mon.b.smithi204.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:34:18.606 INFO:journalctl@ceph.mon.b.smithi204.stdout: format_version: 5 2024-09-13T14:34:18.606 INFO:journalctl@ceph.mon.b.smithi204.stdout: enable_index_compression: 1 2024-09-13T14:34:18.606 INFO:journalctl@ceph.mon.b.smithi204.stdout: block_align: 0 2024-09-13T14:34:18.607 INFO:journalctl@ceph.mon.b.smithi204.stdout: max_auto_readahead_size: 262144 2024-09-13T14:34:18.607 INFO:journalctl@ceph.mon.b.smithi204.stdout: prepopulate_block_cache: 0 2024-09-13T14:34:18.607 INFO:journalctl@ceph.mon.b.smithi204.stdout: initial_auto_readahead_size: 8192 2024-09-13T14:34:18.607 INFO:journalctl@ceph.mon.b.smithi204.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T14:34:18.607 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:34:18.607 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:34:18.607 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compression: NoCompression 2024-09-13T14:34:18.607 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:34:18.607 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:34:18.607 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:34:18.607 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.num_levels: 7 2024-09-13T14:34:18.607 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:34:18.607 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:34:18.607 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:34:18.608 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:34:18.608 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:34:18.608 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:34:18.608 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:34:18.608 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:34:18.608 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:34:18.608 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:34:18.608 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:34:18.608 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:34:18.608 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:34:18.608 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:34:18.608 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:34:18.608 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:34:18.608 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:34:18.609 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:34:18.609 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:34:18.609 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:34:18.609 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:34:18.609 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:34:18.609 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:34:18.609 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:34:18.609 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:34:18.609 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:34:18.609 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:34:18.609 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:34:18.609 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:34:18.609 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:34:18.610 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:34:18.610 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:34:18.610 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:34:18.610 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:34:18.610 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:34:18.610 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:34:18.610 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:34:18.610 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:34:18.610 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:34:18.610 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:34:18.610 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:34:18.610 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:34:18.610 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:34:18.610 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:34:18.610 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:34:18.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:34:18.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:34:18.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:34:18.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:34:18.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:34:18.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:34:18.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:34:18.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:34:18.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:34:18.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:34:18.611 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:34:18.612 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:34:18.612 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:34:18.612 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:34:18.612 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:34:18.612 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:34:18.612 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:34:18.612 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:34:18.612 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:34:18.612 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:34:18.612 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.ttl: 2592000 2024-09-13T14:34:18.612 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:34:18.612 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:34:18.612 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:34:18.613 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:34:18.613 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:34:18.613 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:34:18.613 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:34:18.613 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:34:18.613 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:34:18.613 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:34:18.613 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:34:18.613 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:34:18.613 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:34:18.613 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-13T14:34:18.613 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:34:18.613 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13007, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:34:18.613 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-13T14:34:18.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 751cd824-e39f-4ea8-8f14-a7dccdbb0b52 2024-09-13T14:34:18.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726238058488199, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-13T14:34:18.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-13T14:34:18.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726238058499024, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 3063304, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13012, "largest_seqno": 14608, "table_properties": {"data_size": 3056198, "index_size": 4461, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1605, "raw_key_size": 14465, "raw_average_key_size": 22, "raw_value_size": 3042445, "raw_average_value_size": 4783, "num_data_blocks": 202, "num_entries": 636, "num_filter_entries": 636, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726238058, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "751cd824-e39f-4ea8-8f14-a7dccdbb0b52", "db_session_id": "RTWFCBNKU6KY7OUSPUM1", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:34:18.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726238058499149, "job": 1, "event": "recovery_finished"} 2024-09-13T14:34:18.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-13T14:34:18.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:34:18.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:34:18.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5568efda6e00 2024-09-13T14:34:18.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: DB pointer 0x5568efea6000 2024-09-13T14:34:18.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:34:18.614 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:34:18.614 INFO:journalctl@ceph.mon.b.smithi204.stdout: ** DB Stats ** 2024-09-13T14:34:18.614 INFO:journalctl@ceph.mon.b.smithi204.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:34:18.615 INFO:journalctl@ceph.mon.b.smithi204.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:34:18.615 INFO:journalctl@ceph.mon.b.smithi204.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:34:18.615 INFO:journalctl@ceph.mon.b.smithi204.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:34:18.615 INFO:journalctl@ceph.mon.b.smithi204.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:34:18.615 INFO:journalctl@ceph.mon.b.smithi204.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:34:18.615 INFO:journalctl@ceph.mon.b.smithi204.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:34:18.615 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:34:18.615 INFO:journalctl@ceph.mon.b.smithi204.stdout: ** Compaction Stats [default] ** 2024-09-13T14:34:18.615 INFO:journalctl@ceph.mon.b.smithi204.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:34:18.615 INFO:journalctl@ceph.mon.b.smithi204.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:34:18.615 INFO:journalctl@ceph.mon.b.smithi204.stdout: L0 1/0 2.92 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 698.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:34:18.616 INFO:journalctl@ceph.mon.b.smithi204.stdout: L6 1/0 7.41 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T14:34:18.616 INFO:journalctl@ceph.mon.b.smithi204.stdout: Sum 2/0 10.34 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 698.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:34:18.616 INFO:journalctl@ceph.mon.b.smithi204.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 698.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:34:18.616 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:34:18.616 INFO:journalctl@ceph.mon.b.smithi204.stdout: ** Compaction Stats [default] ** 2024-09-13T14:34:18.616 INFO:journalctl@ceph.mon.b.smithi204.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:34:18.616 INFO:journalctl@ceph.mon.b.smithi204.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:34:18.616 INFO:journalctl@ceph.mon.b.smithi204.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 698.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:34:18.616 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:34:18.616 INFO:journalctl@ceph.mon.b.smithi204.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:34:18.616 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:34:18.616 INFO:journalctl@ceph.mon.b.smithi204.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:34:18.617 INFO:journalctl@ceph.mon.b.smithi204.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-09-13T14:34:18.617 INFO:journalctl@ceph.mon.b.smithi204.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:34:18.617 INFO:journalctl@ceph.mon.b.smithi204.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:34:18.617 INFO:journalctl@ceph.mon.b.smithi204.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:34:18.617 INFO:journalctl@ceph.mon.b.smithi204.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:34:18.617 INFO:journalctl@ceph.mon.b.smithi204.stdout: Cumulative compaction: 0.00 GB write, 172.07 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:34:18.617 INFO:journalctl@ceph.mon.b.smithi204.stdout: Interval compaction: 0.00 GB write, 172.07 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:34:18.617 INFO:journalctl@ceph.mon.b.smithi204.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:34:18.617 INFO:journalctl@ceph.mon.b.smithi204.stdout: Block cache BinnedLRUCache@0x5568efd76850#2 capacity: 512.00 MB usage: 6.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-13T14:34:18.617 INFO:journalctl@ceph.mon.b.smithi204.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-09-13T14:34:18.617 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:34:18.617 INFO:journalctl@ceph.mon.b.smithi204.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:34:18.617 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: starting mon.b rank 1 at public addrs [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] at bind addrs [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:34:18.618 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: mon.b@-1(???) e3 preinit fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:34:18.618 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: mon.b@-1(???).mds e8 new map 2024-09-13T14:34:18.618 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: mon.b@-1(???).mds e8 print_map 2024-09-13T14:34:18.618 INFO:journalctl@ceph.mon.b.smithi204.stdout: e8 2024-09-13T14:34:18.618 INFO:journalctl@ceph.mon.b.smithi204.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T14:34:18.618 INFO:journalctl@ceph.mon.b.smithi204.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:34:18.618 INFO:journalctl@ceph.mon.b.smithi204.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:34:18.618 INFO:journalctl@ceph.mon.b.smithi204.stdout: legacy client fscid: 1 2024-09-13T14:34:18.618 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:34:18.618 INFO:journalctl@ceph.mon.b.smithi204.stdout: Filesystem 'foo' (1) 2024-09-13T14:34:18.618 INFO:journalctl@ceph.mon.b.smithi204.stdout: fs_name foo 2024-09-13T14:34:18.619 INFO:journalctl@ceph.mon.b.smithi204.stdout: epoch 7 2024-09-13T14:34:18.619 INFO:journalctl@ceph.mon.b.smithi204.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T14:34:18.619 INFO:journalctl@ceph.mon.b.smithi204.stdout: created 2024-09-13T14:28:10.489785+0000 2024-09-13T14:34:18.619 INFO:journalctl@ceph.mon.b.smithi204.stdout: modified 2024-09-13T14:28:20.810288+0000 2024-09-13T14:34:18.619 INFO:journalctl@ceph.mon.b.smithi204.stdout: tableserver 0 2024-09-13T14:34:18.619 INFO:journalctl@ceph.mon.b.smithi204.stdout: root 0 2024-09-13T14:34:18.619 INFO:journalctl@ceph.mon.b.smithi204.stdout: session_timeout 60 2024-09-13T14:34:18.619 INFO:journalctl@ceph.mon.b.smithi204.stdout: session_autoclose 300 2024-09-13T14:34:18.619 INFO:journalctl@ceph.mon.b.smithi204.stdout: max_file_size 1099511627776 2024-09-13T14:34:18.619 INFO:journalctl@ceph.mon.b.smithi204.stdout: max_xattr_size 65536 2024-09-13T14:34:18.619 INFO:journalctl@ceph.mon.b.smithi204.stdout: required_client_features {} 2024-09-13T14:34:18.619 INFO:journalctl@ceph.mon.b.smithi204.stdout: last_failure 0 2024-09-13T14:34:18.619 INFO:journalctl@ceph.mon.b.smithi204.stdout: last_failure_osd_epoch 0 2024-09-13T14:34:18.620 INFO:journalctl@ceph.mon.b.smithi204.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:34:18.620 INFO:journalctl@ceph.mon.b.smithi204.stdout: max_mds 1 2024-09-13T14:34:18.620 INFO:journalctl@ceph.mon.b.smithi204.stdout: in 0 2024-09-13T14:34:18.620 INFO:journalctl@ceph.mon.b.smithi204.stdout: up {0=24539} 2024-09-13T14:34:18.620 INFO:journalctl@ceph.mon.b.smithi204.stdout: failed 2024-09-13T14:34:18.620 INFO:journalctl@ceph.mon.b.smithi204.stdout: damaged 2024-09-13T14:34:18.620 INFO:journalctl@ceph.mon.b.smithi204.stdout: stopped 2024-09-13T14:34:18.620 INFO:journalctl@ceph.mon.b.smithi204.stdout: data_pools [4] 2024-09-13T14:34:18.621 INFO:journalctl@ceph.mon.b.smithi204.stdout: metadata_pool 3 2024-09-13T14:34:18.621 INFO:journalctl@ceph.mon.b.smithi204.stdout: inline_data disabled 2024-09-13T14:34:18.621 INFO:journalctl@ceph.mon.b.smithi204.stdout: balancer 2024-09-13T14:34:18.621 INFO:journalctl@ceph.mon.b.smithi204.stdout: bal_rank_mask -1 2024-09-13T14:34:18.621 INFO:journalctl@ceph.mon.b.smithi204.stdout: standby_count_wanted 1 2024-09-13T14:34:18.621 INFO:journalctl@ceph.mon.b.smithi204.stdout: qdb_cluster leader: 0 members: 2024-09-13T14:34:18.621 INFO:journalctl@ceph.mon.b.smithi204.stdout: [mds.foo.smithi096.dqiqpz{0:24539} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.96:6834/1494616691,v1:172.21.15.96:6835/1494616691] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:34:18.621 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:34:18.621 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:34:18.621 INFO:journalctl@ceph.mon.b.smithi204.stdout: Standby daemons: 2024-09-13T14:34:18.621 INFO:journalctl@ceph.mon.b.smithi204.stdout: 2024-09-13T14:34:18.621 INFO:journalctl@ceph.mon.b.smithi204.stdout: [mds.foo.smithi204.hbwbot{-1:24484} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.204:6832/291805447,v1:172.21.15.204:6833/291805447] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:34:18.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: mon.b@-1(???).osd e82 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T14:34:18.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:34:18.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:34:18.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:34:18.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-09-13T14:34:18.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:34:18.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:18.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:34:18.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: from='mgr.26588 ' entity='mgr.y' 2024-09-13T14:34:18.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:34:18.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:34:18.623 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:34:18.623 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:34:18.623 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 bash[87052]: 4cc02299264a7d91ea242d2c1a08e54ed47cae79108e57893ff26235ae235224 2024-09-13T14:34:18.623 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:18 smithi204 systemd[1]: Started Ceph mon.b for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:34:19.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:34:19.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:34:19.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: mgrmap e39: y(active, since 103s), standbys: x 2024-09-13T14:34:19.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: overall HEALTH_OK 2024-09-13T14:34:19.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: mon.b calling monitor election 2024-09-13T14:34:19.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: mon.a calling monitor election 2024-09-13T14:34:19.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: mon.c calling monitor election 2024-09-13T14:34:19.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:34:19.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:34:19.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:34:19.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:34:19.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: monmap epoch 4 2024-09-13T14:34:19.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:34:19.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: last_changed 2024-09-13T14:34:18.526847+0000 2024-09-13T14:34:19.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: created 2024-09-13T14:21:13.478657+0000 2024-09-13T14:34:19.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: min_mon_release 19 (squid) 2024-09-13T14:34:19.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: election_strategy: 1 2024-09-13T14:34:19.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: 0: [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon.a 2024-09-13T14:34:19.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: 1: [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] mon.b 2024-09-13T14:34:19.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: 2: [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0] mon.c 2024-09-13T14:34:19.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:34:19.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:34:19.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: mgrmap e39: y(active, since 103s), standbys: x 2024-09-13T14:34:19.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: overall HEALTH_OK 2024-09-13T14:34:19.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:19.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:19.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:19.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:34:19.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:34:19.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: mgrmap e39: y(active, since 103s), standbys: x 2024-09-13T14:34:19.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: overall HEALTH_OK 2024-09-13T14:34:19.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: mon.b calling monitor election 2024-09-13T14:34:19.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: mon.a calling monitor election 2024-09-13T14:34:19.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: mon.c calling monitor election 2024-09-13T14:34:19.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:34:19.826 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:34:19.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:34:19.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:34:19.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: monmap epoch 4 2024-09-13T14:34:19.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:34:19.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: last_changed 2024-09-13T14:34:18.526847+0000 2024-09-13T14:34:19.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: created 2024-09-13T14:21:13.478657+0000 2024-09-13T14:34:19.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: min_mon_release 19 (squid) 2024-09-13T14:34:19.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: election_strategy: 1 2024-09-13T14:34:19.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: 0: [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon.a 2024-09-13T14:34:19.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: 1: [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] mon.b 2024-09-13T14:34:19.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: 2: [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0] mon.c 2024-09-13T14:34:19.828 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:34:19.828 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:34:19.828 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: mgrmap e39: y(active, since 103s), standbys: x 2024-09-13T14:34:19.828 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: overall HEALTH_OK 2024-09-13T14:34:19.828 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:19.828 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:19.828 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:19 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:19.960 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:34:19.960 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:34:19.960 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: mgrmap e39: y(active, since 103s), standbys: x 2024-09-13T14:34:19.960 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: overall HEALTH_OK 2024-09-13T14:34:19.961 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: mon.b calling monitor election 2024-09-13T14:34:19.961 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: mon.a calling monitor election 2024-09-13T14:34:19.961 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: mon.c calling monitor election 2024-09-13T14:34:19.961 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:34:19.961 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:34:19.961 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:34:19.962 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:34:19.962 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: monmap epoch 4 2024-09-13T14:34:19.962 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:34:19.962 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: last_changed 2024-09-13T14:34:18.526847+0000 2024-09-13T14:34:19.962 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: created 2024-09-13T14:21:13.478657+0000 2024-09-13T14:34:19.962 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: min_mon_release 19 (squid) 2024-09-13T14:34:19.963 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: election_strategy: 1 2024-09-13T14:34:19.963 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: 0: [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon.a 2024-09-13T14:34:19.963 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: 1: [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] mon.b 2024-09-13T14:34:19.963 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: 2: [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0] mon.c 2024-09-13T14:34:19.963 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: fsmap foo:1 {0=foo.smithi096.dqiqpz=up:active} 1 up:standby 2024-09-13T14:34:19.963 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:34:19.964 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: mgrmap e39: y(active, since 103s), standbys: x 2024-09-13T14:34:19.964 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: overall HEALTH_OK 2024-09-13T14:34:19.964 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:19.964 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:19.964 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:19 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:20.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:20.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:20 smithi096 ceph-mon[98901]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:20.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:20 smithi096 ceph-mon[102928]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:20.895 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:20 smithi204 ceph-mon[87071]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:21.811 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:21.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:21.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:22.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:22 smithi204 ceph-mon[87071]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:22.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:22 smithi096 ceph-mon[98901]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:22.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:22 smithi096 ceph-mon[102928]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:23.796 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:23 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:34:23.513+0000 7ff04b218640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-13T14:34:24.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:23 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:24.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:23 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:24.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:23 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:24.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:23 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:24.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:23 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:24.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:23 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:24.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:23 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:24.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:23 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:24.148 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:23 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:24.148 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:23 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:24.148 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:23 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:24.148 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:23 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:24.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:24 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:25.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:24 smithi096 ceph-mon[98901]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:25.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:24 smithi096 ceph-mon[102928]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:25.087 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:24 smithi204 ceph-mon[87071]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:26.135 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:25 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:26.135 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:25 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:27.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[98901]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:27.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:27.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:27.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:27.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:27.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:27.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:27.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:27.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:27.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[102928]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:27.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:27.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:27.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:27.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:27.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:27.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:27.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:27.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:26 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:27.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:26 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:27.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:26 smithi204 ceph-mon[87071]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:27.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:26 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:27.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:26 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:27.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:26 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:27.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:26 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:27.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:26 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:27.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:26 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:27.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:26 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:27.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:26 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:28.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:27 smithi096 ceph-mon[98901]: Reconfiguring mon.a (monmap changed)... 2024-09-13T14:34:28.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:27 smithi096 ceph-mon[98901]: Reconfiguring daemon mon.a on smithi096 2024-09-13T14:34:28.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:34:27] "GET /metrics HTTP/1.1" 200 33767 "" "Prometheus/2.51.0" 2024-09-13T14:34:28.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:27 smithi096 ceph-mon[102928]: Reconfiguring mon.a (monmap changed)... 2024-09-13T14:34:28.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:27 smithi096 ceph-mon[102928]: Reconfiguring daemon mon.a on smithi096 2024-09-13T14:34:28.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:27 smithi204 ceph-mon[87071]: Reconfiguring mon.a (monmap changed)... 2024-09-13T14:34:28.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:27 smithi204 ceph-mon[87071]: Reconfiguring daemon mon.a on smithi096 2024-09-13T14:34:29.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:29 smithi204 ceph-mon[87071]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:29 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:29 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:29 smithi204 ceph-mon[87071]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:34:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:29 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:29 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:29 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:29 smithi204 ceph-mon[87071]: Reconfiguring daemon mgr.y on smithi096 2024-09-13T14:34:29.571 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[102928]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:29.571 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:29.571 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:29.571 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[102928]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:34:29.571 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:29.571 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:29.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:29.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[102928]: Reconfiguring daemon mgr.y on smithi096 2024-09-13T14:34:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[98901]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[98901]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:34:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:29 smithi096 ceph-mon[98901]: Reconfiguring daemon mgr.y on smithi096 2024-09-13T14:34:30.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:30.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:30 smithi204 ceph-mon[87071]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:30.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:30 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:30.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:30 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:30.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:30 smithi204 ceph-mon[87071]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:34:30.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:30 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:30.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:30 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:30.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:30 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:30.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:30 smithi204 ceph-mon[87071]: Reconfiguring daemon mon.c on smithi096 2024-09-13T14:34:31.028 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[98901]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:31.028 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:31.028 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:31.028 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[98901]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:34:31.029 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:31.029 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:31.029 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:31.029 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[98901]: Reconfiguring daemon mon.c on smithi096 2024-09-13T14:34:31.029 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[102928]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:31.029 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:31.029 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:31.029 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[102928]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:34:31.029 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:31.029 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:31.029 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:31.030 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:30 smithi096 ceph-mon[102928]: Reconfiguring daemon mon.c on smithi096 2024-09-13T14:34:32.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:32 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:32.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:32 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:32.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:32 smithi096 ceph-mon[98901]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:34:32.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:32 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:34:32.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:32 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:32.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:32 smithi096 ceph-mon[98901]: Reconfiguring daemon osd.0 on smithi096 2024-09-13T14:34:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:32 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:32 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:32 smithi096 ceph-mon[102928]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:34:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:32 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:34:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:32 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:32 smithi096 ceph-mon[102928]: Reconfiguring daemon osd.0 on smithi096 2024-09-13T14:34:32.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:32 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:32.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:32 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:32.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:32 smithi204 ceph-mon[87071]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:34:32.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:32 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:34:32.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:32 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:32.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:32 smithi204 ceph-mon[87071]: Reconfiguring daemon osd.0 on smithi096 2024-09-13T14:34:33.122 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:33 smithi096 ceph-mon[98901]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:33.123 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:33.123 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:33.123 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:34:33.123 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:33.123 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:33 smithi096 ceph-mon[102928]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:33.123 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:33.123 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:33.123 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:34:33.123 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:33 smithi204 ceph-mon[87071]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:34:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:34.374 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:34 smithi096 ceph-mon[98901]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:34:34.374 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:34 smithi096 ceph-mon[98901]: Reconfiguring daemon osd.1 on smithi096 2024-09-13T14:34:34.374 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:34 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:34.374 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:34 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:34.374 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:34 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:34:34.374 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:34 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:34.375 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:34 smithi096 ceph-mon[102928]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:34:34.375 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:34 smithi096 ceph-mon[102928]: Reconfiguring daemon osd.1 on smithi096 2024-09-13T14:34:34.375 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:34 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:34.375 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:34 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:34.375 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:34 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:34:34.375 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:34 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:34.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:34 smithi204 ceph-mon[87071]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:34:34.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:34 smithi204 ceph-mon[87071]: Reconfiguring daemon osd.1 on smithi096 2024-09-13T14:34:34.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:34 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:34.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:34 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:34.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:34 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:34:34.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:34 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:34.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:34 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:35.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:35 smithi204 ceph-mon[87071]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:35.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:35 smithi204 ceph-mon[87071]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:34:35.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:35 smithi204 ceph-mon[87071]: Reconfiguring daemon osd.2 on smithi096 2024-09-13T14:34:35.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:35 smithi096 ceph-mon[98901]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:35.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:35 smithi096 ceph-mon[98901]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:34:35.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:35 smithi096 ceph-mon[98901]: Reconfiguring daemon osd.2 on smithi096 2024-09-13T14:34:35.484 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:35 smithi096 ceph-mon[102928]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:35.484 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:35 smithi096 ceph-mon[102928]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:34:35.484 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:35 smithi096 ceph-mon[102928]: Reconfiguring daemon osd.2 on smithi096 2024-09-13T14:34:36.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:35 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:36.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:35 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:36.506 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:36.506 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:36.506 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:36 smithi096 ceph-mon[98901]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:34:36.506 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:34:36.506 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:36.506 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:36 smithi096 ceph-mon[98901]: Reconfiguring daemon osd.3 on smithi096 2024-09-13T14:34:36.506 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:36.507 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:36.507 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:36.507 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:36 smithi096 ceph-mon[102928]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:34:36.507 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:34:36.507 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:36.507 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:36 smithi096 ceph-mon[102928]: Reconfiguring daemon osd.3 on smithi096 2024-09-13T14:34:36.507 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:36.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:36.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:36 smithi204 ceph-mon[87071]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:34:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:34:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:36 smithi204 ceph-mon[87071]: Reconfiguring daemon osd.3 on smithi096 2024-09-13T14:34:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:37.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:36 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:37.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:37 smithi096 ceph-mon[98901]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:37.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:37 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:37.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:37 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:37.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:37 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi096.dqiqpz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:34:37.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:37 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:37.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:37 smithi096 ceph-mon[102928]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:37.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:37 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:37.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:37 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:37.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:37 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi096.dqiqpz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:34:37.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:37 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:37.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:37 smithi204 ceph-mon[87071]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:37.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:37 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:37.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:37 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:37.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:37 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi096.dqiqpz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:34:37.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:37 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:38.235 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:34:37] "GET /metrics HTTP/1.1" 200 33838 "" "Prometheus/2.51.0" 2024-09-13T14:34:38.498 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:38 smithi204 ceph-mon[87071]: Reconfiguring mds.foo.smithi096.dqiqpz (monmap changed)... 2024-09-13T14:34:38.498 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:38 smithi204 ceph-mon[87071]: Reconfiguring daemon mds.foo.smithi096.dqiqpz on smithi096 2024-09-13T14:34:38.498 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:38.498 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:38.498 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:38.498 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:38.498 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:38.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:38 smithi096 ceph-mon[102928]: Reconfiguring mds.foo.smithi096.dqiqpz (monmap changed)... 2024-09-13T14:34:38.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:38 smithi096 ceph-mon[102928]: Reconfiguring daemon mds.foo.smithi096.dqiqpz on smithi096 2024-09-13T14:34:38.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:38.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:38.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:38.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:38.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:38.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:38 smithi096 ceph-mon[98901]: Reconfiguring mds.foo.smithi096.dqiqpz (monmap changed)... 2024-09-13T14:34:38.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:38 smithi096 ceph-mon[98901]: Reconfiguring daemon mds.foo.smithi096.dqiqpz on smithi096 2024-09-13T14:34:38.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:38.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:38.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:38.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:38.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:39.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:39 smithi204 ceph-mon[87071]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:39.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:39 smithi204 ceph-mon[87071]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:34:39.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:39 smithi204 ceph-mon[87071]: Reconfiguring daemon mon.b on smithi204 2024-09-13T14:34:39.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:39 smithi096 ceph-mon[102928]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:39.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:39 smithi096 ceph-mon[102928]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:34:39.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:39 smithi096 ceph-mon[102928]: Reconfiguring daemon mon.b on smithi204 2024-09-13T14:34:39.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:39 smithi096 ceph-mon[98901]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:39.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:39 smithi096 ceph-mon[98901]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:34:39.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:39 smithi096 ceph-mon[98901]: Reconfiguring daemon mon.b on smithi204 2024-09-13T14:34:40.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:40 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:40 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:40 smithi204 ceph-mon[87071]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:34:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:40 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:40 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:40 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:40 smithi204 ceph-mon[87071]: Reconfiguring daemon mgr.x on smithi204 2024-09-13T14:34:40.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:40 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:40.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:40 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:40.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:40 smithi096 ceph-mon[102928]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:34:40.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:40 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:40.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:40 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:40.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:40 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:40.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:40 smithi096 ceph-mon[102928]: Reconfiguring daemon mgr.x on smithi204 2024-09-13T14:34:40.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:40 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:40.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:40 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:40.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:40 smithi096 ceph-mon[98901]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:34:40.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:40 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:40.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:40 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:40.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:40 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:40.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:40 smithi096 ceph-mon[98901]: Reconfiguring daemon mgr.x on smithi204 2024-09-13T14:34:41.371 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:41 smithi204 ceph-mon[87071]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:41.371 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:41.371 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:41.371 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:34:41.371 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:41.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:41 smithi096 ceph-mon[98901]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:41.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:41.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:41.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:34:41.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:41.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:41 smithi096 ceph-mon[102928]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:41.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:41.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:41.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:34:41.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:42.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:42 smithi096 ceph-mon[98901]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:34:42.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:42 smithi096 ceph-mon[98901]: Reconfiguring daemon osd.4 on smithi204 2024-09-13T14:34:42.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:42 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:42.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:42 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:42.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:42 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:34:42.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:42 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:42.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:42 smithi096 ceph-mon[102928]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:34:42.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:42 smithi096 ceph-mon[102928]: Reconfiguring daemon osd.4 on smithi204 2024-09-13T14:34:42.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:42 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:42.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:42 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:42.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:42 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:34:42.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:42 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:42.574 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:42 smithi204 ceph-mon[87071]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:34:42.574 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:42 smithi204 ceph-mon[87071]: Reconfiguring daemon osd.4 on smithi204 2024-09-13T14:34:42.574 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:42 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:42.574 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:42 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:42.574 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:42 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:34:42.574 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:42 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:43.424 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:43 smithi204 ceph-mon[87071]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:43.424 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:43 smithi204 ceph-mon[87071]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:34:43.425 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:43 smithi204 ceph-mon[87071]: Reconfiguring daemon osd.5 on smithi204 2024-09-13T14:34:43.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:43 smithi096 ceph-mon[98901]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:43.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:43 smithi096 ceph-mon[98901]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:34:43.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:43 smithi096 ceph-mon[98901]: Reconfiguring daemon osd.5 on smithi204 2024-09-13T14:34:43.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:43 smithi096 ceph-mon[102928]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:43.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:43 smithi096 ceph-mon[102928]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:34:43.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:43 smithi096 ceph-mon[102928]: Reconfiguring daemon osd.5 on smithi204 2024-09-13T14:34:44.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:44.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:44 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:44.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:44 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:44.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:44 smithi096 ceph-mon[98901]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:34:44.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:44 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:34:44.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:44 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:44.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:44 smithi096 ceph-mon[98901]: Reconfiguring daemon osd.6 on smithi204 2024-09-13T14:34:44.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:44 smithi096 ceph-mon[98901]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:44.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:44 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:44.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:44 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:44.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:44 smithi096 ceph-mon[102928]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:34:44.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:44 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:34:44.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:44 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:44.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:44 smithi096 ceph-mon[102928]: Reconfiguring daemon osd.6 on smithi204 2024-09-13T14:34:44.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:44 smithi096 ceph-mon[102928]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:44.824 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:44 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:44.824 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:44 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:44.824 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:44 smithi204 ceph-mon[87071]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:34:44.824 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:44 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:34:44.824 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:44 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:44.824 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:44 smithi204 ceph-mon[87071]: Reconfiguring daemon osd.6 on smithi204 2024-09-13T14:34:44.824 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:44 smithi204 ceph-mon[87071]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:45.916 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:45 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:45.916 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:45 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:46.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:45 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:46.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:45 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:46.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:45 smithi204 ceph-mon[87071]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:34:46.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:45 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:34:46.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:45 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:46.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:45 smithi204 ceph-mon[87071]: Reconfiguring daemon osd.7 on smithi204 2024-09-13T14:34:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:45 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:45 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:45 smithi096 ceph-mon[98901]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:34:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:45 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:34:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:45 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:46.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:45 smithi096 ceph-mon[98901]: Reconfiguring daemon osd.7 on smithi204 2024-09-13T14:34:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:45 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:45 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:45 smithi096 ceph-mon[102928]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:34:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:45 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:34:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:45 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:45 smithi096 ceph-mon[102928]: Reconfiguring daemon osd.7 on smithi204 2024-09-13T14:34:47.009 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:47.621 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:47 smithi204 ceph-mon[87071]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:47.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:47 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:47.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:47 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:47.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:47 smithi204 ceph-mon[87071]: Reconfiguring mds.foo.smithi204.hbwbot (monmap changed)... 2024-09-13T14:34:47.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:47 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi204.hbwbot", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:34:47.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:47 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:47.622 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:47 smithi204 ceph-mon[87071]: Reconfiguring daemon mds.foo.smithi204.hbwbot on smithi204 2024-09-13T14:34:47.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:47 smithi096 ceph-mon[98901]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:47.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:47 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:47.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:47 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:47.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:47 smithi096 ceph-mon[98901]: Reconfiguring mds.foo.smithi204.hbwbot (monmap changed)... 2024-09-13T14:34:47.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:47 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi204.hbwbot", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:34:47.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:47 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:47.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:47 smithi096 ceph-mon[98901]: Reconfiguring daemon mds.foo.smithi204.hbwbot on smithi204 2024-09-13T14:34:47.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:47 smithi096 ceph-mon[102928]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:47.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:47 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:47.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:47 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:47.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:47 smithi096 ceph-mon[102928]: Reconfiguring mds.foo.smithi204.hbwbot (monmap changed)... 2024-09-13T14:34:47.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:47 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi204.hbwbot", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:34:47.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:47 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:47.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:47 smithi096 ceph-mon[102928]: Reconfiguring daemon mds.foo.smithi204.hbwbot on smithi204 2024-09-13T14:34:48.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:34:47] "GET /metrics HTTP/1.1" 200 33838 "" "Prometheus/2.51.0" 2024-09-13T14:34:48.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:48 smithi204 ceph-mon[87071]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:48.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:48 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:48.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:48 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:48.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:48 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:48.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:48 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:48.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:48 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:48.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:48 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:49.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:48 smithi096 ceph-mon[98901]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:49.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:48 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:49.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:48 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:49.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:48 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:49.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:48 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:49.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:48 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:49.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:48 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:49.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:48 smithi096 ceph-mon[102928]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:49.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:48 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:49.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:48 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:49.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:48 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:49.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:48 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:49.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:48 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:49.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:48 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:34:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:50.745 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:50 smithi204 ceph-mon[87071]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:51.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:50 smithi096 ceph-mon[98901]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:51.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:50 smithi096 ceph-mon[102928]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:51.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:52.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:52.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:52 smithi204 ceph-mon[87071]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:53.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:52 smithi096 ceph-mon[98901]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:53.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:52 smithi096 ceph-mon[102928]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:54.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:54 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:54.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:54.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:54 smithi204 ceph-mon[87071]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:55.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:54 smithi096 ceph-mon[98901]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:55.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:54 smithi096 ceph-mon[102928]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:56.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:56.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:56.949 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:56 smithi204 ceph-mon[87071]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:57.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:56 smithi096 ceph-mon[98901]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:57.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:56 smithi096 ceph-mon[102928]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:57.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:34:56 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:34:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:58.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:34:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:34:57] "GET /metrics HTTP/1.1" 200 33849 "" "Prometheus/2.51.0" 2024-09-13T14:34:58.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:34:58 smithi204 ceph-mon[87071]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:59.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:34:58 smithi096 ceph-mon[98901]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:59.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:34:58 smithi096 ceph-mon[102928]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:00.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:00.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:00 smithi204 ceph-mon[87071]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:01.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:00 smithi096 ceph-mon[98901]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:01.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:00 smithi096 ceph-mon[102928]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:02.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:02 smithi204 ceph-mon[87071]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:03.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:02 smithi096 ceph-mon[98901]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:03.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:02 smithi096 ceph-mon[102928]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:04.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:04 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:04.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:04.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:04 smithi204 ceph-mon[87071]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:05.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:04 smithi096 ceph-mon[102928]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:05.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:04 smithi096 ceph-mon[98901]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:06.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:05 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:06.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:05 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:06.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:06 smithi204 ceph-mon[87071]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:06.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:06.969 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:06 smithi096 ceph-mon[98901]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:06 smithi096 ceph-mon[102928]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:08.324 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:35:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:35:07] "GET /metrics HTTP/1.1" 200 33850 "" "Prometheus/2.51.0" 2024-09-13T14:35:08.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:08 smithi204 ceph-mon[87071]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:09.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:08 smithi096 ceph-mon[98901]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:09.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:08 smithi096 ceph-mon[102928]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:10.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:10.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:10 smithi204 ceph-mon[87071]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:11.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:10 smithi096 ceph-mon[98901]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:11.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:10 smithi096 ceph-mon[102928]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:12.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:12 smithi204 ceph-mon[87071]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:13.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:12 smithi096 ceph-mon[98901]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:13.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:12 smithi096 ceph-mon[102928]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:14.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:14 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:14.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:15.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:14 smithi096 ceph-mon[98901]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:15.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:14 smithi096 ceph-mon[102928]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:15.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:14 smithi204 ceph-mon[87071]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:16.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:15 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:16.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:15 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:17.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:16 smithi096 ceph-mon[98901]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:17.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:16 smithi096 ceph-mon[102928]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:16 smithi204 ceph-mon[87071]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:17.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:16 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:18.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:35:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:35:17] "GET /metrics HTTP/1.1" 200 33850 "" "Prometheus/2.51.0" 2024-09-13T14:35:19.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:18 smithi096 ceph-mon[98901]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:19.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:18 smithi096 ceph-mon[102928]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:19.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:18 smithi204 ceph-mon[87071]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:19.885 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade resume' 2024-09-13T14:35:20.139 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:20.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:20.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:20 smithi096 ceph-mon[98901]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:20.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:20 smithi096 ceph-mon[102928]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:21.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:20 smithi204 ceph-mon[87071]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:21.740 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:21.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:21.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:21.957 INFO:teuthology.orchestra.run.smithi096.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:22.643 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for all mons to upgrade' 2024-09-13T14:35:23.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:22 smithi204 ceph-mon[87071]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:23.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:22 smithi204 ceph-mon[87071]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:23.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:22 smithi204 ceph-mon[87071]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:23.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:22 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:23.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:22 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:23.259 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:22 smithi096 ceph-mon[98901]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:23.259 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:22 smithi096 ceph-mon[98901]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:23.259 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:22 smithi096 ceph-mon[98901]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:23.260 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:22 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:23.260 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:22 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:23.260 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:22 smithi096 ceph-mon[102928]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:23.260 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:22 smithi096 ceph-mon[102928]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:23.260 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:22 smithi096 ceph-mon[102928]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:23.260 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:22 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:23.260 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:22 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:23.534 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:24.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:24 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:24.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:24 smithi204 ceph-mon[87071]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:25.281 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:24 smithi096 ceph-mon[98901]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:25.282 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:24 smithi096 ceph-mon[102928]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:25.877 INFO:teuthology.orchestra.run.smithi096.stdout:wait for all mons to upgrade 2024-09-13T14:35:26.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:25 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:26.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:25 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:26.823 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-13T14:35:27.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:26 smithi096 ceph-mon[98901]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:27.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:26 smithi096 ceph-mon[102928]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:27.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:26 smithi204 ceph-mon[87071]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:27.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:26 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:27.258 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:28.257 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:35:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:35:27] "GET /metrics HTTP/1.1" 200 33845 "" "Prometheus/2.51.0" 2024-09-13T14:35:29.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:29.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:29.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:29.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:29.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:29.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:29.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:29.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:29.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:29.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:29.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:35:29.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:35:29.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:35:29.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:35:29.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:35:29.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:35:29.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:29.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:29.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:35:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:29.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:35:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:35:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:35:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:35:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:35:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:35:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:29.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:29.076 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:35:29.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:35:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:35:29.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:35:29.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:35:29.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:35:29.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:35:29.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:29.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:29.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:35:29.941 INFO:teuthology.orchestra.run.smithi096.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:35:30.213 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:29 smithi204 ceph-mon[87071]: Upgrade: Setting container_image for all mon 2024-09-13T14:35:30.213 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:29 smithi204 ceph-mon[87071]: Upgrade: Setting container_image for all crash 2024-09-13T14:35:30.213 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:29 smithi204 ceph-mon[87071]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:35:30.213 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:29 smithi204 ceph-mon[87071]: Upgrade: osd.0 is safe to restart 2024-09-13T14:35:30.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:29 smithi096 ceph-mon[98901]: Upgrade: Setting container_image for all mon 2024-09-13T14:35:30.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:29 smithi096 ceph-mon[98901]: Upgrade: Setting container_image for all crash 2024-09-13T14:35:30.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:29 smithi096 ceph-mon[98901]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:35:30.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:29 smithi096 ceph-mon[98901]: Upgrade: osd.0 is safe to restart 2024-09-13T14:35:30.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:29 smithi096 ceph-mon[102928]: Upgrade: Setting container_image for all mon 2024-09-13T14:35:30.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:29 smithi096 ceph-mon[102928]: Upgrade: Setting container_image for all crash 2024-09-13T14:35:30.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:29 smithi096 ceph-mon[102928]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:35:30.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:29 smithi096 ceph-mon[102928]: Upgrade: osd.0 is safe to restart 2024-09-13T14:35:30.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:31.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:30 smithi204 ceph-mon[87071]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:31.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:30 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2026063501' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:35:31.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:30 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:31.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:30 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:35:31.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:30 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:31.234 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:30 smithi096 ceph-mon[98901]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:31.234 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:30 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2026063501' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:35:31.234 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:30 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:31.234 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:30 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:35:31.234 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:30 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:31.234 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:30 smithi096 ceph-mon[102928]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:31.235 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:30 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2026063501' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:35:31.235 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:30 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:31.235 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:30 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:35:31.235 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:30 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:31.257 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:35:32.207 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:31 smithi204 ceph-mon[87071]: Upgrade: Updating osd.0 2024-09-13T14:35:32.207 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:31 smithi204 ceph-mon[87071]: Deploying daemon osd.0 on smithi096 2024-09-13T14:35:32.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:31 smithi096 ceph-mon[98901]: Upgrade: Updating osd.0 2024-09-13T14:35:32.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:31 smithi096 ceph-mon[98901]: Deploying daemon osd.0 on smithi096 2024-09-13T14:35:32.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:31 smithi096 ceph-mon[102928]: Upgrade: Updating osd.0 2024-09-13T14:35:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:31 smithi096 ceph-mon[102928]: Deploying daemon osd.0 on smithi096 2024-09-13T14:35:32.477 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:33.043 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:32 smithi096 ceph-mon[98901]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:33.043 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:32 smithi096 ceph-mon[102928]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:33.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:32 smithi204 ceph-mon[87071]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:34.322 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:34 smithi096 systemd[1]: Stopping Ceph osd.0 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:35:34.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:34 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:34.924 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:34 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0[43517]: 2024-09-13T14:35:34.818+0000 7f9caefdc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:35:34.924 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:34 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0[43517]: 2024-09-13T14:35:34.818+0000 7f9caefdc640 -1 osd.0 82 *** Got signal Terminated *** 2024-09-13T14:35:34.924 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:34 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0[43517]: 2024-09-13T14:35:34.818+0000 7f9caefdc640 -1 osd.0 82 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:35:35.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:34 smithi204 ceph-mon[87071]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:35.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:34 smithi204 ceph-mon[87071]: osd.0 marked itself down and dead 2024-09-13T14:35:35.244 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:34 smithi096 ceph-mon[98901]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:35.244 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:34 smithi096 ceph-mon[98901]: osd.0 marked itself down and dead 2024-09-13T14:35:35.244 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:34 smithi096 ceph-mon[102928]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:35.244 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:34 smithi096 ceph-mon[102928]: osd.0 marked itself down and dead 2024-09-13T14:35:35.458 INFO:teuthology.orchestra.run.smithi096.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:35:35.458 INFO:teuthology.orchestra.run.smithi096.stdout:alertmanager.a smithi096 *:9093,9094 running (4m) 7s ago 8m 18.0M - 0.27.0 11f11916f8cd e04c10a0dca2 2024-09-13T14:35:35.458 INFO:teuthology.orchestra.run.smithi096.stdout:grafana.a smithi204 *:3000 running (3m) 72s ago 8m 79.1M - 9.4.7 954c08fa6188 451523aebd73 2024-09-13T14:35:35.458 INFO:teuthology.orchestra.run.smithi096.stdout:mds.foo.smithi096.dqiqpz smithi096 running (7m) 7s ago 7m 20.5M - 17.2.7-1627-gbbdadfa3 a994b00deb3a ab8d33048bb7 2024-09-13T14:35:35.459 INFO:teuthology.orchestra.run.smithi096.stdout:mds.foo.smithi204.hbwbot smithi204 running (7m) 72s ago 7m 15.6M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 97a3ff635de2 2024-09-13T14:35:35.459 INFO:teuthology.orchestra.run.smithi096.stdout:mgr.x smithi204 *:8443,9283,8765 running (3m) 72s ago 12m 454M - 19.3.0-4697-gd58f7306 fec07016a2b0 04572bea1e6f 2024-09-13T14:35:35.459 INFO:teuthology.orchestra.run.smithi096.stdout:mgr.y smithi096 *:8443,9283,8765 running (3m) 7s ago 14m 509M - 19.3.0-4697-gd58f7306 fec07016a2b0 a4bc51171f6e 2024-09-13T14:35:35.459 INFO:teuthology.orchestra.run.smithi096.stdout:mon.a smithi096 running (3m) 7s ago 14m 46.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 97a94128e048 2024-09-13T14:35:35.459 INFO:teuthology.orchestra.run.smithi096.stdout:mon.b smithi204 running (77s) 72s ago 12m 22.5M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 4cc02299264a 2024-09-13T14:35:35.459 INFO:teuthology.orchestra.run.smithi096.stdout:mon.c smithi096 running (96s) 7s ago 12m 35.1M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 babdd691d9b6 2024-09-13T14:35:35.459 INFO:teuthology.orchestra.run.smithi096.stdout:node-exporter.a smithi096 *:9100 running (4m) 7s ago 8m 11.3M - 1.7.0 72c9c2088986 b02b57424615 2024-09-13T14:35:35.459 INFO:teuthology.orchestra.run.smithi096.stdout:node-exporter.b smithi204 *:9100 running (3m) 72s ago 8m 10.0M - 1.7.0 72c9c2088986 c58ad3b83def 2024-09-13T14:35:35.459 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0 smithi096 running (12m) 7s ago 12m 48.2M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a a35fe012b65d 2024-09-13T14:35:35.459 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1 smithi096 running (11m) 7s ago 11m 51.7M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a edbcf0f2ed44 2024-09-13T14:35:35.459 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2 smithi096 running (11m) 7s ago 11m 48.5M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 565505fd2e48 2024-09-13T14:35:35.459 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3 smithi096 running (10m) 7s ago 10m 45.5M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a afd4dcb32d2d 2024-09-13T14:35:35.459 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4 smithi204 running (10m) 72s ago 10m 45.6M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a b48920b2b9e7 2024-09-13T14:35:35.460 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5 smithi204 running (10m) 72s ago 10m 46.8M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 0ed6e334d2ea 2024-09-13T14:35:35.460 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6 smithi204 running (9m) 72s ago 9m 48.0M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8f12f56d751c 2024-09-13T14:35:35.460 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7 smithi204 running (9m) 72s ago 9m 50.8M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a ffd75c8be38a 2024-09-13T14:35:35.460 INFO:teuthology.orchestra.run.smithi096.stdout:prometheus.a smithi204 *:9095 running (3m) 72s ago 8m 48.4M - 2.51.0 1d3b7f56885b c1d744b6a96b 2024-09-13T14:35:35.545 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:35 smithi096 podman[109312]: 2024-09-13 14:35:35.306426394 +0000 UTC m=+0.585894678 container died a35fe012b65dc66b60759255c4c52af1fe33228c3e11c4e593540899d40fd647 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-13T14:35:35.818 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:35 smithi096 podman[109312]: 2024-09-13 14:35:35.713304259 +0000 UTC m=+0.992772531 container cleanup a35fe012b65dc66b60759255c4c52af1fe33228c3e11c4e593540899d40fd647 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=quincy-bbdadfa) 2024-09-13T14:35:35.818 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:35 smithi096 bash[109312]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0 2024-09-13T14:35:36.072 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:35 smithi096 podman[109368]: 2024-09-13 14:35:35.939636172 +0000 UTC m=+0.626636890 container remove a35fe012b65dc66b60759255c4c52af1fe33228c3e11c4e593540899d40fd647 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-13T14:35:36.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[102928]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:35:36.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[102928]: osdmap e83: 8 total, 7 up, 8 in 2024-09-13T14:35:36.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 2]} : dispatch 2024-09-13T14:35:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]} : dispatch 2024-09-13T14:35:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 0]} : dispatch 2024-09-13T14:35:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [7, 2]} : dispatch 2024-09-13T14:35:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 0]} : dispatch 2024-09-13T14:35:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [3, 0]} : dispatch 2024-09-13T14:35:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]} : dispatch 2024-09-13T14:35:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2, 3, 0]} : dispatch 2024-09-13T14:35:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]} : dispatch 2024-09-13T14:35:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:36.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[98901]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:35:36.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[98901]: osdmap e83: 8 total, 7 up, 8 in 2024-09-13T14:35:36.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 2]} : dispatch 2024-09-13T14:35:36.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]} : dispatch 2024-09-13T14:35:36.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 0]} : dispatch 2024-09-13T14:35:36.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [7, 2]} : dispatch 2024-09-13T14:35:36.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 0]} : dispatch 2024-09-13T14:35:36.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [3, 0]} : dispatch 2024-09-13T14:35:36.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]} : dispatch 2024-09-13T14:35:36.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2, 3, 0]} : dispatch 2024-09-13T14:35:36.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]} : dispatch 2024-09-13T14:35:36.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:36.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:36.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:35 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:36.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:35 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:35 smithi204 ceph-mon[87071]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:35:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:35 smithi204 ceph-mon[87071]: osdmap e83: 8 total, 7 up, 8 in 2024-09-13T14:35:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 2]} : dispatch 2024-09-13T14:35:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]} : dispatch 2024-09-13T14:35:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 0]} : dispatch 2024-09-13T14:35:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [7, 2]} : dispatch 2024-09-13T14:35:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 0]} : dispatch 2024-09-13T14:35:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [3, 0]} : dispatch 2024-09-13T14:35:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]} : dispatch 2024-09-13T14:35:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2, 3, 0]} : dispatch 2024-09-13T14:35:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]} : dispatch 2024-09-13T14:35:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:36.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:36.511 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade pause' 2024-09-13T14:35:37.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[98901]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:37.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[98901]: pgmap v94: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:37.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 2]}]': finished 2024-09-13T14:35:37.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]': finished 2024-09-13T14:35:37.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 0]}]': finished 2024-09-13T14:35:37.049 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [7, 2]}]': finished 2024-09-13T14:35:37.049 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 0]}]': finished 2024-09-13T14:35:37.049 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [3, 0]}]': finished 2024-09-13T14:35:37.049 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]}]': finished 2024-09-13T14:35:37.049 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2, 3, 0]}]': finished 2024-09-13T14:35:37.049 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]': finished 2024-09-13T14:35:37.049 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[98901]: osdmap e84: 8 total, 7 up, 8 in 2024-09-13T14:35:37.049 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:36 smithi096 podman[109500]: 2024-09-13 14:35:36.794577872 +0000 UTC m=+0.107117643 container create de83e195fcf69d4a1b0c7758e31278d4b94886b3d05e33bb9b45ba1955f57fdd (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-13T14:35:37.050 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:36 smithi096 podman[109500]: 2024-09-13 14:35:36.704343165 +0000 UTC m=+0.016882949 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:37.050 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:37 smithi096 podman[109500]: 2024-09-13 14:35:37.047394033 +0000 UTC m=+0.359933804 container init de83e195fcf69d4a1b0c7758e31278d4b94886b3d05e33bb9b45ba1955f57fdd (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-13T14:35:37.050 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[102928]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:37.050 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[102928]: pgmap v94: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:37.050 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 2]}]': finished 2024-09-13T14:35:37.050 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]': finished 2024-09-13T14:35:37.050 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 0]}]': finished 2024-09-13T14:35:37.050 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [7, 2]}]': finished 2024-09-13T14:35:37.050 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 0]}]': finished 2024-09-13T14:35:37.050 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [3, 0]}]': finished 2024-09-13T14:35:37.051 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]}]': finished 2024-09-13T14:35:37.051 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2, 3, 0]}]': finished 2024-09-13T14:35:37.051 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]': finished 2024-09-13T14:35:37.051 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:36 smithi096 ceph-mon[102928]: osdmap e84: 8 total, 7 up, 8 in 2024-09-13T14:35:37.198 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:37.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:36 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:36 smithi204 ceph-mon[87071]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:36 smithi204 ceph-mon[87071]: pgmap v94: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 2]}]': finished 2024-09-13T14:35:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]': finished 2024-09-13T14:35:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 0]}]': finished 2024-09-13T14:35:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [7, 2]}]': finished 2024-09-13T14:35:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 0]}]': finished 2024-09-13T14:35:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [3, 0]}]': finished 2024-09-13T14:35:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]}]': finished 2024-09-13T14:35:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2, 3, 0]}]': finished 2024-09-13T14:35:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]': finished 2024-09-13T14:35:37.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:36 smithi204 ceph-mon[87071]: osdmap e84: 8 total, 7 up, 8 in 2024-09-13T14:35:37.319 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:37 smithi096 podman[109500]: 2024-09-13 14:35:37.052181696 +0000 UTC m=+0.364721470 container start de83e195fcf69d4a1b0c7758e31278d4b94886b3d05e33bb9b45ba1955f57fdd (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux ) 2024-09-13T14:35:37.319 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:37 smithi096 podman[109500]: 2024-09-13 14:35:37.130429952 +0000 UTC m=+0.442969746 container attach de83e195fcf69d4a1b0c7758e31278d4b94886b3d05e33bb9b45ba1955f57fdd (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-13T14:35:38.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:37 smithi096 ceph-mon[102928]: osdmap e85: 8 total, 7 up, 8 in 2024-09-13T14:35:38.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:37 smithi096 ceph-mon[98901]: osdmap e85: 8 total, 7 up, 8 in 2024-09-13T14:35:38.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:35:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:35:37] "GET /metrics HTTP/1.1" 200 33910 "" "Prometheus/2.51.0" 2024-09-13T14:35:38.323 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:37 smithi096 podman[109500]: 2024-09-13 14:35:37.874012251 +0000 UTC m=+1.186552037 container died de83e195fcf69d4a1b0c7758e31278d4b94886b3d05e33bb9b45ba1955f57fdd (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:35:38.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:37 smithi204 ceph-mon[87071]: osdmap e85: 8 total, 7 up, 8 in 2024-09-13T14:35:38.822 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:38 smithi096 podman[109579]: 2024-09-13 14:35:38.40535766 +0000 UTC m=+1.155763299 container remove de83e195fcf69d4a1b0c7758e31278d4b94886b3d05e33bb9b45ba1955f57fdd (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-13T14:35:38.822 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:38 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-13T14:35:38.822 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:38 smithi096 systemd[1]: Stopped Ceph osd.0 for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:35:38.823 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:38 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.0.service: Consumed 5.055s CPU time. 2024-09-13T14:35:38.823 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:38 smithi096 systemd[1]: Starting Ceph osd.0 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:35:39.142 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:38 smithi096 ceph-mon[98901]: pgmap v97: 73 pgs: 6 peering, 16 active+undersized, 1 stale+active+clean, 6 active+undersized+degraded, 44 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%) 2024-09-13T14:35:39.142 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:38 smithi096 ceph-mon[98901]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-13T14:35:39.142 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:38 smithi096 ceph-mon[98901]: Health check failed: Degraded data redundancy: 11/84 objects degraded (13.095%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:39.143 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:38 smithi096 ceph-mon[98901]: osdmap e86: 8 total, 7 up, 8 in 2024-09-13T14:35:39.143 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:38 smithi096 ceph-mon[102928]: pgmap v97: 73 pgs: 6 peering, 16 active+undersized, 1 stale+active+clean, 6 active+undersized+degraded, 44 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%) 2024-09-13T14:35:39.143 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:38 smithi096 ceph-mon[102928]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-13T14:35:39.143 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:38 smithi096 ceph-mon[102928]: Health check failed: Degraded data redundancy: 11/84 objects degraded (13.095%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:39.143 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:38 smithi096 ceph-mon[102928]: osdmap e86: 8 total, 7 up, 8 in 2024-09-13T14:35:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:38 smithi204 ceph-mon[87071]: pgmap v97: 73 pgs: 6 peering, 16 active+undersized, 1 stale+active+clean, 6 active+undersized+degraded, 44 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%) 2024-09-13T14:35:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:38 smithi204 ceph-mon[87071]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-13T14:35:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:38 smithi204 ceph-mon[87071]: Health check failed: Degraded data redundancy: 11/84 objects degraded (13.095%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:38 smithi204 ceph-mon[87071]: osdmap e86: 8 total, 7 up, 8 in 2024-09-13T14:35:39.699 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:39 smithi096 podman[109728]: 2024-09-13 14:35:39.320834635 +0000 UTC m=+0.016525451 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:39.699 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:39 smithi096 podman[109728]: 2024-09-13 14:35:39.429599373 +0000 UTC m=+0.125290179 container create debe0bfdeadfa389cf7e2e73b1f982542365936d4ce982940bd7c873c854bc41 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-13T14:35:39.902 INFO:teuthology.orchestra.run.smithi096.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:39.990 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:39 smithi096 podman[109728]: 2024-09-13 14:35:39.849930842 +0000 UTC m=+0.545621642 container init debe0bfdeadfa389cf7e2e73b1f982542365936d4ce982940bd7c873c854bc41 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-13T14:35:39.990 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:39 smithi096 podman[109728]: 2024-09-13 14:35:39.85372984 +0000 UTC m=+0.549420642 container start debe0bfdeadfa389cf7e2e73b1f982542365936d4ce982940bd7c873c854bc41 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T14:35:39.990 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:39 smithi096 podman[109728]: 2024-09-13 14:35:39.933095253 +0000 UTC m=+0.628786054 container attach debe0bfdeadfa389cf7e2e73b1f982542365936d4ce982940bd7c873c854bc41 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True) 2024-09-13T14:35:40.322 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:39 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate[109743]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:40.322 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:39 smithi096 bash[109728]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:40.322 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate[109743]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:40.322 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 bash[109728]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:40.323 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate[109743]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:35:40.323 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 bash[109728]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:35:40.323 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate[109743]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:40.323 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 bash[109728]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:40.323 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate[109743]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:40.323 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 bash[109728]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:40.323 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate[109743]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:35:40.323 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 bash[109728]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:35:40.323 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate[109743]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-13T14:35:40.323 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 bash[109728]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-13T14:35:40.454 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'sleep 60' 2024-09-13T14:35:40.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:40.691 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate[109743]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:35:40.691 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 bash[109728]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:35:40.691 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate[109743]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:35:40.691 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 bash[109728]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:35:40.691 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate[109743]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T14:35:40.691 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 bash[109728]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T14:35:40.691 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate[109743]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:35:40.691 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 bash[109728]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:35:40.691 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate[109743]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-13T14:35:40.691 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 bash[109728]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-13T14:35:40.692 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:40 smithi096 podman[109728]: 2024-09-13 14:35:40.430509341 +0000 UTC m=+1.126200139 container died debe0bfdeadfa389cf7e2e73b1f982542365936d4ce982940bd7c873c854bc41 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-13T14:35:41.033 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-mon[98901]: pgmap v99: 73 pgs: 6 peering, 16 active+undersized, 1 stale+active+clean, 6 active+undersized+degraded, 44 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%) 2024-09-13T14:35:41.033 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-mon[98901]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:41.033 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-mon[98901]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:41.033 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:41.034 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-mon[102928]: pgmap v99: 73 pgs: 6 peering, 16 active+undersized, 1 stale+active+clean, 6 active+undersized+degraded, 44 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%) 2024-09-13T14:35:41.034 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-mon[102928]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:41.034 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-mon[102928]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:41.034 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:40 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:41.140 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:41.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:40 smithi204 ceph-mon[87071]: pgmap v99: 73 pgs: 6 peering, 16 active+undersized, 1 stale+active+clean, 6 active+undersized+degraded, 44 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%) 2024-09-13T14:35:41.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:40 smithi204 ceph-mon[87071]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:41.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:40 smithi204 ceph-mon[87071]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:41.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:40 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:41.322 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:41 smithi096 podman[109890]: 2024-09-13 14:35:41.032925091 +0000 UTC m=+0.603876728 container remove debe0bfdeadfa389cf7e2e73b1f982542365936d4ce982940bd7c873c854bc41 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True) 2024-09-13T14:35:41.322 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:41 smithi096 podman[110007]: 2024-09-13 14:35:41.148644863 +0000 UTC m=+0.017326649 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:41.322 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:41 smithi096 podman[110007]: 2024-09-13 14:35:41.258678663 +0000 UTC m=+0.127360447 container create ab741f6f9e89b46014f5bad7029e73a146f897aa8195dd59deb225b0b80e6e97 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-13T14:35:41.780 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:41 smithi096 podman[110007]: 2024-09-13 14:35:41.511899798 +0000 UTC m=+0.380581581 container init ab741f6f9e89b46014f5bad7029e73a146f897aa8195dd59deb225b0b80e6e97 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-13T14:35:41.780 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:41 smithi096 podman[110007]: 2024-09-13 14:35:41.515225281 +0000 UTC m=+0.383907070 container start ab741f6f9e89b46014f5bad7029e73a146f897aa8195dd59deb225b0b80e6e97 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:35:41.780 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:41 smithi096 ceph-osd[110042]: -- 172.21.15.96:0/1106998469 <== mon.2 v2:172.21.15.96:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d46b377a40 con 0x55d46b355c00 2024-09-13T14:35:41.780 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:41 smithi096 bash[110007]: ab741f6f9e89b46014f5bad7029e73a146f897aa8195dd59deb225b0b80e6e97 2024-09-13T14:35:41.780 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:41 smithi096 systemd[1]: Started Ceph osd.0 for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:35:42.744 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:42 smithi096 ceph-mon[98901]: pgmap v100: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 6 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 9/84 objects misplaced (10.714%); 3 B/s, 0 objects/s recovering 2024-09-13T14:35:42.744 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:42 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:42.744 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:42 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:42.745 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:42 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:42.745 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:42 smithi096 ceph-mon[98901]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-09-13T14:35:42.745 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:42 smithi096 ceph-mon[102928]: pgmap v100: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 6 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 9/84 objects misplaced (10.714%); 3 B/s, 0 objects/s recovering 2024-09-13T14:35:42.745 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:42 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:42.745 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:42 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:42.745 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:42 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:42.745 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:42 smithi096 ceph-mon[102928]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-09-13T14:35:42.953 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:42 smithi204 ceph-mon[87071]: pgmap v100: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 6 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 9/84 objects misplaced (10.714%); 3 B/s, 0 objects/s recovering 2024-09-13T14:35:42.953 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:42 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:42.953 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:42 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:42.953 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:42 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:42.953 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:42 smithi204 ceph-mon[87071]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-09-13T14:35:43.674 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:43 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0[110036]: 2024-09-13T14:35:43.412+0000 7f1d2ba06640 -1 Falling back to public interface 2024-09-13T14:35:44.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:44.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-13T14:35:44.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:44.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:44 smithi204 ceph-mon[87071]: pgmap v101: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 6 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 9/84 objects misplaced (10.714%); 2 B/s, 0 objects/s recovering 2024-09-13T14:35:45.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:44 smithi096 ceph-mon[98901]: pgmap v101: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 6 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 9/84 objects misplaced (10.714%); 2 B/s, 0 objects/s recovering 2024-09-13T14:35:45.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:44 smithi096 ceph-mon[102928]: pgmap v101: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 6 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 9/84 objects misplaced (10.714%); 2 B/s, 0 objects/s recovering 2024-09-13T14:35:46.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:45 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:46.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:45 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:46.950 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:46 smithi204 ceph-mon[87071]: pgmap v102: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 6 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 9/84 objects misplaced (10.714%); 2 B/s, 0 objects/s recovering 2024-09-13T14:35:46.950 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:46 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:46.950 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:46 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:46.950 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:46 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:46.950 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:46 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:46.962 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:46 smithi096 ceph-mon[98901]: pgmap v102: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 6 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 9/84 objects misplaced (10.714%); 2 B/s, 0 objects/s recovering 2024-09-13T14:35:46.963 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:46 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:46.963 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:46 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:46.963 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:46 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:46.963 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:46 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:46.963 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:46 smithi096 ceph-mon[102928]: pgmap v102: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 6 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 9/84 objects misplaced (10.714%); 2 B/s, 0 objects/s recovering 2024-09-13T14:35:46.963 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:46 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:46.963 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:46 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:46.963 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:46 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:46.963 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:46 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:47.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:47.322 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:46 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0[110036]: 2024-09-13T14:35:46.960+0000 7f1d2ba06640 -1 osd.0 82 log_to_monitors true 2024-09-13T14:35:48.218 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:47 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:48.218 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:47 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:48.218 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:47 smithi096 ceph-mon[98901]: from='osd.0 [v2:172.21.15.96:6802/2849588101,v1:172.21.15.96:6803/2849588101]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:35:48.218 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:47 smithi096 ceph-mon[98901]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:35:48.219 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:35:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:35:47] "GET /metrics HTTP/1.1" 200 33910 "" "Prometheus/2.51.0" 2024-09-13T14:35:48.219 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 14:35:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0[110036]: 2024-09-13T14:35:47.978+0000 7f1d237b0640 -1 osd.0 82 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:35:48.219 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:47 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:48.219 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:47 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:48.219 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:47 smithi096 ceph-mon[102928]: from='osd.0 [v2:172.21.15.96:6802/2849588101,v1:172.21.15.96:6803/2849588101]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:35:48.219 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:47 smithi096 ceph-mon[102928]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:35:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:47 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:47 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:47 smithi204 ceph-mon[87071]: from='osd.0 [v2:172.21.15.96:6802/2849588101,v1:172.21.15.96:6803/2849588101]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:35:48.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:47 smithi204 ceph-mon[87071]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:35:49.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:48 smithi204 ceph-mon[87071]: pgmap v103: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 6 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 9/84 objects misplaced (10.714%); 2 B/s, 0 objects/s recovering 2024-09-13T14:35:49.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:48 smithi204 ceph-mon[87071]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:35:49.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:48 smithi204 ceph-mon[87071]: osdmap e87: 8 total, 7 up, 8 in 2024-09-13T14:35:49.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:48 smithi204 ceph-mon[87071]: from='osd.0 [v2:172.21.15.96:6802/2849588101,v1:172.21.15.96:6803/2849588101]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:35:49.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:48 smithi204 ceph-mon[87071]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:35:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:48 smithi096 ceph-mon[98901]: pgmap v103: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 6 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 9/84 objects misplaced (10.714%); 2 B/s, 0 objects/s recovering 2024-09-13T14:35:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:48 smithi096 ceph-mon[98901]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:35:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:48 smithi096 ceph-mon[98901]: osdmap e87: 8 total, 7 up, 8 in 2024-09-13T14:35:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:48 smithi096 ceph-mon[98901]: from='osd.0 [v2:172.21.15.96:6802/2849588101,v1:172.21.15.96:6803/2849588101]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:35:49.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:48 smithi096 ceph-mon[98901]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:35:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:48 smithi096 ceph-mon[102928]: pgmap v103: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 6 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 9/84 objects misplaced (10.714%); 2 B/s, 0 objects/s recovering 2024-09-13T14:35:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:48 smithi096 ceph-mon[102928]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:35:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:48 smithi096 ceph-mon[102928]: osdmap e87: 8 total, 7 up, 8 in 2024-09-13T14:35:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:48 smithi096 ceph-mon[102928]: from='osd.0 [v2:172.21.15.96:6802/2849588101,v1:172.21.15.96:6803/2849588101]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:35:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:48 smithi096 ceph-mon[102928]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:35:50.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[98901]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:35:50.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[98901]: osd.0 [v2:172.21.15.96:6802/2849588101,v1:172.21.15.96:6803/2849588101] boot 2024-09-13T14:35:50.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[98901]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T14:35:50.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:35:50.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:50.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:50.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:50.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:50.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:50.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:50.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:50.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:50.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:50.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[102928]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:35:50.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[102928]: osd.0 [v2:172.21.15.96:6802/2849588101,v1:172.21.15.96:6803/2849588101] boot 2024-09-13T14:35:50.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[102928]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T14:35:50.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:35:50.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:50.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:50.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:50.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:50.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:50.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:50.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:50.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:50.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:49 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:50.383 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:50.383 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:49 smithi204 ceph-mon[87071]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:35:50.384 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:49 smithi204 ceph-mon[87071]: osd.0 [v2:172.21.15.96:6802/2849588101,v1:172.21.15.96:6803/2849588101] boot 2024-09-13T14:35:50.384 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:49 smithi204 ceph-mon[87071]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T14:35:50.384 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:49 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:35:50.384 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:49 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:50.384 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:49 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:50.384 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:49 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:50.384 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:49 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:50.384 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:49 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:50.384 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:49 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:50.384 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:49 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:50.384 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:49 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:50.384 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:49 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:50 smithi096 ceph-mon[98901]: pgmap v106: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 6 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 9/84 objects misplaced (10.714%) 2024-09-13T14:35:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:50 smithi096 ceph-mon[98901]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T14:35:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:50 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:50 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:50 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:50 smithi096 ceph-mon[98901]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T14:35:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:50 smithi096 ceph-mon[102928]: pgmap v106: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 6 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 9/84 objects misplaced (10.714%) 2024-09-13T14:35:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:50 smithi096 ceph-mon[102928]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T14:35:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:50 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:50 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:50 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:50 smithi096 ceph-mon[102928]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T14:35:51.342 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:50 smithi204 ceph-mon[87071]: pgmap v106: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 6 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 9/84 objects misplaced (10.714%) 2024-09-13T14:35:51.342 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:50 smithi204 ceph-mon[87071]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T14:35:51.342 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:50 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:51.342 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:50 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:51.342 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:50 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:35:51.342 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:50 smithi204 ceph-mon[87071]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T14:35:52.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:52 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:35:52.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:52 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:35:52.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:52 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:35:53.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:53 smithi096 ceph-mon[98901]: pgmap v109: 73 pgs: 5 active+clean+remapped, 8 active+undersized, 1 active+undersized+degraded, 59 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%); 5/84 objects misplaced (5.952%); 22 B/s, 0 objects/s recovering 2024-09-13T14:35:53.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:53 smithi096 ceph-mon[102928]: pgmap v109: 73 pgs: 5 active+clean+remapped, 8 active+undersized, 1 active+undersized+degraded, 59 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%); 5/84 objects misplaced (5.952%); 22 B/s, 0 objects/s recovering 2024-09-13T14:35:53.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:53 smithi204 ceph-mon[87071]: pgmap v109: 73 pgs: 5 active+clean+remapped, 8 active+undersized, 1 active+undersized+degraded, 59 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%); 5/84 objects misplaced (5.952%); 22 B/s, 0 objects/s recovering 2024-09-13T14:35:54.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:54 smithi204 ceph-mon[87071]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-09-13T14:35:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:54 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:35:54.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:54 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:54.146Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-13T14:35:54.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:54 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:54.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:54 smithi096 ceph-mon[98901]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-09-13T14:35:54.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:54 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:35:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:54 smithi096 ceph-mon[102928]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-09-13T14:35:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:54 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:35:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:55 smithi204 ceph-mon[87071]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 861 B/s, 1 objects/s recovering 2024-09-13T14:35:55.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:55 smithi096 ceph-mon[98901]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 861 B/s, 1 objects/s recovering 2024-09-13T14:35:55.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:55 smithi096 ceph-mon[102928]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 861 B/s, 1 objects/s recovering 2024-09-13T14:35:56.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:56.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:57.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:57 smithi204 ceph-mon[87071]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 731 B/s, 1 objects/s recovering 2024-09-13T14:35:57.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:35:56 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:35:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:57.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:57 smithi096 ceph-mon[98901]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 731 B/s, 1 objects/s recovering 2024-09-13T14:35:57.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:57 smithi096 ceph-mon[102928]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 731 B/s, 1 objects/s recovering 2024-09-13T14:35:58.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:35:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:35:57] "GET /metrics HTTP/1.1" 200 33978 "" "Prometheus/2.51.0" 2024-09-13T14:35:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:35:59 smithi204 ceph-mon[87071]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 604 B/s, 0 objects/s recovering 2024-09-13T14:35:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:35:59 smithi096 ceph-mon[98901]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 604 B/s, 0 objects/s recovering 2024-09-13T14:35:59.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:35:59 smithi096 ceph-mon[102928]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 604 B/s, 0 objects/s recovering 2024-09-13T14:36:00.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:01.415 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:01 smithi204 ceph-mon[87071]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 503 B/s, 0 objects/s recovering 2024-09-13T14:36:01.415 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:01 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:01.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:01 smithi096 ceph-mon[98901]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 503 B/s, 0 objects/s recovering 2024-09-13T14:36:01.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:01 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:01 smithi096 ceph-mon[102928]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 503 B/s, 0 objects/s recovering 2024-09-13T14:36:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:01 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:03.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:03 smithi204 ceph-mon[87071]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 456 B/s, 0 objects/s recovering 2024-09-13T14:36:03.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:03 smithi096 ceph-mon[98901]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 456 B/s, 0 objects/s recovering 2024-09-13T14:36:03.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:03 smithi096 ceph-mon[102928]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 456 B/s, 0 objects/s recovering 2024-09-13T14:36:04.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:04 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:05.374 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:05 smithi096 ceph-mon[98901]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 395 B/s, 0 objects/s recovering 2024-09-13T14:36:05.374 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:05 smithi096 ceph-mon[102928]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 395 B/s, 0 objects/s recovering 2024-09-13T14:36:05.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:05 smithi204 ceph-mon[87071]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 395 B/s, 0 objects/s recovering 2024-09-13T14:36:06.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:05 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:05.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:06.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:05 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:05.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:06 smithi096 ceph-mon[98901]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:07.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:06 smithi096 ceph-mon[102928]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:07.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:07.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:06 smithi204 ceph-mon[87071]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:07.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:08.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:36:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:36:07] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.51.0" 2024-09-13T14:36:09.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:08 smithi096 ceph-mon[98901]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:09.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:08 smithi096 ceph-mon[102928]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:09.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:08 smithi204 ceph-mon[87071]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:10.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:11.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:10 smithi096 ceph-mon[98901]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:11.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:10 smithi096 ceph-mon[102928]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:11.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:10 smithi204 ceph-mon[87071]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:13.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:12 smithi096 ceph-mon[98901]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:13.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:12 smithi096 ceph-mon[102928]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:13.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:12 smithi204 ceph-mon[87071]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:14.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:14 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:15.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:14 smithi096 ceph-mon[98901]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:15.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:14 smithi096 ceph-mon[102928]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:15.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:14 smithi204 ceph-mon[87071]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:16.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:15 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:16.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:15 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:17.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:16 smithi096 ceph-mon[98901]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:17.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:16 smithi096 ceph-mon[102928]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:16 smithi204 ceph-mon[87071]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:17.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:16 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:18.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:36:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:36:17] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.51.0" 2024-09-13T14:36:19.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:18 smithi204 ceph-mon[87071]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:19.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:18 smithi096 ceph-mon[98901]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:19.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:18 smithi096 ceph-mon[102928]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:20.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:21.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:20 smithi204 ceph-mon[87071]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:20 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:21.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:20 smithi096 ceph-mon[98901]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:21.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:20 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:21.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:20 smithi096 ceph-mon[102928]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:21.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:20 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:23.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:22 smithi204 ceph-mon[87071]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:23.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:22 smithi096 ceph-mon[98901]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:23.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:22 smithi096 ceph-mon[102928]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:24.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:24 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.96\", device_class=\"ssd\", hostname=\"smithi096\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.96\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:24 smithi204 ceph-mon[87071]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:25.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:24 smithi096 ceph-mon[98901]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:25.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:24 smithi096 ceph-mon[102928]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:26.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:25 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:26.218 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:25 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:27.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:26 smithi204 ceph-mon[87071]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:27.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:26 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi096\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi096\", job=\"node\", machine=\"x86_64\", nodename=\"smithi096\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:27.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:26 smithi096 ceph-mon[98901]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:27.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:26 smithi096 ceph-mon[102928]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:28.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:36:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:36:27] "GET /metrics HTTP/1.1" 200 33975 "" "Prometheus/2.51.0" 2024-09-13T14:36:29.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:28 smithi204 ceph-mon[87071]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:28 smithi096 ceph-mon[98901]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:29.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:28 smithi096 ceph-mon[102928]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:30.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:31.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:30 smithi204 ceph-mon[87071]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:31.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:30 smithi096 ceph-mon[98901]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:31.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:30 smithi096 ceph-mon[102928]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:33.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:32 smithi204 ceph-mon[87071]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:33.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:32 smithi096 ceph-mon[98901]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:33.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:32 smithi096 ceph-mon[102928]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:35.124 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:34 smithi096 ceph-mon[98901]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:35.125 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:34 smithi096 ceph-mon[102928]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:35.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:34 smithi204 ceph-mon[87071]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:36.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:36.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:36.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:36 smithi204 ceph-mon[87071]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:37.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:36 smithi096 ceph-mon[98901]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:37.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:36 smithi096 ceph-mon[102928]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:38.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:36:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:36:37] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.51.0" 2024-09-13T14:36:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:38 smithi204 ceph-mon[87071]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:39.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:38 smithi096 ceph-mon[98901]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:39.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:38 smithi096 ceph-mon[102928]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:40.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:41.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:40 smithi204 ceph-mon[87071]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:41.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:40 smithi096 ceph-mon[98901]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:41.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:40 smithi096 ceph-mon[102928]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:43.129 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:42 smithi096 ceph-mon[98901]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:43.130 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:42 smithi096 ceph-mon[102928]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:43.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:42 smithi204 ceph-mon[87071]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:43.909 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade resume' 2024-09-13T14:36:44.157 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:45.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:44 smithi096 ceph-mon[98901]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:45.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:44 smithi096 ceph-mon[102928]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:45.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:44 smithi204 ceph-mon[87071]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:46.050 INFO:teuthology.orchestra.run.smithi096.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:36:46.786 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for half of osds to upgrade' 2024-09-13T14:36:47.051 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:47.320 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:47 smithi096 ceph-mon[98901]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:47.320 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:47 smithi096 ceph-mon[98901]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:47.320 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:47 smithi096 ceph-mon[98901]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:36:47.320 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:47 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:47.320 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:47 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:47.320 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:47 smithi096 ceph-mon[102928]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:47.320 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:47 smithi096 ceph-mon[102928]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:47.320 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:47 smithi096 ceph-mon[102928]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:36:47.321 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:47 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:47.321 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:47 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:47.368 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:47 smithi204 ceph-mon[87071]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:47.369 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:47 smithi204 ceph-mon[87071]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:47.369 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:47 smithi204 ceph-mon[87071]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:36:47.369 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:47 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:47.369 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:47 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:48.144 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:36:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:36:47] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.51.0" 2024-09-13T14:36:48.622 INFO:teuthology.orchestra.run.smithi096.stdout:wait for half of osds to upgrade 2024-09-13T14:36:49.185 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:49 smithi096 ceph-mon[98901]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:49.185 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:49 smithi096 ceph-mon[102928]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:49.211 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-13T14:36:49.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:49 smithi204 ceph-mon[87071]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:49.456 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:50.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:36:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:36:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:51.301 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[98901]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:51.301 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:51.301 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:51.301 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:51.301 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:51.301 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:51.301 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:51.302 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:51.302 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:51.302 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:51.302 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:36:51.302 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:51.302 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[102928]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:51.302 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:51.302 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:51.302 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:51.302 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:51.303 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:51.303 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:51.303 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:51.303 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:51.303 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:51.303 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:36:51.303 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:51 smithi204 ceph-mon[87071]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:36:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:51.481 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:36:52.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:52 smithi096 ceph-mon[98901]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:36:52.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:52 smithi096 ceph-mon[98901]: Upgrade: osd.1 is safe to restart 2024-09-13T14:36:52.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:52 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2393818156' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:36:52.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:52 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:52.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:52 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:36:52.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:52 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:52.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:52 smithi096 ceph-mon[102928]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:36:52.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:52 smithi096 ceph-mon[102928]: Upgrade: osd.1 is safe to restart 2024-09-13T14:36:52.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:52 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2393818156' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:36:52.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:52 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:52.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:52 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:36:52.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:52 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:52.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:52 smithi204 ceph-mon[87071]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:36:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:52 smithi204 ceph-mon[87071]: Upgrade: osd.1 is safe to restart 2024-09-13T14:36:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:52 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2393818156' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:36:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:52 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:36:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:52 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:36:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:52 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:53.450 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:53 smithi096 ceph-mon[98901]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:53.450 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:53 smithi096 ceph-mon[98901]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:53.450 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:53 smithi096 ceph-mon[98901]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:53.450 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:53 smithi096 ceph-mon[98901]: Upgrade: Updating osd.1 2024-09-13T14:36:53.451 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:53 smithi096 ceph-mon[98901]: Deploying daemon osd.1 on smithi096 2024-09-13T14:36:53.451 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:53 smithi096 ceph-mon[102928]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:53.451 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:53 smithi096 ceph-mon[102928]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:53.451 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:53 smithi096 ceph-mon[102928]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:53.451 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:53 smithi096 ceph-mon[102928]: Upgrade: Updating osd.1 2024-09-13T14:36:53.451 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:53 smithi096 ceph-mon[102928]: Deploying daemon osd.1 on smithi096 2024-09-13T14:36:53.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:53 smithi204 ceph-mon[87071]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:53.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:53 smithi204 ceph-mon[87071]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:53.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:53 smithi204 ceph-mon[87071]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:53.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:53 smithi204 ceph-mon[87071]: Upgrade: Updating osd.1 2024-09-13T14:36:53.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:53 smithi204 ceph-mon[87071]: Deploying daemon osd.1 on smithi096 2024-09-13T14:36:54.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:54 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1094835336' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:36:54.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:54 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1094835336' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:36:54.352 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:36:54.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:54 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1094835336' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:36:55.246 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:55 smithi096 ceph-mon[98901]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:55.247 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:55 smithi096 ceph-mon[98901]: from='client.34302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:55.247 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:55 smithi096 ceph-mon[102928]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:55.247 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:55 smithi096 ceph-mon[102928]: from='client.34302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:55.247 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:54 smithi096 systemd[1]: Stopping Ceph osd.1 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:36:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:55 smithi204 ceph-mon[87071]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:55 smithi204 ceph-mon[87071]: from='client.34302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:55.572 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:55 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1[49132]: 2024-09-13T14:36:55.245+0000 7ff739d66640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:36:55.572 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:55 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1[49132]: 2024-09-13T14:36:55.245+0000 7ff739d66640 -1 osd.1 90 *** Got signal Terminated *** 2024-09-13T14:36:55.572 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:55 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1[49132]: 2024-09-13T14:36:55.245+0000 7ff739d66640 -1 osd.1 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:36:56.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:56 smithi204 ceph-mon[87071]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:56.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:56 smithi204 ceph-mon[87071]: osd.1 marked itself down and dead 2024-09-13T14:36:56.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:56 smithi096 ceph-mon[98901]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:56.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:56 smithi096 ceph-mon[98901]: osd.1 marked itself down and dead 2024-09-13T14:36:56.493 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:56 smithi096 ceph-mon[102928]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:56.493 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:56 smithi096 ceph-mon[102928]: osd.1 marked itself down and dead 2024-09-13T14:36:56.794 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:56 smithi096 podman[114741]: 2024-09-13 14:36:56.492634855 +0000 UTC m=+1.373960961 container died edbcf0f2ed4477bfb34f87f8fe07dcac0a45c8f7db67079767881d8aa324c9a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-13T14:36:57.072 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:56 smithi096 podman[114741]: 2024-09-13 14:36:56.94566498 +0000 UTC m=+1.826991083 container cleanup edbcf0f2ed4477bfb34f87f8fe07dcac0a45c8f7db67079767881d8aa324c9a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True) 2024-09-13T14:36:57.072 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:56 smithi096 bash[114741]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1 2024-09-13T14:36:57.221 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:36:57.401 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:57 smithi096 ceph-mon[98901]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:57.401 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:57 smithi096 ceph-mon[98901]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:36:57.401 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:57 smithi096 ceph-mon[98901]: osdmap e91: 8 total, 7 up, 8 in 2024-09-13T14:36:57.401 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:57 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1355512575' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:36:57.401 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:57 smithi096 podman[114759]: 2024-09-13 14:36:57.222198569 +0000 UTC m=+0.725779609 container remove edbcf0f2ed4477bfb34f87f8fe07dcac0a45c8f7db67079767881d8aa324c9a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-13T14:36:57.402 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:57 smithi096 ceph-mon[102928]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:57.402 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:57 smithi096 ceph-mon[102928]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:36:57.402 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:57 smithi096 ceph-mon[102928]: osdmap e91: 8 total, 7 up, 8 in 2024-09-13T14:36:57.402 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:57 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1355512575' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:36:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:57 smithi204 ceph-mon[87071]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:57 smithi204 ceph-mon[87071]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:36:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:57 smithi204 ceph-mon[87071]: osdmap e91: 8 total, 7 up, 8 in 2024-09-13T14:36:57.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:57 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1355512575' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:36:57.713 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:57 smithi096 podman[114913]: 2024-09-13 14:36:57.486230796 +0000 UTC m=+0.016776931 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:36:57.713 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:57 smithi096 podman[114913]: 2024-09-13 14:36:57.606770429 +0000 UTC m=+0.137316606 container create f3cb7d25feffb9bd9b2a04726df6a005f86d0bc0a67dae9c438d8d1b33bc7776 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-13T14:36:58.013 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:36:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:36:57] "GET /metrics HTTP/1.1" 200 33973 "" "Prometheus/2.51.0" 2024-09-13T14:36:58.013 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:57 smithi096 podman[114913]: 2024-09-13 14:36:57.817673955 +0000 UTC m=+0.348220091 container init f3cb7d25feffb9bd9b2a04726df6a005f86d0bc0a67dae9c438d8d1b33bc7776 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-13T14:36:58.014 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:57 smithi096 podman[114913]: 2024-09-13 14:36:57.822022403 +0000 UTC m=+0.352568539 container start f3cb7d25feffb9bd9b2a04726df6a005f86d0bc0a67dae9c438d8d1b33bc7776 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-13T14:36:58.014 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:57 smithi096 podman[114913]: 2024-09-13 14:36:57.900886113 +0000 UTC m=+0.431432257 container attach f3cb7d25feffb9bd9b2a04726df6a005f86d0bc0a67dae9c438d8d1b33bc7776 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-13T14:36:58.304 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:58 smithi096 ceph-mon[98901]: osdmap e92: 8 total, 7 up, 8 in 2024-09-13T14:36:58.304 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:58 smithi096 ceph-mon[98901]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:58.304 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:58 smithi096 ceph-mon[102928]: osdmap e92: 8 total, 7 up, 8 in 2024-09-13T14:36:58.304 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:58 smithi096 ceph-mon[102928]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:58.305 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:58 smithi096 podman[114913]: 2024-09-13 14:36:58.012901225 +0000 UTC m=+0.543447369 container died f3cb7d25feffb9bd9b2a04726df6a005f86d0bc0a67dae9c438d8d1b33bc7776 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0) 2024-09-13T14:36:58.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:58 smithi204 ceph-mon[87071]: osdmap e92: 8 total, 7 up, 8 in 2024-09-13T14:36:58.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:58 smithi204 ceph-mon[87071]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:58.822 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:58 smithi096 podman[114939]: 2024-09-13 14:36:58.554447765 +0000 UTC m=+0.535760631 container remove f3cb7d25feffb9bd9b2a04726df6a005f86d0bc0a67dae9c438d8d1b33bc7776 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:36:58.822 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:58 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-13T14:36:58.822 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:58 smithi096 systemd[1]: Stopped Ceph osd.1 for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:36:58.822 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:58 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.1.service: Consumed 6.108s CPU time. 2024-09-13T14:36:58.822 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:58 smithi096 systemd[1]: Starting Ceph osd.1 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:36:59.225 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-mon[98901]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:59.225 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-mon[98901]: pgmap v144: 73 pgs: 27 peering, 1 stale+active+clean, 45 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:59.225 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-mon[98901]: Health check failed: Reduced data availability: 8 pgs inactive, 27 pgs peering (PG_AVAILABILITY) 2024-09-13T14:36:59.226 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:58 smithi096 podman[115040]: 2024-09-13 14:36:58.97222548 +0000 UTC m=+0.100062904 container create 714b8472e0c81238944b2b186a0bd7f031e7f6c4f5c88709854a8d75ec541dd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-13T14:36:59.226 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:58 smithi096 podman[115040]: 2024-09-13 14:36:58.888131902 +0000 UTC m=+0.015969324 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:36:59.226 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-mon[102928]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:59.226 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-mon[102928]: pgmap v144: 73 pgs: 27 peering, 1 stale+active+clean, 45 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:59.226 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-mon[102928]: Health check failed: Reduced data availability: 8 pgs inactive, 27 pgs peering (PG_AVAILABILITY) 2024-09-13T14:36:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:59 smithi204 ceph-mon[87071]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:59 smithi204 ceph-mon[87071]: pgmap v144: 73 pgs: 27 peering, 1 stale+active+clean, 45 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:36:59 smithi204 ceph-mon[87071]: Health check failed: Reduced data availability: 8 pgs inactive, 27 pgs peering (PG_AVAILABILITY) 2024-09-13T14:36:59.573 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 podman[115040]: 2024-09-13 14:36:59.225175229 +0000 UTC m=+0.353012658 container init 714b8472e0c81238944b2b186a0bd7f031e7f6c4f5c88709854a8d75ec541dd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-13T14:36:59.573 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 podman[115040]: 2024-09-13 14:36:59.228761434 +0000 UTC m=+0.356598856 container start 714b8472e0c81238944b2b186a0bd7f031e7f6c4f5c88709854a8d75ec541dd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-13T14:36:59.574 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 podman[115040]: 2024-09-13 14:36:59.324757476 +0000 UTC m=+0.452594900 container attach 714b8472e0c81238944b2b186a0bd7f031e7f6c4f5c88709854a8d75ec541dd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T14:36:59.574 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate[115055]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:36:59.574 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 bash[115040]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:36:59.574 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate[115055]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:36:59.574 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 bash[115040]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:36:59.574 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate[115055]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:36:59.574 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 bash[115040]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:36:59.574 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate[115055]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:36:59.574 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 bash[115040]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:36:59.574 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate[115055]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:36:59.574 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 bash[115040]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:36:59.574 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate[115055]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:36:59.575 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 bash[115040]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:36:59.575 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate[115055]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-13T14:36:59.575 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 bash[115040]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-13T14:37:00.071 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate[115055]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:37:00.071 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 bash[115040]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:37:00.071 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate[115055]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:37:00.071 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 bash[115040]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:37:00.071 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate[115055]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T14:37:00.072 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 bash[115040]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T14:37:00.072 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate[115055]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:37:00.072 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 bash[115040]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:37:00.072 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate[115055]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-13T14:37:00.072 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 bash[115040]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-13T14:37:00.072 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:36:59 smithi096 podman[115040]: 2024-09-13 14:36:59.776191617 +0000 UTC m=+0.904029034 container died 714b8472e0c81238944b2b186a0bd7f031e7f6c4f5c88709854a8d75ec541dd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-13T14:37:00.089 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:00.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:00 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3547081523' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:00.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:00 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3547081523' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:00 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3547081523' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:00.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:37:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:37:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:00.739 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:37:00 smithi096 podman[115182]: 2024-09-13 14:37:00.387731183 +0000 UTC m=+0.605628251 container remove 714b8472e0c81238944b2b186a0bd7f031e7f6c4f5c88709854a8d75ec541dd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-activate, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T14:37:00.740 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:37:00 smithi096 podman[115290]: 2024-09-13 14:37:00.588576046 +0000 UTC m=+0.111746610 container create bbf0998790c29e4d23f884f8270db9338a3cb8b8571e5d34badf28ca2ca470f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-13T14:37:00.740 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:37:00 smithi096 podman[115290]: 2024-09-13 14:37:00.492317947 +0000 UTC m=+0.015488468 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:37:01.072 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:37:00 smithi096 podman[115290]: 2024-09-13 14:37:00.815907819 +0000 UTC m=+0.339078340 container init bbf0998790c29e4d23f884f8270db9338a3cb8b8571e5d34badf28ca2ca470f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-13T14:37:01.072 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:37:00 smithi096 podman[115290]: 2024-09-13 14:37:00.819181842 +0000 UTC m=+0.342352366 container start bbf0998790c29e4d23f884f8270db9338a3cb8b8571e5d34badf28ca2ca470f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:37:01.072 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:37:00 smithi096 ceph-osd[115308]: -- 172.21.15.96:0/1438381272 <== mon.2 v2:172.21.15.96:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b40a8c7a40 con 0x55b40b696000 2024-09-13T14:37:01.073 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:37:00 smithi096 bash[115290]: bbf0998790c29e4d23f884f8270db9338a3cb8b8571e5d34badf28ca2ca470f6 2024-09-13T14:37:01.073 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:37:00 smithi096 systemd[1]: Started Ceph osd.1 for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:37:01.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:01 smithi204 ceph-mon[87071]: pgmap v145: 73 pgs: 2 active+undersized, 27 peering, 2 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:37:01.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:01 smithi204 ceph-mon[87071]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:01.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:01 smithi204 ceph-mon[87071]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:01 smithi204 ceph-mon[87071]: from='client.44213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:01 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:01 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:01 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:01.529 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:01 smithi096 ceph-mon[98901]: pgmap v145: 73 pgs: 2 active+undersized, 27 peering, 2 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:37:01.529 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:01 smithi096 ceph-mon[98901]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:01.529 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:01 smithi096 ceph-mon[98901]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:01.529 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:01 smithi096 ceph-mon[98901]: from='client.44213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:01.529 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:01 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:01.530 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:01 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:01.530 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:01 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:01.530 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:01 smithi096 ceph-mon[102928]: pgmap v145: 73 pgs: 2 active+undersized, 27 peering, 2 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:37:01.530 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:01 smithi096 ceph-mon[102928]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:01.530 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:01 smithi096 ceph-mon[102928]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:01.530 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:01 smithi096 ceph-mon[102928]: from='client.44213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:01.530 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:01 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:01.530 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:01 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:01.530 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:01 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:02.435 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:02 smithi096 ceph-mon[98901]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 27 pgs peering) 2024-09-13T14:37:02.435 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:02 smithi096 ceph-mon[102928]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 27 pgs peering) 2024-09-13T14:37:02.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:02 smithi204 ceph-mon[87071]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 27 pgs peering) 2024-09-13T14:37:02.822 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:37:02 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1[115304]: 2024-09-13T14:37:02.464+0000 7f1aaa6b8640 -1 Falling back to public interface 2024-09-13T14:37:02.948 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:03.276 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:03 smithi096 ceph-mon[102928]: pgmap v146: 73 pgs: 26 active+undersized, 5 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:37:03.276 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:03 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3935117637' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:03.277 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:03 smithi096 ceph-mon[98901]: pgmap v146: 73 pgs: 26 active+undersized, 5 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:37:03.277 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:03 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3935117637' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:03.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:03 smithi204 ceph-mon[87071]: pgmap v146: 73 pgs: 26 active+undersized, 5 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:37:03.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:03 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3935117637' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:04.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:04 smithi204 ceph-mon[87071]: from='client.34359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:04.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:04 smithi204 ceph-mon[87071]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:04.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:04 smithi096 ceph-mon[98901]: from='client.34359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:04.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:04 smithi096 ceph-mon[98901]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:04.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:04 smithi096 ceph-mon[102928]: from='client.34359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:04.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:04 smithi096 ceph-mon[102928]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:05.660 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:05 smithi096 ceph-mon[98901]: pgmap v147: 73 pgs: 26 active+undersized, 5 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:37:05.660 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:05 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:05.661 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:05 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:05.661 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:05 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:05.661 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:05 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:05.661 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:05 smithi096 ceph-mon[102928]: pgmap v147: 73 pgs: 26 active+undersized, 5 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:37:05.661 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:05 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:05.661 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:05 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:05.661 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:05 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:05.661 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:05 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:05 smithi204 ceph-mon[87071]: pgmap v147: 73 pgs: 26 active+undersized, 5 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:37:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:05 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:05 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:05 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:05 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:05.848 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:06.322 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:37:06 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1[115304]: 2024-09-13T14:37:06.040+0000 7f1aaa6b8640 -1 osd.1 90 log_to_monitors true 2024-09-13T14:37:06.644 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:06 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3248241619' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:06.645 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:06.645 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:06.645 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:06 smithi096 ceph-mon[102928]: from='osd.1 [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:37:06.645 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:06 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3248241619' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:06.645 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:06.645 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:06.645 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:06 smithi096 ceph-mon[98901]: from='osd.1 [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:37:06.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:06 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3248241619' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:06.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:06 smithi204 ceph-mon[87071]: from='osd.1 [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:37:07.572 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1[115304]: 2024-09-13T14:37:07.273+0000 7f1a9e45a640 -1 osd.1 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:37:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-mon[98901]: pgmap v148: 73 pgs: 26 active+undersized, 5 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:37:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-mon[98901]: from='client.34377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-mon[98901]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-mon[98901]: from='osd.1 [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:37:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-mon[98901]: osdmap e93: 8 total, 7 up, 8 in 2024-09-13T14:37:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-mon[98901]: from='osd.1 [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:37:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-mon[98901]: from='osd.1 [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923]' entity='osd.1' 2024-09-13T14:37:07.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-mon[102928]: pgmap v148: 73 pgs: 26 active+undersized, 5 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:37:07.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-mon[102928]: from='client.34377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:07.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-mon[102928]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:07.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-mon[102928]: from='osd.1 [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:37:07.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-mon[102928]: osdmap e93: 8 total, 7 up, 8 in 2024-09-13T14:37:07.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-mon[102928]: from='osd.1 [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:37:07.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-mon[102928]: from='osd.1 [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923]' entity='osd.1' 2024-09-13T14:37:07.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:07 smithi204 ceph-mon[87071]: pgmap v148: 73 pgs: 26 active+undersized, 5 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:37:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:07 smithi204 ceph-mon[87071]: from='client.34377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:07 smithi204 ceph-mon[87071]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:07 smithi204 ceph-mon[87071]: from='osd.1 [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:37:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:07 smithi204 ceph-mon[87071]: osdmap e93: 8 total, 7 up, 8 in 2024-09-13T14:37:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:07 smithi204 ceph-mon[87071]: from='osd.1 [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:37:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:07 smithi204 ceph-mon[87071]: from='osd.1 [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923]' entity='osd.1' 2024-09-13T14:37:08.227 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:37:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:37:07] "GET /metrics HTTP/1.1" 200 33973 "" "Prometheus/2.51.0" 2024-09-13T14:37:08.227 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:37:08 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:37:08.226+0000 7ff04b218640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-13T14:37:08.717 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:09.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: pgmap v150: 73 pgs: 26 active+undersized, 5 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:37:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-13T14:37:09.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:37:09.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: osd.1 [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923] boot 2024-09-13T14:37:09.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T14:37:09.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:37:09.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2829861451' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:09.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:09 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: pgmap v150: 73 pgs: 26 active+undersized, 5 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:37:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:09.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:09.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:09.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-13T14:37:09.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:37:09.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: osd.1 [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923] boot 2024-09-13T14:37:09.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T14:37:09.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:37:09.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2829861451' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:09.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:09.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: pgmap v150: 73 pgs: 26 active+undersized, 5 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:37:09.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:09.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:09.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:09.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:09.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:09.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:09.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:09.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:09.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:09.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:09.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:09.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-13T14:37:09.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:37:09.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: osd.1 [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923] boot 2024-09-13T14:37:09.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T14:37:09.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:37:09.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2829861451' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:09.576 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:09 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:10.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:10 smithi204 ceph-mon[87071]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:10 smithi204 ceph-mon[87071]: from='client.44273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:10 smithi204 ceph-mon[87071]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T14:37:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:37:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:37:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:10.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:10 smithi096 ceph-mon[98901]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:10.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:10 smithi096 ceph-mon[98901]: from='client.44273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:10.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:10 smithi096 ceph-mon[98901]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T14:37:10.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:10 smithi096 ceph-mon[102928]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:10.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:10 smithi096 ceph-mon[102928]: from='client.44273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:10.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:10 smithi096 ceph-mon[102928]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T14:37:11.557 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:11.558 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:11 smithi096 ceph-mon[102928]: pgmap v153: 73 pgs: 26 active+undersized, 5 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:37:11.558 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:11 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1127932861' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:11.558 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:11 smithi096 ceph-mon[98901]: pgmap v153: 73 pgs: 26 active+undersized, 5 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:37:11.558 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:11 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1127932861' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:11 smithi204 ceph-mon[87071]: pgmap v153: 73 pgs: 26 active+undersized, 5 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:37:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:11 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1127932861' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:13.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:13 smithi096 ceph-mon[98901]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:13.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:13 smithi096 ceph-mon[98901]: pgmap v154: 73 pgs: 11 active+undersized, 2 active+undersized+degraded, 60 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:37:13.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:13 smithi096 ceph-mon[98901]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:13.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:13 smithi096 ceph-mon[102928]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:13.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:13 smithi096 ceph-mon[102928]: pgmap v154: 73 pgs: 11 active+undersized, 2 active+undersized+degraded, 60 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:37:13.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:13 smithi096 ceph-mon[102928]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:13.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:13 smithi204 ceph-mon[87071]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:13.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:13 smithi204 ceph-mon[87071]: pgmap v154: 73 pgs: 11 active+undersized, 2 active+undersized+degraded, 60 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:37:13.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:13 smithi204 ceph-mon[87071]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:14.398 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:14.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:14 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:14.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:14 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3979607201' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:14.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:14 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:14.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:14 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3979607201' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:14.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:14 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:14.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:14 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3979607201' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:15.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:15 smithi096 ceph-mon[98901]: pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:15.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:15 smithi096 ceph-mon[98901]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-09-13T14:37:15.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:15 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:37:15.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:15 smithi096 ceph-mon[98901]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:15.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:15 smithi096 ceph-mon[102928]: pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:15.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:15 smithi096 ceph-mon[102928]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-09-13T14:37:15.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:15 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:37:15.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:15 smithi096 ceph-mon[102928]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:15.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:15 smithi204 ceph-mon[87071]: pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:15.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:15 smithi204 ceph-mon[87071]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-09-13T14:37:15.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:15 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:37:15.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:15 smithi204 ceph-mon[87071]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:16.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:16 smithi096 ceph-mon[102928]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:16.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:16 smithi096 ceph-mon[98901]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:16.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:16 smithi204 ceph-mon[87071]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:17.285 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:17.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:17 smithi096 ceph-mon[98901]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:17.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:17 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/860704111' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:17.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:17 smithi096 ceph-mon[102928]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:17.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:17 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/860704111' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:17.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:17 smithi204 ceph-mon[87071]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:17.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:17 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/860704111' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:18.302 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:37:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:37:17] "GET /metrics HTTP/1.1" 200 33973 "" "Prometheus/2.51.0" 2024-09-13T14:37:18.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:18 smithi096 ceph-mon[98901]: from='client.44321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:18.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:18 smithi096 ceph-mon[102928]: from='client.44321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:18.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:18 smithi204 ceph-mon[87071]: from='client.44321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:19.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:19 smithi096 ceph-mon[98901]: from='client.44327 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:19.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:19 smithi096 ceph-mon[98901]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:19.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:19 smithi096 ceph-mon[102928]: from='client.44327 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:19.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:19 smithi096 ceph-mon[102928]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:19.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:19 smithi204 ceph-mon[87071]: from='client.44327 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:19.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:19 smithi204 ceph-mon[87071]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:20.131 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:20.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:20 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2224210581' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:20.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:37:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:37:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:20.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:20 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2224210581' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:20.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:20 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2224210581' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:21.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:21 smithi096 ceph-mon[98901]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:21.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:21 smithi096 ceph-mon[98901]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:21.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:21 smithi096 ceph-mon[98901]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:21.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:21.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:21.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:21 smithi096 ceph-mon[102928]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:21 smithi096 ceph-mon[102928]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:21 smithi096 ceph-mon[102928]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:21.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:21 smithi204 ceph-mon[87071]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:21.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:21 smithi204 ceph-mon[87071]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:21.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:21 smithi204 ceph-mon[87071]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:21.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:21.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:21.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:22.977 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:23.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:23 smithi096 ceph-mon[98901]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:23.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:23 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/184706941' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:23.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:23 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:23.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:23 smithi096 ceph-mon[102928]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:23.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:23 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/184706941' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:23.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:23 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:23.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:23 smithi204 ceph-mon[87071]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:23.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:23 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/184706941' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:23.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:23 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:24.610 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:24 smithi096 ceph-mon[98901]: from='client.44357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:24.611 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:24 smithi096 ceph-mon[98901]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:24.611 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:24 smithi096 ceph-mon[98901]: Upgrade: osd.2 is safe to restart 2024-09-13T14:37:24.611 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:24 smithi096 ceph-mon[98901]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:24.611 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:24 smithi096 ceph-mon[102928]: from='client.44357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:24.611 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:24 smithi096 ceph-mon[102928]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:24.611 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:24 smithi096 ceph-mon[102928]: Upgrade: osd.2 is safe to restart 2024-09-13T14:37:24.611 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:24 smithi096 ceph-mon[102928]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:24.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:24 smithi204 ceph-mon[87071]: from='client.44357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:24.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:24 smithi204 ceph-mon[87071]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:24.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:24 smithi204 ceph-mon[87071]: Upgrade: osd.2 is safe to restart 2024-09-13T14:37:24.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:24 smithi204 ceph-mon[87071]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:25.701 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:25 smithi096 ceph-mon[98901]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:25.701 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:25 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:25.701 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:25 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:37:25.701 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:25 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:25.702 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:25 smithi096 ceph-mon[102928]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:25.702 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:25 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:25.702 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:25 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:37:25.702 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:25 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:25.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:25 smithi204 ceph-mon[87071]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:25.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:25 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:25.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:25 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:37:25.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:25 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:25.820 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:26.433 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:26 smithi096 ceph-mon[98901]: Upgrade: Updating osd.2 2024-09-13T14:37:26.433 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:26 smithi096 ceph-mon[98901]: Deploying daemon osd.2 on smithi096 2024-09-13T14:37:26.433 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:26 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1038002291' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:26.433 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:26 smithi096 ceph-mon[102928]: Upgrade: Updating osd.2 2024-09-13T14:37:26.434 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:26 smithi096 ceph-mon[102928]: Deploying daemon osd.2 on smithi096 2024-09-13T14:37:26.434 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:26 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1038002291' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:26.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:26 smithi204 ceph-mon[87071]: Upgrade: Updating osd.2 2024-09-13T14:37:26.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:26 smithi204 ceph-mon[87071]: Deploying daemon osd.2 on smithi096 2024-09-13T14:37:26.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:26 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1038002291' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:27.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:27 smithi204 ceph-mon[87071]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:27.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:27 smithi204 ceph-mon[87071]: from='client.44375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:27.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:27 smithi204 ceph-mon[87071]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:27.766 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:27 smithi096 ceph-mon[98901]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:27.766 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:27 smithi096 ceph-mon[98901]: from='client.44375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:27.766 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:27 smithi096 ceph-mon[98901]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:27.766 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:27 smithi096 ceph-mon[102928]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:27.766 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:27 smithi096 ceph-mon[102928]: from='client.44375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:27.766 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:27 smithi096 ceph-mon[102928]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:28.024 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:37:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:37:27] "GET /metrics HTTP/1.1" 200 33967 "" "Prometheus/2.51.0" 2024-09-13T14:37:28.025 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:27 smithi096 systemd[1]: Stopping Ceph osd.2 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:37:28.322 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:28 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2[55161]: 2024-09-13T14:37:28.037+0000 7f0892eeb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:37:28.322 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:28 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2[55161]: 2024-09-13T14:37:28.037+0000 7f0892eeb640 -1 osd.2 95 *** Got signal Terminated *** 2024-09-13T14:37:28.322 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:28 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2[55161]: 2024-09-13T14:37:28.037+0000 7f0892eeb640 -1 osd.2 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:37:28.704 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:28 smithi096 ceph-mon[98901]: osd.2 marked itself down and dead 2024-09-13T14:37:28.705 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:28 smithi096 ceph-mon[102928]: osd.2 marked itself down and dead 2024-09-13T14:37:28.705 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:28.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:28 smithi204 ceph-mon[87071]: osd.2 marked itself down and dead 2024-09-13T14:37:29.037 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:28 smithi096 podman[119958]: 2024-09-13 14:37:28.750357572 +0000 UTC m=+0.826839379 container died 565505fd2e48a70ab02d71ff1d6b0ccbe4afd177c4be02995f64e6c96cbb56c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True) 2024-09-13T14:37:29.322 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:29 smithi096 podman[119958]: 2024-09-13 14:37:29.147611972 +0000 UTC m=+1.224093777 container cleanup 565505fd2e48a70ab02d71ff1d6b0ccbe4afd177c4be02995f64e6c96cbb56c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:37:29.322 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:29 smithi096 bash[119958]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2 2024-09-13T14:37:29.707 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:29 smithi096 ceph-mon[98901]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:29.708 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:29 smithi096 ceph-mon[98901]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:37:29.708 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:29 smithi096 ceph-mon[98901]: osdmap e96: 8 total, 7 up, 8 in 2024-09-13T14:37:29.708 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:29 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3672349209' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:29.708 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:29 smithi096 ceph-mon[102928]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:29.708 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:29 smithi096 ceph-mon[102928]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:37:29.708 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:29 smithi096 ceph-mon[102928]: osdmap e96: 8 total, 7 up, 8 in 2024-09-13T14:37:29.709 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:29 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3672349209' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:29.709 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:29 smithi096 podman[120034]: 2024-09-13 14:37:29.391009386 +0000 UTC m=+0.644136797 container remove 565505fd2e48a70ab02d71ff1d6b0ccbe4afd177c4be02995f64e6c96cbb56c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:37:29.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:29 smithi204 ceph-mon[87071]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:29.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:29 smithi204 ceph-mon[87071]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:37:29.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:29 smithi204 ceph-mon[87071]: osdmap e96: 8 total, 7 up, 8 in 2024-09-13T14:37:29.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:29 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3672349209' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:30.036 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:29 smithi096 podman[120143]: 2024-09-13 14:37:29.610100402 +0000 UTC m=+0.016293105 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:37:30.036 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:29 smithi096 podman[120143]: 2024-09-13 14:37:29.72575516 +0000 UTC m=+0.131947895 container create e31f158c83f6a4ea0907d97638d90039e75935e4c5754b3d46c0befb78b787bb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:37:30.036 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:29 smithi096 podman[120143]: 2024-09-13 14:37:29.935820982 +0000 UTC m=+0.342013672 container init e31f158c83f6a4ea0907d97638d90039e75935e4c5754b3d46c0befb78b787bb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:37:30.036 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:29 smithi096 podman[120143]: 2024-09-13 14:37:29.939243702 +0000 UTC m=+0.345436394 container start e31f158c83f6a4ea0907d97638d90039e75935e4c5754b3d46c0befb78b787bb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS) 2024-09-13T14:37:30.036 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:30 smithi096 podman[120143]: 2024-09-13 14:37:30.035754369 +0000 UTC m=+0.441947061 container attach e31f158c83f6a4ea0907d97638d90039e75935e4c5754b3d46c0befb78b787bb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-13T14:37:30.322 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:30 smithi096 podman[120143]: 2024-09-13 14:37:30.130691539 +0000 UTC m=+0.536884242 container died e31f158c83f6a4ea0907d97638d90039e75935e4c5754b3d46c0befb78b787bb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-13T14:37:30.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:30 smithi204 ceph-mon[87071]: from='client.44393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:30.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:30 smithi204 ceph-mon[87071]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:30.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:30 smithi204 ceph-mon[87071]: osdmap e97: 8 total, 7 up, 8 in 2024-09-13T14:37:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:37:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:37:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:30.698 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:30 smithi096 ceph-mon[98901]: from='client.44393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:30.698 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:30 smithi096 ceph-mon[98901]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:30.698 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:30 smithi096 ceph-mon[98901]: osdmap e97: 8 total, 7 up, 8 in 2024-09-13T14:37:30.698 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:30 smithi096 ceph-mon[102928]: from='client.44393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:30.698 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:30 smithi096 ceph-mon[102928]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:30.698 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:30 smithi096 ceph-mon[102928]: osdmap e97: 8 total, 7 up, 8 in 2024-09-13T14:37:31.072 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:30 smithi096 podman[120167]: 2024-09-13 14:37:30.697881932 +0000 UTC m=+0.558621667 container remove e31f158c83f6a4ea0907d97638d90039e75935e4c5754b3d46c0befb78b787bb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-13T14:37:31.073 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:30 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-13T14:37:31.073 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:30 smithi096 systemd[1]: Stopped Ceph osd.2 for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:37:31.073 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:30 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.2.service: Consumed 5.338s CPU time. 2024-09-13T14:37:31.073 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:30 smithi096 systemd[1]: Starting Ceph osd.2 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:37:31.468 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-mon[98901]: pgmap v165: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:31.468 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1791044221' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:31.473 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-mon[102928]: pgmap v165: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:31.473 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1791044221' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:31.473 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 podman[120266]: 2024-09-13 14:37:31.030114804 +0000 UTC m=+0.016904481 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:37:31.473 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 podman[120266]: 2024-09-13 14:37:31.166084207 +0000 UTC m=+0.152873887 container create a1f3ef4e72a1b546df5097b0df8747399f2c48720fd78b26a3b9b6c0f1f6bfc0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:37:31.473 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 podman[120266]: 2024-09-13 14:37:31.360911801 +0000 UTC m=+0.347701481 container init a1f3ef4e72a1b546df5097b0df8747399f2c48720fd78b26a3b9b6c0f1f6bfc0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:37:31.474 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 podman[120266]: 2024-09-13 14:37:31.365304579 +0000 UTC m=+0.352094261 container start a1f3ef4e72a1b546df5097b0df8747399f2c48720fd78b26a3b9b6c0f1f6bfc0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-13T14:37:31.474 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 podman[120266]: 2024-09-13 14:37:31.468459592 +0000 UTC m=+0.455249262 container attach a1f3ef4e72a1b546df5097b0df8747399f2c48720fd78b26a3b9b6c0f1f6bfc0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-13T14:37:31.585 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:31.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:31 smithi204 ceph-mon[87071]: pgmap v165: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:37:31.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:31 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1791044221' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:31.822 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate[120302]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:37:31.822 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 bash[120266]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:37:31.823 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate[120302]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:37:31.823 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 bash[120266]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:37:31.823 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate[120302]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:37:31.823 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 bash[120266]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:37:31.823 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate[120302]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:37:31.823 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 bash[120266]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:37:31.823 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate[120302]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:37:31.823 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 bash[120266]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:37:31.823 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate[120302]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:37:31.823 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 bash[120266]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:37:31.823 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate[120302]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-13T14:37:31.823 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 bash[120266]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-13T14:37:32.281 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate[120302]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:37:32.281 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 bash[120266]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:37:32.282 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate[120302]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:37:32.282 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 bash[120266]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:37:32.282 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate[120302]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T14:37:32.282 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 bash[120266]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T14:37:32.282 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate[120302]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:37:32.282 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 bash[120266]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:37:32.282 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate[120302]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-13T14:37:32.282 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 bash[120266]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-13T14:37:32.282 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:31 smithi096 podman[120266]: 2024-09-13 14:37:31.941026459 +0000 UTC m=+0.927816135 container died a1f3ef4e72a1b546df5097b0df8747399f2c48720fd78b26a3b9b6c0f1f6bfc0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=) 2024-09-13T14:37:32.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:32 smithi096 ceph-mon[98901]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:32.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:32 smithi096 ceph-mon[102928]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:32.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:32 smithi204 ceph-mon[87071]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:32.946 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:32 smithi096 podman[120474]: 2024-09-13 14:37:32.589584344 +0000 UTC m=+0.642307275 container remove a1f3ef4e72a1b546df5097b0df8747399f2c48720fd78b26a3b9b6c0f1f6bfc0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:37:32.947 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:32 smithi096 podman[120533]: 2024-09-13 14:37:32.69816215 +0000 UTC m=+0.015682774 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:37:32.947 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:32 smithi096 podman[120533]: 2024-09-13 14:37:32.823619705 +0000 UTC m=+0.141140322 container create eaf4f3b21ff8e9dda6f29cf9195bbdd44fb39f39136edae9fd9c90aee7453100 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:37:33.324 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:33 smithi096 podman[120533]: 2024-09-13 14:37:33.076286084 +0000 UTC m=+0.393806706 container init eaf4f3b21ff8e9dda6f29cf9195bbdd44fb39f39136edae9fd9c90aee7453100 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:37:33.324 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:33 smithi096 podman[120533]: 2024-09-13 14:37:33.079543228 +0000 UTC m=+0.397063844 container start eaf4f3b21ff8e9dda6f29cf9195bbdd44fb39f39136edae9fd9c90aee7453100 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-13T14:37:33.324 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:33 smithi096 ceph-osd[120552]: -- 172.21.15.96:0/813615431 <== mon.2 v2:172.21.15.96:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561f60e93a40 con 0x561f60e71c00 2024-09-13T14:37:33.324 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:33 smithi096 bash[120533]: eaf4f3b21ff8e9dda6f29cf9195bbdd44fb39f39136edae9fd9c90aee7453100 2024-09-13T14:37:33.324 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:33 smithi096 systemd[1]: Started Ceph osd.2 for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:37:33.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:33 smithi204 ceph-mon[87071]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:33.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:33 smithi204 ceph-mon[87071]: pgmap v166: 73 pgs: 9 active+undersized, 1 stale+active+clean, 3 active+undersized+degraded, 60 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:37:33.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:33 smithi204 ceph-mon[87071]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:33.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:33.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:33.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:33 smithi096 ceph-mon[98901]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:33 smithi096 ceph-mon[98901]: pgmap v166: 73 pgs: 9 active+undersized, 1 stale+active+clean, 3 active+undersized+degraded, 60 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:37:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:33 smithi096 ceph-mon[98901]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:33.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:33 smithi096 ceph-mon[102928]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:33.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:33 smithi096 ceph-mon[102928]: pgmap v166: 73 pgs: 9 active+undersized, 1 stale+active+clean, 3 active+undersized+degraded, 60 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:37:33.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:33 smithi096 ceph-mon[102928]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:33.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:33.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:33.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:34.459 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:34.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:34 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3922780860' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:34.747 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:34 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3922780860' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:34.747 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:34 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3922780860' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:34.748 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:34 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2[120548]: 2024-09-13T14:37:34.746+0000 7f230f6ae640 -1 Falling back to public interface 2024-09-13T14:37:35.708 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:35 smithi096 ceph-mon[98901]: pgmap v167: 73 pgs: 18 active+undersized, 5 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7/84 objects degraded (8.333%) 2024-09-13T14:37:35.708 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:35 smithi096 ceph-mon[102928]: pgmap v167: 73 pgs: 18 active+undersized, 5 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7/84 objects degraded (8.333%) 2024-09-13T14:37:35.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:35 smithi204 ceph-mon[87071]: pgmap v167: 73 pgs: 18 active+undersized, 5 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7/84 objects degraded (8.333%) 2024-09-13T14:37:36.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:36 smithi096 ceph-mon[98901]: from='client.44432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:36.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:36 smithi096 ceph-mon[98901]: from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:36.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:36.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:36.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:36 smithi096 ceph-mon[102928]: from='client.44432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:36.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:36 smithi096 ceph-mon[102928]: from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:36.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:36.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:36.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:36 smithi204 ceph-mon[87071]: from='client.44432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:36 smithi204 ceph-mon[87071]: from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:37.367 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:37.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:37 smithi204 ceph-mon[87071]: pgmap v168: 73 pgs: 18 active+undersized, 5 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7/84 objects degraded (8.333%) 2024-09-13T14:37:37.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:37 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:37.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:37 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:37.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:37 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2566851831' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:37.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:37 smithi096 ceph-mon[98901]: pgmap v168: 73 pgs: 18 active+undersized, 5 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7/84 objects degraded (8.333%) 2024-09-13T14:37:37.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:37 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:37.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:37 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:37.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:37 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2566851831' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:37.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:37 smithi096 ceph-mon[102928]: pgmap v168: 73 pgs: 18 active+undersized, 5 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7/84 objects degraded (8.333%) 2024-09-13T14:37:37.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:37 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:37.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:37 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:37.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:37 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2566851831' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:38.172 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:37:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:37:37] "GET /metrics HTTP/1.1" 200 33966 "" "Prometheus/2.51.0" 2024-09-13T14:37:38.428 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:38 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2[120548]: 2024-09-13T14:37:38.292+0000 7f230f6ae640 -1 osd.2 95 log_to_monitors true 2024-09-13T14:37:38.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:38 smithi204 ceph-mon[87071]: from='client.34581 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:38.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:38.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:38.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:38 smithi204 ceph-mon[87071]: from='osd.2 [v2:172.21.15.96:6818/4202655894,v1:172.21.15.96:6819/4202655894]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:37:38.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:38 smithi204 ceph-mon[87071]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:37:38.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:38 smithi096 ceph-mon[102928]: from='client.34581 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:38.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:38.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:38.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:38 smithi096 ceph-mon[102928]: from='osd.2 [v2:172.21.15.96:6818/4202655894,v1:172.21.15.96:6819/4202655894]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:37:38.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:38 smithi096 ceph-mon[102928]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:37:38.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:38 smithi096 ceph-mon[98901]: from='client.34581 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:38.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:38.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:38.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:38 smithi096 ceph-mon[98901]: from='osd.2 [v2:172.21.15.96:6818/4202655894,v1:172.21.15.96:6819/4202655894]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:37:38.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:38 smithi096 ceph-mon[98901]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:37:39.251 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 14:37:38 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2[120548]: 2024-09-13T14:37:38.918+0000 7f2303450640 -1 osd.2 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:37:39.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[102928]: pgmap v169: 73 pgs: 18 active+undersized, 5 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7/84 objects degraded (8.333%) 2024-09-13T14:37:39.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[102928]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:39.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[102928]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:37:39.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[102928]: osdmap e98: 8 total, 7 up, 8 in 2024-09-13T14:37:39.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[102928]: from='osd.2 [v2:172.21.15.96:6818/4202655894,v1:172.21.15.96:6819/4202655894]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:37:39.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[102928]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:37:39.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[102928]: from='osd.2 ' entity='osd.2' 2024-09-13T14:37:39.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 7/84 objects degraded (8.333%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:39.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[98901]: pgmap v169: 73 pgs: 18 active+undersized, 5 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7/84 objects degraded (8.333%) 2024-09-13T14:37:39.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[98901]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:39.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[98901]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:37:39.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[98901]: osdmap e98: 8 total, 7 up, 8 in 2024-09-13T14:37:39.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[98901]: from='osd.2 [v2:172.21.15.96:6818/4202655894,v1:172.21.15.96:6819/4202655894]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:37:39.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[98901]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:37:39.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[98901]: from='osd.2 ' entity='osd.2' 2024-09-13T14:37:39.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:39 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 7/84 objects degraded (8.333%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:39.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:39 smithi204 ceph-mon[87071]: pgmap v169: 73 pgs: 18 active+undersized, 5 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7/84 objects degraded (8.333%) 2024-09-13T14:37:39.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:39 smithi204 ceph-mon[87071]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:39.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:39 smithi204 ceph-mon[87071]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:37:39.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:39 smithi204 ceph-mon[87071]: osdmap e98: 8 total, 7 up, 8 in 2024-09-13T14:37:39.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:39 smithi204 ceph-mon[87071]: from='osd.2 [v2:172.21.15.96:6818/4202655894,v1:172.21.15.96:6819/4202655894]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:37:39.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:39 smithi204 ceph-mon[87071]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:37:39.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:39 smithi204 ceph-mon[87071]: from='osd.2 ' entity='osd.2' 2024-09-13T14:37:39.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:39 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 7/84 objects degraded (8.333%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:40.252 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:40.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:40 smithi204 ceph-mon[87071]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:37:40.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:40 smithi204 ceph-mon[87071]: osd.2 [v2:172.21.15.96:6818/4202655894,v1:172.21.15.96:6819/4202655894] boot 2024-09-13T14:37:40.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:40 smithi204 ceph-mon[87071]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T14:37:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:40 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:37:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:40 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/431000871' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:37:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:37:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:40.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:40 smithi096 ceph-mon[98901]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:37:40.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:40 smithi096 ceph-mon[98901]: osd.2 [v2:172.21.15.96:6818/4202655894,v1:172.21.15.96:6819/4202655894] boot 2024-09-13T14:37:40.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:40 smithi096 ceph-mon[98901]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T14:37:40.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:40 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:37:40.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:40 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/431000871' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:40.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:40 smithi096 ceph-mon[102928]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:37:40.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:40 smithi096 ceph-mon[102928]: osd.2 [v2:172.21.15.96:6818/4202655894,v1:172.21.15.96:6819/4202655894] boot 2024-09-13T14:37:40.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:40 smithi096 ceph-mon[102928]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T14:37:40.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:40 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:37:40.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:40 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/431000871' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:41.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:37:40 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:37:40.824+0000 7ff04b218640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-13T14:37:41.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: pgmap v171: 73 pgs: 18 active+undersized, 5 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7/84 objects degraded (8.333%) 2024-09-13T14:37:41.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:41.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:41.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:41.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:41.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:41.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:41.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:41.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:41.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:41.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:41.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:41.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:37:41.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:37:41.722 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-13T14:37:41.722 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:41 smithi204 ceph-mon[87071]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T14:37:41.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: pgmap v171: 73 pgs: 18 active+undersized, 5 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7/84 objects degraded (8.333%) 2024-09-13T14:37:41.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:41.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:41.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:41.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:41.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:41.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:41.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:41.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:41.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:41.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:41.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:41.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:37:41.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:37:41.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-13T14:37:41.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[98901]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T14:37:41.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: pgmap v171: 73 pgs: 18 active+undersized, 5 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7/84 objects degraded (8.333%) 2024-09-13T14:37:41.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:41.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:41.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:41.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:41.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:41.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:41.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:41.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:41.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:41.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:41.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:41.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:37:41.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:37:41.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-13T14:37:41.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:41 smithi096 ceph-mon[102928]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T14:37:42.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:42 smithi204 ceph-mon[87071]: pgmap v174: 73 pgs: 10 active+undersized, 3 active+undersized+degraded, 60 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:37:42.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:42 smithi096 ceph-mon[98901]: pgmap v174: 73 pgs: 10 active+undersized, 3 active+undersized+degraded, 60 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:37:42.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:42 smithi096 ceph-mon[102928]: pgmap v174: 73 pgs: 10 active+undersized, 3 active+undersized+degraded, 60 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:37:43.110 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:43.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:43 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1534030704' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:43.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:43 smithi204 ceph-mon[87071]: from='client.44489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:43.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:43 smithi204 ceph-mon[87071]: from='client.44495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:43.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:43 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1534030704' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:43.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:43 smithi096 ceph-mon[98901]: from='client.44489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:43.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:43 smithi096 ceph-mon[98901]: from='client.44495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:43.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:43 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1534030704' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:43.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:43 smithi096 ceph-mon[102928]: from='client.44489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:43.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:43 smithi096 ceph-mon[102928]: from='client.44495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:44.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:44 smithi204 ceph-mon[87071]: pgmap v175: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:44.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:44 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:44.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:44 smithi096 ceph-mon[98901]: pgmap v175: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:44.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:44 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:44.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:44 smithi096 ceph-mon[102928]: pgmap v175: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:44.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:44 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:45.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:45 smithi204 ceph-mon[87071]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2024-09-13T14:37:45.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:45 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:37:45.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:45 smithi096 ceph-mon[98901]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2024-09-13T14:37:45.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:45 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:37:45.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:45 smithi096 ceph-mon[102928]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2024-09-13T14:37:45.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:45 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:37:45.944 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:46.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:46 smithi204 ceph-mon[87071]: pgmap v176: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:46.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:46 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4183679895' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:46.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:46 smithi204 ceph-mon[87071]: from='client.44507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:46.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:46 smithi204 ceph-mon[87071]: from='client.34644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:46.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:46 smithi096 ceph-mon[102928]: pgmap v176: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:46.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:46 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4183679895' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:46.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:46 smithi096 ceph-mon[102928]: from='client.44507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:46.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:46 smithi096 ceph-mon[102928]: from='client.34644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:46.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:46 smithi096 ceph-mon[98901]: pgmap v176: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:46.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:46 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4183679895' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:46.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:46 smithi096 ceph-mon[98901]: from='client.44507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:46.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:46 smithi096 ceph-mon[98901]: from='client.34644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:48.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:37:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:37:47] "GET /metrics HTTP/1.1" 200 33966 "" "Prometheus/2.51.0" 2024-09-13T14:37:48.788 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:48.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:48 smithi204 ceph-mon[87071]: pgmap v177: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:48.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:48 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4050653968' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:49.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:48 smithi096 ceph-mon[102928]: pgmap v177: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:49.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:48 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4050653968' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:49.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:48 smithi096 ceph-mon[98901]: pgmap v177: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:49.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:48 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4050653968' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:49.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:49 smithi204 ceph-mon[87071]: from='client.34656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:49.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:49 smithi204 ceph-mon[87071]: from='client.44531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:50.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:49 smithi096 ceph-mon[98901]: from='client.34656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:50.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:49 smithi096 ceph-mon[98901]: from='client.44531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:50.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:49 smithi096 ceph-mon[102928]: from='client.34656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:50.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:49 smithi096 ceph-mon[102928]: from='client.44531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:50.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:37:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:37:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:50.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:50 smithi204 ceph-mon[87071]: pgmap v178: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:51.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:50 smithi096 ceph-mon[98901]: pgmap v178: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:51.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:50 smithi096 ceph-mon[102928]: pgmap v178: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:51.636 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:52.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:52.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:52.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:52.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:51 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/265540092' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:51 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/265540092' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:52.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:52.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:52.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:52.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:51 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/265540092' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:53.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:52 smithi204 ceph-mon[87071]: pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:53.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:52 smithi204 ceph-mon[87071]: from='client.44543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:53.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:52 smithi204 ceph-mon[87071]: from='client.44549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:53.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:52 smithi096 ceph-mon[98901]: pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:53.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:52 smithi096 ceph-mon[98901]: from='client.44543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:53.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:52 smithi096 ceph-mon[98901]: from='client.44549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:53.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:52 smithi096 ceph-mon[102928]: pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:53.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:52 smithi096 ceph-mon[102928]: from='client.44543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:53.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:52 smithi096 ceph-mon[102928]: from='client.44549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:54.483 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:55.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:54 smithi204 ceph-mon[87071]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:55.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:54 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4066673612' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:55.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:54 smithi096 ceph-mon[98901]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:55.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:54 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4066673612' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:55.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:54 smithi096 ceph-mon[102928]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:55.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:54 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4066673612' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:56.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:55 smithi204 ceph-mon[87071]: from='client.34692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:56.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:55 smithi204 ceph-mon[87071]: from='client.34698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:56.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:55 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:37:56.222 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:55 smithi096 ceph-mon[102928]: from='client.34692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:56.222 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:55 smithi096 ceph-mon[102928]: from='client.34698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:56.222 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:55 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:37:56.222 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:55 smithi096 ceph-mon[98901]: from='client.34692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:56.222 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:55 smithi096 ceph-mon[98901]: from='client.34698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:56.222 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:55 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:37:57.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:56 smithi204 ceph-mon[87071]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:57.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:56 smithi204 ceph-mon[87071]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:37:57.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:56 smithi204 ceph-mon[87071]: Upgrade: osd.3 is safe to restart 2024-09-13T14:37:57.236 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:56 smithi096 ceph-mon[98901]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:57.236 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:56 smithi096 ceph-mon[98901]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:37:57.236 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:56 smithi096 ceph-mon[98901]: Upgrade: osd.3 is safe to restart 2024-09-13T14:37:57.236 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:56 smithi096 ceph-mon[102928]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:57.236 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:56 smithi096 ceph-mon[102928]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:37:57.236 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:56 smithi096 ceph-mon[102928]: Upgrade: osd.3 is safe to restart 2024-09-13T14:37:57.334 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:37:57.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:57 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2141929801' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:57.947 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:57 smithi096 ceph-mon[98901]: Upgrade: Updating osd.3 2024-09-13T14:37:57.947 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:57 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:57.947 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:57 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:37:57.947 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:57 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:57.947 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:57 smithi096 ceph-mon[98901]: Deploying daemon osd.3 on smithi096 2024-09-13T14:37:57.947 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:57 smithi096 ceph-mon[98901]: from='client.44579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:57 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2141929801' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:57 smithi096 ceph-mon[102928]: Upgrade: Updating osd.3 2024-09-13T14:37:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:57 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:57.948 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:57 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:37:57.948 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:57 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:57.948 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:57 smithi096 ceph-mon[102928]: Deploying daemon osd.3 on smithi096 2024-09-13T14:37:57.948 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:57 smithi096 ceph-mon[102928]: from='client.44579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:58.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:57 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2141929801' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:37:58.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:57 smithi204 ceph-mon[87071]: Upgrade: Updating osd.3 2024-09-13T14:37:58.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:57 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:37:58.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:57 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:37:58.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:57 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:58.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:57 smithi204 ceph-mon[87071]: Deploying daemon osd.3 on smithi096 2024-09-13T14:37:58.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:57 smithi204 ceph-mon[87071]: from='client.44579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:58.321 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:37:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:37:57] "GET /metrics HTTP/1.1" 200 33965 "" "Prometheus/2.51.0" 2024-09-13T14:37:59.460 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:59 smithi096 ceph-mon[98901]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:59.460 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:37:59 smithi096 ceph-mon[98901]: from='client.44585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:59.460 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:59 smithi096 ceph-mon[102928]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:59.461 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:37:59 smithi096 ceph-mon[102928]: from='client.44585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:59.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:59 smithi204 ceph-mon[87071]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:37:59.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:37:59 smithi204 ceph-mon[87071]: from='client.44585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:00.256 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:38:00.418 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:00 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1446219396' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:00.419 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:00 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1446219396' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:00.419 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:00 smithi096 systemd[1]: Stopping Ceph osd.3 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:38:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:00 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1446219396' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:00.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:38:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:38:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:00.822 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:00 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[61087]: 2024-09-13T14:38:00.416+0000 7fdb84c6a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:38:00.822 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:00 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[61087]: 2024-09-13T14:38:00.416+0000 7fdb84c6a640 -1 osd.3 100 *** Got signal Terminated *** 2024-09-13T14:38:00.823 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:00 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[61087]: 2024-09-13T14:38:00.416+0000 7fdb84c6a640 -1 osd.3 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:38:01.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:01 smithi096 ceph-mon[102928]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:01.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:01 smithi096 ceph-mon[102928]: from='client.34728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:01.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:01 smithi096 ceph-mon[102928]: osd.3 marked itself down and dead 2024-09-13T14:38:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:01 smithi096 ceph-mon[98901]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:01 smithi096 ceph-mon[98901]: from='client.34728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:01 smithi096 ceph-mon[98901]: osd.3 marked itself down and dead 2024-09-13T14:38:01.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:01 smithi204 ceph-mon[87071]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:01.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:01 smithi204 ceph-mon[87071]: from='client.34728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:01.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:01 smithi204 ceph-mon[87071]: osd.3 marked itself down and dead 2024-09-13T14:38:02.010 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:01 smithi096 podman[125704]: 2024-09-13 14:38:01.735165821 +0000 UTC m=+1.426637499 container died afd4dcb32d2dde46f61b4a23993afdcdb45e06a4553bcf752548fd4feb808e0b (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3, RELEASE=quincy-bbdadfa, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-13T14:38:02.277 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:02 smithi096 podman[125704]: 2024-09-13 14:38:02.108631992 +0000 UTC m=+1.800103656 container cleanup afd4dcb32d2dde46f61b4a23993afdcdb45e06a4553bcf752548fd4feb808e0b (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-13T14:38:02.277 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:02 smithi096 bash[125704]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3 2024-09-13T14:38:02.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:02 smithi096 ceph-mon[98901]: from='client.44603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:02.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:02 smithi096 ceph-mon[98901]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:38:02.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:02 smithi096 ceph-mon[98901]: osdmap e101: 8 total, 7 up, 8 in 2024-09-13T14:38:02.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:02 smithi096 ceph-mon[102928]: from='client.44603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:02.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:02 smithi096 ceph-mon[102928]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:38:02.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:02 smithi096 ceph-mon[102928]: osdmap e101: 8 total, 7 up, 8 in 2024-09-13T14:38:02.573 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:02 smithi096 podman[125743]: 2024-09-13 14:38:02.27645636 +0000 UTC m=+0.536048630 container remove afd4dcb32d2dde46f61b4a23993afdcdb45e06a4553bcf752548fd4feb808e0b (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2) 2024-09-13T14:38:02.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:02 smithi204 ceph-mon[87071]: from='client.44603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:02.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:02 smithi204 ceph-mon[87071]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:38:02.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:02 smithi204 ceph-mon[87071]: osdmap e101: 8 total, 7 up, 8 in 2024-09-13T14:38:03.039 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:02 smithi096 podman[125835]: 2024-09-13 14:38:02.627985637 +0000 UTC m=+0.016106641 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:38:03.040 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:02 smithi096 podman[125835]: 2024-09-13 14:38:02.744166318 +0000 UTC m=+0.132287312 container create cc8420b81c0202f9bf8c4b54ccc267b5b6c1b376b7ee6fae4f5f32676d761d34 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-13T14:38:03.040 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:02 smithi096 podman[125835]: 2024-09-13 14:38:02.938221123 +0000 UTC m=+0.326342120 container init cc8420b81c0202f9bf8c4b54ccc267b5b6c1b376b7ee6fae4f5f32676d761d34 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:38:03.040 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:02 smithi096 podman[125835]: 2024-09-13 14:38:02.941934791 +0000 UTC m=+0.330055794 container start cc8420b81c0202f9bf8c4b54ccc267b5b6c1b376b7ee6fae4f5f32676d761d34 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:38:03.134 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:38:03.302 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:03 smithi096 podman[125835]: 2024-09-13 14:38:03.038335885 +0000 UTC m=+0.426456894 container attach cc8420b81c0202f9bf8c4b54ccc267b5b6c1b376b7ee6fae4f5f32676d761d34 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:38:03.302 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:03 smithi096 podman[125835]: 2024-09-13 14:38:03.140715182 +0000 UTC m=+0.528836181 container died cc8420b81c0202f9bf8c4b54ccc267b5b6c1b376b7ee6fae4f5f32676d761d34 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-13T14:38:03.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:03 smithi096 ceph-mon[102928]: pgmap v185: 73 pgs: 12 peering, 6 stale+active+clean, 55 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:03.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:03 smithi096 ceph-mon[102928]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-13T14:38:03.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:03 smithi096 ceph-mon[102928]: osdmap e102: 8 total, 7 up, 8 in 2024-09-13T14:38:03.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:03 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3623627787' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:03.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:03 smithi096 ceph-mon[98901]: pgmap v185: 73 pgs: 12 peering, 6 stale+active+clean, 55 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:03 smithi096 ceph-mon[98901]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-13T14:38:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:03 smithi096 ceph-mon[98901]: osdmap e102: 8 total, 7 up, 8 in 2024-09-13T14:38:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:03 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3623627787' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:03.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:03 smithi204 ceph-mon[87071]: pgmap v185: 73 pgs: 12 peering, 6 stale+active+clean, 55 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:03.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:03 smithi204 ceph-mon[87071]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-13T14:38:03.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:03 smithi204 ceph-mon[87071]: osdmap e102: 8 total, 7 up, 8 in 2024-09-13T14:38:03.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:03 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3623627787' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:04.072 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:03 smithi096 podman[125906]: 2024-09-13 14:38:03.725740067 +0000 UTC m=+0.578970943 container remove cc8420b81c0202f9bf8c4b54ccc267b5b6c1b376b7ee6fae4f5f32676d761d34 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-13T14:38:04.072 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:03 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-13T14:38:04.072 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:03 smithi096 systemd[1]: Stopped Ceph osd.3 for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:38:04.072 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:03 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.3.service: Consumed 4.907s CPU time. 2024-09-13T14:38:04.072 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:03 smithi096 systemd[1]: Starting Ceph osd.3 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:38:04.430 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-mon[98901]: from='client.34746 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:04.430 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-mon[98901]: from='client.44621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:04.430 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-mon[102928]: from='client.34746 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:04.430 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-mon[102928]: from='client.44621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:04.430 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 podman[126026]: 2024-09-13 14:38:04.072025591 +0000 UTC m=+0.015499096 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:38:04.430 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 podman[126026]: 2024-09-13 14:38:04.176959081 +0000 UTC m=+0.120432582 container create 8eca092c98ea652e4aca489e8245d50e291e4812968b8481cf4b57a686772cf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:38:04.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:04 smithi204 ceph-mon[87071]: from='client.34746 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:04.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:04 smithi204 ceph-mon[87071]: from='client.44621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:04.823 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 podman[126026]: 2024-09-13 14:38:04.429667752 +0000 UTC m=+0.373141255 container init 8eca092c98ea652e4aca489e8245d50e291e4812968b8481cf4b57a686772cf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-13T14:38:04.823 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 podman[126026]: 2024-09-13 14:38:04.433309818 +0000 UTC m=+0.376783322 container start 8eca092c98ea652e4aca489e8245d50e291e4812968b8481cf4b57a686772cf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-13T14:38:04.823 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 podman[126026]: 2024-09-13 14:38:04.52960316 +0000 UTC m=+0.473076664 container attach 8eca092c98ea652e4aca489e8245d50e291e4812968b8481cf4b57a686772cf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T14:38:04.823 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[126041]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:38:04.823 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 bash[126026]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:38:04.823 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[126041]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:38:04.824 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 bash[126026]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:38:04.824 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[126041]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:38:04.824 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 bash[126026]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:38:04.824 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[126041]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:38:04.824 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 bash[126026]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:38:04.824 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[126041]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:38:04.824 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 bash[126026]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:38:04.824 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[126041]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:38:04.824 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 bash[126026]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:38:04.824 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[126041]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-13T14:38:04.824 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 bash[126026]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-13T14:38:05.283 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[126041]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:38:05.283 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 bash[126026]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:38:05.283 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[126041]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:38:05.283 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 bash[126026]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:38:05.284 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[126041]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T14:38:05.284 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 bash[126026]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T14:38:05.284 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[126041]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:38:05.284 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 bash[126026]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:38:05.284 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[126041]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-13T14:38:05.284 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 bash[126026]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-13T14:38:05.284 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:04 smithi096 podman[126026]: 2024-09-13 14:38:04.982290877 +0000 UTC m=+0.925764379 container died 8eca092c98ea652e4aca489e8245d50e291e4812968b8481cf4b57a686772cf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:38:05.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:05 smithi096 ceph-mon[98901]: pgmap v187: 73 pgs: 9 active+undersized, 18 peering, 1 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-09-13T14:38:05.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:05 smithi096 ceph-mon[98901]: Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:38:05.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:05 smithi096 ceph-mon[102928]: pgmap v187: 73 pgs: 9 active+undersized, 18 peering, 1 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-09-13T14:38:05.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:05 smithi096 ceph-mon[102928]: Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:38:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:05 smithi204 ceph-mon[87071]: pgmap v187: 73 pgs: 9 active+undersized, 18 peering, 1 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-09-13T14:38:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:05 smithi204 ceph-mon[87071]: Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:38:05.940 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:05 smithi096 podman[126147]: 2024-09-13 14:38:05.608428713 +0000 UTC m=+0.618226875 container remove 8eca092c98ea652e4aca489e8245d50e291e4812968b8481cf4b57a686772cf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-13T14:38:05.940 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:05 smithi096 podman[126228]: 2024-09-13 14:38:05.809679472 +0000 UTC m=+0.108437124 container create 23f73e3db413d5be7981f58e0a82848e994159130080718b7a2e98d36eb9897d (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:38:05.940 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:05 smithi096 podman[126228]: 2024-09-13 14:38:05.723539281 +0000 UTC m=+0.022296952 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:38:06.010 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:38:06.319 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:06 smithi096 podman[126228]: 2024-09-13 14:38:06.045709758 +0000 UTC m=+0.344467410 container init 23f73e3db413d5be7981f58e0a82848e994159130080718b7a2e98d36eb9897d (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-13T14:38:06.319 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:06 smithi096 podman[126228]: 2024-09-13 14:38:06.048930733 +0000 UTC m=+0.347688386 container start 23f73e3db413d5be7981f58e0a82848e994159130080718b7a2e98d36eb9897d (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731) 2024-09-13T14:38:06.319 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:06 smithi096 ceph-osd[126276]: -- 172.21.15.96:0/2399032959 <== mon.0 v2:172.21.15.96:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555d095c3a40 con 0x555d0a392000 2024-09-13T14:38:06.320 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:06 smithi096 bash[126228]: 23f73e3db413d5be7981f58e0a82848e994159130080718b7a2e98d36eb9897d 2024-09-13T14:38:06.320 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:06 smithi096 systemd[1]: Started Ceph osd.3 for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T14:38:06.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:06 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2855521166' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:06.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:06.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:06.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:06.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:06.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:06.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:06 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2855521166' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:06.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:06.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:06.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:06.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:06.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:06.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:06 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2855521166' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:06.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:07.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:07 smithi096 ceph-mon[98901]: pgmap v188: 73 pgs: 9 active+undersized, 18 peering, 1 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-09-13T14:38:07.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:07 smithi096 ceph-mon[98901]: from='client.34764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:07.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:07 smithi096 ceph-mon[98901]: from='client.34773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:07 smithi096 ceph-mon[102928]: pgmap v188: 73 pgs: 9 active+undersized, 18 peering, 1 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-09-13T14:38:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:07 smithi096 ceph-mon[102928]: from='client.34764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:07 smithi096 ceph-mon[102928]: from='client.34773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:07.643 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:07 smithi204 ceph-mon[87071]: pgmap v188: 73 pgs: 9 active+undersized, 18 peering, 1 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-09-13T14:38:07.643 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:07 smithi204 ceph-mon[87071]: from='client.34764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:07.643 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:07 smithi204 ceph-mon[87071]: from='client.34773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:07.946 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[126267]: 2024-09-13T14:38:07.669+0000 7fa5049f9640 -1 Falling back to public interface 2024-09-13T14:38:08.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:38:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:38:07] "GET /metrics HTTP/1.1" 200 34033 "" "Prometheus/2.51.0" 2024-09-13T14:38:08.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:08 smithi204 ceph-mon[87071]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-09-13T14:38:08.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:08 smithi096 ceph-mon[98901]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-09-13T14:38:08.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:08 smithi096 ceph-mon[102928]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-09-13T14:38:08.891 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:38:09.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:09 smithi096 ceph-mon[102928]: pgmap v189: 73 pgs: 26 active+undersized, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:38:09.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:09 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4107260737' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:09.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:09 smithi096 ceph-mon[98901]: pgmap v189: 73 pgs: 26 active+undersized, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:38:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:09 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4107260737' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:09.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:09 smithi204 ceph-mon[87071]: pgmap v189: 73 pgs: 26 active+undersized, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:38:09.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:09 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4107260737' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:10.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:10 smithi204 ceph-mon[87071]: from='client.44654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:10 smithi204 ceph-mon[87071]: from='client.44660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:10 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:10 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:10 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:10 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:10 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:10 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:38:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:38:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:10.674 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[98901]: from='client.44654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.674 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[98901]: from='client.44660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.674 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.674 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.674 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.674 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.674 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.674 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.675 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[102928]: from='client.44654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.675 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[102928]: from='client.44660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.675 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.675 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.675 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.675 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.675 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:10.675 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:11.322 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:10 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[126267]: 2024-09-13T14:38:10.954+0000 7fa5049f9640 -1 osd.3 100 log_to_monitors true 2024-09-13T14:38:11.685 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:11 smithi096 ceph-mon[98901]: pgmap v190: 73 pgs: 26 active+undersized, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:38:11.685 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:11 smithi096 ceph-mon[98901]: from='osd.3 [v2:172.21.15.96:6826/3452335742,v1:172.21.15.96:6827/3452335742]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:38:11.686 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:11 smithi096 ceph-mon[98901]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:38:11.686 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:11 smithi096 ceph-mon[102928]: pgmap v190: 73 pgs: 26 active+undersized, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:38:11.686 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:11 smithi096 ceph-mon[102928]: from='osd.3 [v2:172.21.15.96:6826/3452335742,v1:172.21.15.96:6827/3452335742]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:38:11.686 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:11 smithi096 ceph-mon[102928]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:38:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:11 smithi204 ceph-mon[87071]: pgmap v190: 73 pgs: 26 active+undersized, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:38:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:11 smithi204 ceph-mon[87071]: from='osd.3 [v2:172.21.15.96:6826/3452335742,v1:172.21.15.96:6827/3452335742]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:38:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:11 smithi204 ceph-mon[87071]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:38:11.798 INFO:teuthology.orchestra.run.smithi096.stdout:true 2024-09-13T14:38:12.322 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:38:11 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[126267]: 2024-09-13T14:38:11.961+0000 7fa4f879b640 -1 osd.3 100 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:38:12.658 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:12 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1693124390' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:12.658 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:12 smithi096 ceph-mon[102928]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:38:12.658 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:12 smithi096 ceph-mon[102928]: osdmap e103: 8 total, 7 up, 8 in 2024-09-13T14:38:12.658 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:12 smithi096 ceph-mon[102928]: from='osd.3 [v2:172.21.15.96:6826/3452335742,v1:172.21.15.96:6827/3452335742]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:38:12.659 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:12 smithi096 ceph-mon[102928]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:38:12.659 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:12 smithi096 ceph-mon[102928]: from='osd.3 ' entity='osd.3' 2024-09-13T14:38:12.659 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:12 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1693124390' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:12.659 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:12 smithi096 ceph-mon[98901]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:38:12.659 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:12 smithi096 ceph-mon[98901]: osdmap e103: 8 total, 7 up, 8 in 2024-09-13T14:38:12.659 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:12 smithi096 ceph-mon[98901]: from='osd.3 [v2:172.21.15.96:6826/3452335742,v1:172.21.15.96:6827/3452335742]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:38:12.659 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:12 smithi096 ceph-mon[98901]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:38:12.659 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:12 smithi096 ceph-mon[98901]: from='osd.3 ' entity='osd.3' 2024-09-13T14:38:12.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:12 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1693124390' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:12.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:12 smithi204 ceph-mon[87071]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:38:12.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:12 smithi204 ceph-mon[87071]: osdmap e103: 8 total, 7 up, 8 in 2024-09-13T14:38:12.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:12 smithi204 ceph-mon[87071]: from='osd.3 [v2:172.21.15.96:6826/3452335742,v1:172.21.15.96:6827/3452335742]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:38:12.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:12 smithi204 ceph-mon[87071]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T14:38:12.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:12 smithi204 ceph-mon[87071]: from='osd.3 ' entity='osd.3' 2024-09-13T14:38:13.572 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T14:38:13.425+0000 7ff04b218640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-13T14:38:13.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[98901]: pgmap v192: 73 pgs: 26 active+undersized, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:38:13.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[98901]: from='client.44675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:13.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[98901]: from='client.44681 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:13.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[98901]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:38:13.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[98901]: osd.3 [v2:172.21.15.96:6826/3452335742,v1:172.21.15.96:6827/3452335742] boot 2024-09-13T14:38:13.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[98901]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T14:38:13.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:38:13.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:13.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:13.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:13.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:13.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:13.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[102928]: pgmap v192: 73 pgs: 26 active+undersized, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:38:13.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[102928]: from='client.44675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:13.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[102928]: from='client.44681 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:13.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[102928]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:38:13.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[102928]: osd.3 [v2:172.21.15.96:6826/3452335742,v1:172.21.15.96:6827/3452335742] boot 2024-09-13T14:38:13.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[102928]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T14:38:13.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:38:13.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:13.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:13.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:13.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:13.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:13 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:13.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:13 smithi204 ceph-mon[87071]: pgmap v192: 73 pgs: 26 active+undersized, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:38:13.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:13 smithi204 ceph-mon[87071]: from='client.44675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:13.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:13 smithi204 ceph-mon[87071]: from='client.44681 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:13.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:13 smithi204 ceph-mon[87071]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:38:13.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:13 smithi204 ceph-mon[87071]: osd.3 [v2:172.21.15.96:6826/3452335742,v1:172.21.15.96:6827/3452335742] boot 2024-09-13T14:38:13.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:13 smithi204 ceph-mon[87071]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T14:38:13.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:13 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:38:13.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:13 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:13.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:13 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:13.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:13 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:13.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:13 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:13.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:13 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:14.378 INFO:teuthology.orchestra.run.smithi096.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 4 2024-09-13T14:38:14.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:14 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:14.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:14 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:14.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:14 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:14.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:14 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:14.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:14 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:38:14.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:14 smithi204 ceph-mon[87071]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:38:14.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:14 smithi204 ceph-mon[87071]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-13T14:38:14.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:14 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:14.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:14 smithi204 ceph-mon[87071]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T14:38:14.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:14 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1975116022' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:14.729 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:14.729 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:14.729 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:14.730 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:14.730 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:38:14.730 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[98901]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:38:14.730 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[98901]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-13T14:38:14.730 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:14.730 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[98901]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T14:38:14.730 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1975116022' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:14.730 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:14.730 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:14.730 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:14.730 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:14.731 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:38:14.731 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[102928]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:38:14.731 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[102928]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-13T14:38:14.731 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:14.731 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[102928]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T14:38:14.731 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:14 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1975116022' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:15.031 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade pause' 2024-09-13T14:38:15.278 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:15.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:15 smithi204 ceph-mon[87071]: pgmap v194: 73 pgs: 7 peering, 19 active+undersized, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:38:15.727 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:15 smithi096 ceph-mon[98901]: pgmap v194: 73 pgs: 7 peering, 19 active+undersized, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:38:15.728 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:15 smithi096 ceph-mon[102928]: pgmap v194: 73 pgs: 7 peering, 19 active+undersized, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:38:17.057 INFO:teuthology.orchestra.run.smithi096.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:38:17.710 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:17 smithi096 ceph-mon[98901]: pgmap v196: 73 pgs: 7 peering, 19 active+undersized, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:38:17.710 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:17 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:17.710 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:17 smithi096 ceph-mon[102928]: pgmap v196: 73 pgs: 7 peering, 19 active+undersized, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:38:17.711 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:17 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:17.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:17 smithi204 ceph-mon[87071]: pgmap v196: 73 pgs: 7 peering, 19 active+undersized, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:38:17.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:17 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:17.735 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:38:17.981 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:18.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:38:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:38:17] "GET /metrics HTTP/1.1" 200 34033 "" "Prometheus/2.51.0" 2024-09-13T14:38:18.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:18 smithi204 ceph-mon[87071]: from='client.44693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:18.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:18 smithi204 ceph-mon[87071]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:38:18.724 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:18 smithi096 ceph-mon[98901]: from='client.44693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:18.724 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:18 smithi096 ceph-mon[98901]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:38:18.724 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:18 smithi096 ceph-mon[102928]: from='client.44693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:18.725 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:18 smithi096 ceph-mon[102928]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:38:19.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:19 smithi096 ceph-mon[102928]: pgmap v197: 73 pgs: 7 peering, 66 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:19.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:19 smithi096 ceph-mon[102928]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:19 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:19 smithi096 ceph-mon[98901]: pgmap v197: 73 pgs: 7 peering, 66 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:19 smithi096 ceph-mon[98901]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:19 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:38:19.673 INFO:teuthology.orchestra.run.smithi096.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:38:19.674 INFO:teuthology.orchestra.run.smithi096.stdout:alertmanager.a smithi096 *:9093,9094 running (7m) 10s ago 11m 18.3M - 0.27.0 11f11916f8cd e04c10a0dca2 2024-09-13T14:38:19.674 INFO:teuthology.orchestra.run.smithi096.stdout:grafana.a smithi204 *:3000 running (6m) 9s ago 11m 82.7M - 9.4.7 954c08fa6188 451523aebd73 2024-09-13T14:38:19.674 INFO:teuthology.orchestra.run.smithi096.stdout:mds.foo.smithi096.dqiqpz smithi096 running (10m) 10s ago 10m 21.2M - 17.2.7-1627-gbbdadfa3 a994b00deb3a ab8d33048bb7 2024-09-13T14:38:19.674 INFO:teuthology.orchestra.run.smithi096.stdout:mds.foo.smithi204.hbwbot smithi204 running (10m) 9s ago 10m 15.9M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 97a3ff635de2 2024-09-13T14:38:19.674 INFO:teuthology.orchestra.run.smithi096.stdout:mgr.x smithi204 *:8443,9283,8765 running (6m) 9s ago 15m 456M - 19.3.0-4697-gd58f7306 fec07016a2b0 04572bea1e6f 2024-09-13T14:38:19.674 INFO:teuthology.orchestra.run.smithi096.stdout:mgr.y smithi096 *:8443,9283,8765 running (6m) 10s ago 16m 517M - 19.3.0-4697-gd58f7306 fec07016a2b0 a4bc51171f6e 2024-09-13T14:38:19.674 INFO:teuthology.orchestra.run.smithi096.stdout:mon.a smithi096 running (5m) 10s ago 17m 51.0M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 97a94128e048 2024-09-13T14:38:19.674 INFO:teuthology.orchestra.run.smithi096.stdout:mon.b smithi204 running (4m) 9s ago 15m 37.0M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 4cc02299264a 2024-09-13T14:38:19.674 INFO:teuthology.orchestra.run.smithi096.stdout:mon.c smithi096 running (4m) 10s ago 15m 41.0M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 babdd691d9b6 2024-09-13T14:38:19.674 INFO:teuthology.orchestra.run.smithi096.stdout:node-exporter.a smithi096 *:9100 running (6m) 10s ago 10m 10.6M - 1.7.0 72c9c2088986 b02b57424615 2024-09-13T14:38:19.674 INFO:teuthology.orchestra.run.smithi096.stdout:node-exporter.b smithi204 *:9100 running (6m) 9s ago 10m 10.0M - 1.7.0 72c9c2088986 c58ad3b83def 2024-09-13T14:38:19.674 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0 smithi096 running (2m) 10s ago 14m 43.0M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 ab741f6f9e89 2024-09-13T14:38:19.674 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1 smithi096 running (79s) 10s ago 14m 48.2M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 bbf0998790c2 2024-09-13T14:38:19.675 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2 smithi096 running (46s) 10s ago 14m 45.3M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 eaf4f3b21ff8 2024-09-13T14:38:19.675 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3 smithi096 running (13s) 10s ago 13m 12.2M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 23f73e3db413 2024-09-13T14:38:19.675 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4 smithi204 running (13m) 9s ago 13m 49.9M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a b48920b2b9e7 2024-09-13T14:38:19.675 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5 smithi204 running (12m) 9s ago 12m 49.7M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 0ed6e334d2ea 2024-09-13T14:38:19.675 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6 smithi204 running (12m) 9s ago 12m 49.9M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8f12f56d751c 2024-09-13T14:38:19.675 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7 smithi204 running (12m) 9s ago 12m 54.1M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a ffd75c8be38a 2024-09-13T14:38:19.675 INFO:teuthology.orchestra.run.smithi096.stdout:prometheus.a smithi204 *:9095 running (6m) 9s ago 11m 54.0M - 2.51.0 1d3b7f56885b c1d744b6a96b 2024-09-13T14:38:19.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:19 smithi204 ceph-mon[87071]: pgmap v197: 73 pgs: 7 peering, 66 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:19.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:19 smithi204 ceph-mon[87071]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-09-13T14:38:19.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:19 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:38:20.316 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:38:20.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:38:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:38:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:20.559 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:21.694 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:21 smithi096 ceph-mon[98901]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:21.694 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:21 smithi096 ceph-mon[98901]: from='client.44699 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:21.694 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:21.694 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:21.695 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:21 smithi096 ceph-mon[102928]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:21.695 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:21 smithi096 ceph-mon[102928]: from='client.44699 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:21.695 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:21.695 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:21.705 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:21 smithi204 ceph-mon[87071]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:21.705 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:21 smithi204 ceph-mon[87071]: from='client.44699 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:21.705 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:21.706 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:22.237 INFO:teuthology.orchestra.run.smithi096.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:38:22.238 INFO:teuthology.orchestra.run.smithi096.stdout:alertmanager.a smithi096 *:9093,9094 running (7m) 12s ago 11m 18.3M - 0.27.0 11f11916f8cd e04c10a0dca2 2024-09-13T14:38:22.238 INFO:teuthology.orchestra.run.smithi096.stdout:grafana.a smithi204 *:3000 running (6m) 11s ago 11m 82.7M - 9.4.7 954c08fa6188 451523aebd73 2024-09-13T14:38:22.238 INFO:teuthology.orchestra.run.smithi096.stdout:mds.foo.smithi096.dqiqpz smithi096 running (10m) 12s ago 10m 21.2M - 17.2.7-1627-gbbdadfa3 a994b00deb3a ab8d33048bb7 2024-09-13T14:38:22.238 INFO:teuthology.orchestra.run.smithi096.stdout:mds.foo.smithi204.hbwbot smithi204 running (10m) 11s ago 10m 15.9M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 97a3ff635de2 2024-09-13T14:38:22.238 INFO:teuthology.orchestra.run.smithi096.stdout:mgr.x smithi204 *:8443,9283,8765 running (6m) 11s ago 15m 456M - 19.3.0-4697-gd58f7306 fec07016a2b0 04572bea1e6f 2024-09-13T14:38:22.238 INFO:teuthology.orchestra.run.smithi096.stdout:mgr.y smithi096 *:8443,9283,8765 running (6m) 12s ago 16m 517M - 19.3.0-4697-gd58f7306 fec07016a2b0 a4bc51171f6e 2024-09-13T14:38:22.238 INFO:teuthology.orchestra.run.smithi096.stdout:mon.a smithi096 running (5m) 12s ago 17m 51.0M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 97a94128e048 2024-09-13T14:38:22.238 INFO:teuthology.orchestra.run.smithi096.stdout:mon.b smithi204 running (4m) 11s ago 15m 37.0M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 4cc02299264a 2024-09-13T14:38:22.238 INFO:teuthology.orchestra.run.smithi096.stdout:mon.c smithi096 running (4m) 12s ago 15m 41.0M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 babdd691d9b6 2024-09-13T14:38:22.239 INFO:teuthology.orchestra.run.smithi096.stdout:node-exporter.a smithi096 *:9100 running (7m) 12s ago 10m 10.6M - 1.7.0 72c9c2088986 b02b57424615 2024-09-13T14:38:22.239 INFO:teuthology.orchestra.run.smithi096.stdout:node-exporter.b smithi204 *:9100 running (6m) 11s ago 10m 10.0M - 1.7.0 72c9c2088986 c58ad3b83def 2024-09-13T14:38:22.239 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0 smithi096 running (2m) 12s ago 14m 43.0M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 ab741f6f9e89 2024-09-13T14:38:22.239 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1 smithi096 running (81s) 12s ago 14m 48.2M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 bbf0998790c2 2024-09-13T14:38:22.239 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2 smithi096 running (49s) 12s ago 14m 45.3M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 eaf4f3b21ff8 2024-09-13T14:38:22.239 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3 smithi096 running (16s) 12s ago 13m 12.2M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 23f73e3db413 2024-09-13T14:38:22.240 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4 smithi204 running (13m) 11s ago 13m 49.9M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a b48920b2b9e7 2024-09-13T14:38:22.240 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5 smithi204 running (13m) 11s ago 12m 49.7M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 0ed6e334d2ea 2024-09-13T14:38:22.240 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6 smithi204 running (12m) 11s ago 12m 49.9M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8f12f56d751c 2024-09-13T14:38:22.240 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7 smithi204 running (12m) 11s ago 12m 54.1M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a ffd75c8be38a 2024-09-13T14:38:22.240 INFO:teuthology.orchestra.run.smithi096.stdout:prometheus.a smithi204 *:9095 running (6m) 11s ago 11m 54.0M - 2.51.0 1d3b7f56885b c1d744b6a96b 2024-09-13T14:38:22.828 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph versions' 2024-09-13T14:38:23.070 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:23.465 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:23 smithi096 ceph-mon[98901]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:23.466 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:23 smithi096 ceph-mon[98901]: from='client.44705 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:23.466 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:23 smithi096 ceph-mon[102928]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:23.466 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:23 smithi096 ceph-mon[102928]: from='client.44705 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:23.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:23 smithi204 ceph-mon[87071]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:23.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:23 smithi204 ceph-mon[87071]: from='client.44705 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:24.811 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:38:24.811 INFO:teuthology.orchestra.run.smithi096.stdout: "mon": { 2024-09-13T14:38:24.811 INFO:teuthology.orchestra.run.smithi096.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:38:24.811 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:38:24.812 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr": { 2024-09-13T14:38:24.812 INFO:teuthology.orchestra.run.smithi096.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:38:24.812 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:38:24.812 INFO:teuthology.orchestra.run.smithi096.stdout: "osd": { 2024-09-13T14:38:24.812 INFO:teuthology.orchestra.run.smithi096.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 4, 2024-09-13T14:38:24.812 INFO:teuthology.orchestra.run.smithi096.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 4 2024-09-13T14:38:24.812 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:38:24.812 INFO:teuthology.orchestra.run.smithi096.stdout: "mds": { 2024-09-13T14:38:24.812 INFO:teuthology.orchestra.run.smithi096.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:38:24.812 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:38:24.812 INFO:teuthology.orchestra.run.smithi096.stdout: "overall": { 2024-09-13T14:38:24.812 INFO:teuthology.orchestra.run.smithi096.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 6, 2024-09-13T14:38:24.812 INFO:teuthology.orchestra.run.smithi096.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 9 2024-09-13T14:38:24.813 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:38:24.813 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:38:25.435 DEBUG:teuthology.parallel:result is None 2024-09-13T14:38:25.435 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:38:25.449 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-09-13T14:38:25.450 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-13T14:38:25.467 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T14:38:25.468 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T14:38:25.468 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T14:38:25.712 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:25.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:25 smithi204 ceph-mon[87071]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:25.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:25 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/417277416' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:25.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:25 smithi096 ceph-mon[98901]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:25.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:25 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/417277416' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:25.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:25 smithi096 ceph-mon[102928]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:25.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:25 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/417277416' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:27.050 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:26 smithi096 ceph-mon[98901]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:27.050 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:26 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:27.050 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:26 smithi096 ceph-mon[102928]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:27.050 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:26 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:27.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:26 smithi204 ceph-mon[87071]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:27.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:26 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:27.524 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:38:28.149 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:27 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3340037' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-13T14:38:28.149 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:27 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3340037' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-13T14:38:28.150 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:38:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:38:27] "GET /metrics HTTP/1.1" 200 34026 "" "Prometheus/2.51.0" 2024-09-13T14:38:28.172 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":39,"flags":0,"active_gid":26588,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6800","nonce":3581340838},{"type":"v1","addr":"172.21.15.96:6801","nonce":3581340838}]},"active_addr":"172.21.15.96:6801/3581340838","active_change":"2024-09-13T14:32:35.504887+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[{"gid":25930,"name":"x","mgr_features":4540719139924082687,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.96:8443/","prometheus":"http://172.21.15.96:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":82,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.96:0","nonce":256518555}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.96:0","nonce":482176647}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.96:0","nonce":4022198372}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.96:0","nonce":271467928}]}]} 2024-09-13T14:38:28.177 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T14:38:28.177 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T14:38:28.177 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:38:28.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:27 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3340037' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-13T14:38:28.419 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:29.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:28 smithi204 ceph-mon[87071]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:29.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:29.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[98901]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:29.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:29.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[102928]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:29.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:29.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:38:30.134 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:38:30.134 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":105,"fsid":"51e50ed6-71db-11ef-bceb-c7b262605968","created":"2024-09-13T14:21:16.367777+0000","modified":"2024-09-13T14:38:13.960806+0000","last_up_change":"2024-09-13T14:38:12.958673+0000","last_in_change":"2024-09-13T14:26:00.677688+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:24:20.514144+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:26:54.629267+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":78,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:28:09.271608+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:28:09.734465+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":88,"up_thru":104,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6802","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6803","nonce":2849588101}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6804","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6805","nonce":2849588101}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6808","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6809","nonce":2849588101}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6806","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6807","nonce":2849588101}]},"public_addr":"172.21.15.96:6803/2849588101","cluster_addr":"172.21.15.96:6805/2849588101","heartbeat_back_addr":"172.21.15.96:6809/2849588101","heartbeat_front_addr":"172.21.15.96:6807/2849588101","state":["exists","up"]},{"osd":1,"uuid":"1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":90,"up_from":94,"up_thru":104,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6810","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6811","nonce":289441923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6812","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6813","nonce":289441923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6816","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6817","nonce":289441923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6814","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6815","nonce":289441923}]},"public_addr":"172.21.15.96:6811/289441923","cluster_addr":"172.21.15.96:6813/289441923","heartbeat_back_addr":"172.21.15.96:6817/289441923","heartbeat_front_addr":"172.21.15.96:6815/289441923","state":["exists","up"]},{"osd":2,"uuid":"6a92a5b3-bdc7-499a-8b0f-2b0936fd9598","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":95,"up_from":99,"up_thru":104,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6818","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6819","nonce":4202655894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6820","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6821","nonce":4202655894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6824","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6825","nonce":4202655894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6822","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6823","nonce":4202655894}]},"public_addr":"172.21.15.96:6819/4202655894","cluster_addr":"172.21.15.96:6821/4202655894","heartbeat_back_addr":"172.21.15.96:6825/4202655894","heartbeat_front_addr":"172.21.15.96:6823/4202655894","state":["exists","up"]},{"osd":3,"uuid":"1230646c-934e-4eb4-b4b9-16be52ffba66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":100,"up_from":104,"up_thru":104,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6826","nonce":3452335742},{"type":"v1","addr":"172.21.15.96:6827","nonce":3452335742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6828","nonce":3452335742},{"type":"v1","addr":"172.21.15.96:6829","nonce":3452335742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6832","nonce":3452335742},{"type":"v1","addr":"172.21.15.96:6833","nonce":3452335742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6830","nonce":3452335742},{"type":"v1","addr":"172.21.15.96:6831","nonce":3452335742}]},"public_addr":"172.21.15.96:6827/3452335742","cluster_addr":"172.21.15.96:6829/3452335742","heartbeat_back_addr":"172.21.15.96:6833/3452335742","heartbeat_front_addr":"172.21.15.96:6831/3452335742","state":["exists","up"]},{"osd":4,"uuid":"14115269-3d41-4ebf-8613-71ed59ec0f29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":104,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6800","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6801","nonce":1188704632}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6802","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6803","nonce":1188704632}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6806","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6807","nonce":1188704632}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6804","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6805","nonce":1188704632}]},"public_addr":"172.21.15.204:6801/1188704632","cluster_addr":"172.21.15.204:6803/1188704632","heartbeat_back_addr":"172.21.15.204:6807/1188704632","heartbeat_front_addr":"172.21.15.204:6805/1188704632","state":["exists","up"]},{"osd":5,"uuid":"15148a0c-df44-419e-b0f0-2b1600c207f4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":104,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6808","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6809","nonce":580729752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6810","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6811","nonce":580729752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6814","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6815","nonce":580729752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6812","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6813","nonce":580729752}]},"public_addr":"172.21.15.204:6809/580729752","cluster_addr":"172.21.15.204:6811/580729752","heartbeat_back_addr":"172.21.15.204:6815/580729752","heartbeat_front_addr":"172.21.15.204:6813/580729752","state":["exists","up"]},{"osd":6,"uuid":"8404e088-3523-4c21-b7c2-0a31b22e0279","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6816","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6817","nonce":118365497}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6818","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6819","nonce":118365497}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6822","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6823","nonce":118365497}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6820","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6821","nonce":118365497}]},"public_addr":"172.21.15.204:6817/118365497","cluster_addr":"172.21.15.204:6819/118365497","heartbeat_back_addr":"172.21.15.204:6823/118365497","heartbeat_front_addr":"172.21.15.204:6821/118365497","state":["exists","up"]},{"osd":7,"uuid":"93b23a35-0059-4e03-a2ba-28fee6431892","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6824","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6825","nonce":3529639424}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6826","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6827","nonce":3529639424}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6830","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6831","nonce":3529639424}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6828","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6829","nonce":3529639424}]},"public_addr":"172.21.15.204:6825/3529639424","cluster_addr":"172.21.15.204:6827/3529639424","heartbeat_back_addr":"172.21.15.204:6831/3529639424","heartbeat_front_addr":"172.21.15.204:6829/3529639424","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:28.841233+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:53.021149+0000","dead_epoch":90},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:17.120825+0000","dead_epoch":95},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:41.339674+0000","dead_epoch":100},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:02.571454+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:26.608523+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:51.119331+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:26:16.050687+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.1","mappings":[{"from":4,"to":2}]},{"pgid":"3.6","mappings":[{"from":4,"to":2}]},{"pgid":"3.9","mappings":[{"from":4,"to":0}]},{"pgid":"3.f","mappings":[{"from":7,"to":2}]},{"pgid":"3.14","mappings":[{"from":4,"to":0}]},{"pgid":"3.15","mappings":[{"from":3,"to":0}]},{"pgid":"3.1d","mappings":[{"from":4,"to":0}]},{"pgid":"4.9","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":2},{"from":3,"to":0}]},{"pgid":"4.18","mappings":[{"from":1,"to":5}]},{"pgid":"4.19","mappings":[{"from":3,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.96:0/378263291":"2024-09-14T14:26:50.514276+0000","172.21.15.204:0/1453713144":"2024-09-14T14:31:56.307623+0000","172.21.15.96:6801/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/1494034928":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6800/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/3047961062":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/2862942348":"2024-09-14T14:32:35.504695+0000","172.21.15.96:6800/2749644659":"2024-09-14T14:26:50.514276+0000","172.21.15.204:0/4099954218":"2024-09-14T14:31:56.307623+0000","172.21.15.96:0/960238088":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/260172418":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/1652886381":"2024-09-14T14:30:47.049718+0000","172.21.15.204:0/3779551551":"2024-09-14T14:31:56.307623+0000","172.21.15.96:0/19119516":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6801/2724610080":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/3050864675":"2024-09-14T14:30:47.049718+0000","172.21.15.96:6800/2724610080":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/902309444":"2024-09-14T14:30:47.049718+0000","172.21.15.96:0/1790922234":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/667264267":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/1761759058":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/3329894409":"2024-09-14T14:32:35.504695+0000","172.21.15.204:6834/2474101951":"2024-09-14T14:31:56.307623+0000","172.21.15.96:6800/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/1644412166":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/2744909732":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/752011643":"2024-09-14T14:26:50.514276+0000","172.21.15.96:6800/776211071":"2024-09-14T14:30:47.049718+0000","172.21.15.96:6801/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/2292899363":"2024-09-14T14:21:45.592521+0000","172.21.15.204:0/4067568101":"2024-09-14T14:31:56.307623+0000","172.21.15.96:6801/776211071":"2024-09-14T14:30:47.049718+0000","172.21.15.96:6801/2749644659":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/3782790019":"2024-09-14T14:30:47.049718+0000","172.21.15.204:0/3780556706":"2024-09-14T14:31:56.307623+0000","172.21.15.96:0/376655558":"2024-09-14T14:21:45.592521+0000","172.21.15.204:6835/2474101951":"2024-09-14T14:31:56.307623+0000","172.21.15.204:0/2380512971":"2024-09-14T14:31:56.307623+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:38:30.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:38:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:38:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:30.721 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T14:38:30.722 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:38:30.976 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:31.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:30 smithi096 ceph-mon[102928]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:31.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:30 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2811811508' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:38:31.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:30 smithi096 ceph-mon[98901]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:31.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:30 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2811811508' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:38:31.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:30 smithi204 ceph-mon[87071]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:31.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:30 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2811811508' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:38:32.809 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:38:32.809 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":105,"fsid":"51e50ed6-71db-11ef-bceb-c7b262605968","created":"2024-09-13T14:21:16.367777+0000","modified":"2024-09-13T14:38:13.960806+0000","last_up_change":"2024-09-13T14:38:12.958673+0000","last_in_change":"2024-09-13T14:26:00.677688+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:24:20.514144+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:26:54.629267+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":78,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:28:09.271608+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:28:09.734465+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":88,"up_thru":104,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6802","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6803","nonce":2849588101}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6804","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6805","nonce":2849588101}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6808","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6809","nonce":2849588101}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6806","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6807","nonce":2849588101}]},"public_addr":"172.21.15.96:6803/2849588101","cluster_addr":"172.21.15.96:6805/2849588101","heartbeat_back_addr":"172.21.15.96:6809/2849588101","heartbeat_front_addr":"172.21.15.96:6807/2849588101","state":["exists","up"]},{"osd":1,"uuid":"1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":90,"up_from":94,"up_thru":104,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6810","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6811","nonce":289441923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6812","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6813","nonce":289441923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6816","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6817","nonce":289441923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6814","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6815","nonce":289441923}]},"public_addr":"172.21.15.96:6811/289441923","cluster_addr":"172.21.15.96:6813/289441923","heartbeat_back_addr":"172.21.15.96:6817/289441923","heartbeat_front_addr":"172.21.15.96:6815/289441923","state":["exists","up"]},{"osd":2,"uuid":"6a92a5b3-bdc7-499a-8b0f-2b0936fd9598","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":95,"up_from":99,"up_thru":104,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6818","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6819","nonce":4202655894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6820","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6821","nonce":4202655894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6824","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6825","nonce":4202655894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6822","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6823","nonce":4202655894}]},"public_addr":"172.21.15.96:6819/4202655894","cluster_addr":"172.21.15.96:6821/4202655894","heartbeat_back_addr":"172.21.15.96:6825/4202655894","heartbeat_front_addr":"172.21.15.96:6823/4202655894","state":["exists","up"]},{"osd":3,"uuid":"1230646c-934e-4eb4-b4b9-16be52ffba66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":100,"up_from":104,"up_thru":104,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6826","nonce":3452335742},{"type":"v1","addr":"172.21.15.96:6827","nonce":3452335742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6828","nonce":3452335742},{"type":"v1","addr":"172.21.15.96:6829","nonce":3452335742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6832","nonce":3452335742},{"type":"v1","addr":"172.21.15.96:6833","nonce":3452335742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6830","nonce":3452335742},{"type":"v1","addr":"172.21.15.96:6831","nonce":3452335742}]},"public_addr":"172.21.15.96:6827/3452335742","cluster_addr":"172.21.15.96:6829/3452335742","heartbeat_back_addr":"172.21.15.96:6833/3452335742","heartbeat_front_addr":"172.21.15.96:6831/3452335742","state":["exists","up"]},{"osd":4,"uuid":"14115269-3d41-4ebf-8613-71ed59ec0f29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":104,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6800","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6801","nonce":1188704632}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6802","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6803","nonce":1188704632}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6806","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6807","nonce":1188704632}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6804","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6805","nonce":1188704632}]},"public_addr":"172.21.15.204:6801/1188704632","cluster_addr":"172.21.15.204:6803/1188704632","heartbeat_back_addr":"172.21.15.204:6807/1188704632","heartbeat_front_addr":"172.21.15.204:6805/1188704632","state":["exists","up"]},{"osd":5,"uuid":"15148a0c-df44-419e-b0f0-2b1600c207f4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":104,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6808","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6809","nonce":580729752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6810","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6811","nonce":580729752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6814","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6815","nonce":580729752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6812","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6813","nonce":580729752}]},"public_addr":"172.21.15.204:6809/580729752","cluster_addr":"172.21.15.204:6811/580729752","heartbeat_back_addr":"172.21.15.204:6815/580729752","heartbeat_front_addr":"172.21.15.204:6813/580729752","state":["exists","up"]},{"osd":6,"uuid":"8404e088-3523-4c21-b7c2-0a31b22e0279","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6816","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6817","nonce":118365497}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6818","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6819","nonce":118365497}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6822","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6823","nonce":118365497}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6820","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6821","nonce":118365497}]},"public_addr":"172.21.15.204:6817/118365497","cluster_addr":"172.21.15.204:6819/118365497","heartbeat_back_addr":"172.21.15.204:6823/118365497","heartbeat_front_addr":"172.21.15.204:6821/118365497","state":["exists","up"]},{"osd":7,"uuid":"93b23a35-0059-4e03-a2ba-28fee6431892","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6824","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6825","nonce":3529639424}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6826","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6827","nonce":3529639424}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6830","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6831","nonce":3529639424}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6828","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6829","nonce":3529639424}]},"public_addr":"172.21.15.204:6825/3529639424","cluster_addr":"172.21.15.204:6827/3529639424","heartbeat_back_addr":"172.21.15.204:6831/3529639424","heartbeat_front_addr":"172.21.15.204:6829/3529639424","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:28.841233+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:53.021149+0000","dead_epoch":90},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:17.120825+0000","dead_epoch":95},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:41.339674+0000","dead_epoch":100},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:02.571454+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:26.608523+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:51.119331+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:26:16.050687+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.1","mappings":[{"from":4,"to":2}]},{"pgid":"3.6","mappings":[{"from":4,"to":2}]},{"pgid":"3.9","mappings":[{"from":4,"to":0}]},{"pgid":"3.f","mappings":[{"from":7,"to":2}]},{"pgid":"3.14","mappings":[{"from":4,"to":0}]},{"pgid":"3.15","mappings":[{"from":3,"to":0}]},{"pgid":"3.1d","mappings":[{"from":4,"to":0}]},{"pgid":"4.9","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":2},{"from":3,"to":0}]},{"pgid":"4.18","mappings":[{"from":1,"to":5}]},{"pgid":"4.19","mappings":[{"from":3,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.96:0/2744909732":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/2862942348":"2024-09-14T14:32:35.504695+0000","172.21.15.96:6800/2724610080":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/1761759058":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/3329894409":"2024-09-14T14:32:35.504695+0000","172.21.15.96:6800/776211071":"2024-09-14T14:30:47.049718+0000","172.21.15.96:6801/776211071":"2024-09-14T14:30:47.049718+0000","172.21.15.96:6800/2749644659":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/902309444":"2024-09-14T14:30:47.049718+0000","172.21.15.96:0/752011643":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/1494034928":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/1652886381":"2024-09-14T14:30:47.049718+0000","172.21.15.204:0/3779551551":"2024-09-14T14:31:56.307623+0000","172.21.15.96:0/19119516":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6801/2724610080":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/3050864675":"2024-09-14T14:30:47.049718+0000","172.21.15.96:6801/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.204:0/4067568101":"2024-09-14T14:31:56.307623+0000","172.21.15.96:0/2292899363":"2024-09-14T14:21:45.592521+0000","172.21.15.204:0/4099954218":"2024-09-14T14:31:56.307623+0000","172.21.15.96:0/960238088":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/260172418":"2024-09-14T14:26:50.514276+0000","172.21.15.96:6800/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/1644412166":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6801/2749644659":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/3782790019":"2024-09-14T14:30:47.049718+0000","172.21.15.204:0/3780556706":"2024-09-14T14:31:56.307623+0000","172.21.15.96:0/376655558":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/378263291":"2024-09-14T14:26:50.514276+0000","172.21.15.204:6834/2474101951":"2024-09-14T14:31:56.307623+0000","172.21.15.204:6835/2474101951":"2024-09-14T14:31:56.307623+0000","172.21.15.204:0/1453713144":"2024-09-14T14:31:56.307623+0000","172.21.15.204:0/2380512971":"2024-09-14T14:31:56.307623+0000","172.21.15.96:6801/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/3047961062":"2024-09-14T14:32:35.504695+0000","172.21.15.96:6800/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/667264267":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/1790922234":"2024-09-14T14:21:45.592521+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:38:33.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:32 smithi096 ceph-mon[98901]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:33.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:32 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1095208968' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:38:33.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:32 smithi096 ceph-mon[102928]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:33.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:32 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1095208968' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:38:33.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:32 smithi204 ceph-mon[87071]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:33.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:32 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1095208968' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:38:33.452 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T14:38:33.453 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T14:38:33.453 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T14:38:33.454 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T14:38:33.454 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T14:38:33.454 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T14:38:33.455 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T14:38:33.455 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T14:38:33.932 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:33.935 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:33.936 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:33.939 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:33.940 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:33.942 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:33.970 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:33.990 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:35.063 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:34 smithi096 ceph-mon[98901]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:35.063 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:34 smithi096 ceph-mon[102928]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:35.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:34 smithi204 ceph-mon[87071]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:36.034 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:36.034 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:36.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:37.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:36 smithi204 ceph-mon[87071]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:37.269 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:36 smithi096 ceph-mon[98901]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:37.269 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:36 smithi096 ceph-mon[102928]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:38.216 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:38:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:38:37] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.51.0" 2024-09-13T14:38:39.241 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:38 smithi096 ceph-mon[98901]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:39.241 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:38 smithi096 ceph-mon[102928]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:39.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:38 smithi204 ceph-mon[87071]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:38:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:38:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:40.990 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:40 smithi096 ceph-mon[98901]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:40.990 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:40 smithi096 ceph-mon[102928]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:41.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:40 smithi204 ceph-mon[87071]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:43.200 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:42 smithi096 ceph-mon[98901]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:43.200 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:42 smithi096 ceph-mon[102928]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:43.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:42 smithi204 ceph-mon[87071]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:43.828 INFO:teuthology.orchestra.run.smithi096.stdout:210453397656 2024-09-13T14:38:43.828 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T14:38:44.280 INFO:teuthology.orchestra.run.smithi096.stdout:133143986344 2024-09-13T14:38:44.280 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T14:38:44.708 INFO:teuthology.orchestra.run.smithi096.stdout:403726925845 2024-09-13T14:38:44.708 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T14:38:45.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:44 smithi096 ceph-mon[98901]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:45.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:44 smithi096 ceph-mon[102928]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:45.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:44 smithi204 ceph-mon[87071]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:47.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:46 smithi096 ceph-mon[98901]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:47.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:46 smithi096 ceph-mon[102928]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:47.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:46 smithi204 ceph-mon[87071]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:48.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:38:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:38:47] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.51.0" 2024-09-13T14:38:48.685 INFO:teuthology.orchestra.run.smithi096.stdout:158913790116 2024-09-13T14:38:48.686 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T14:38:48.688 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:49.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:48 smithi096 ceph-mon[98901]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:49.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:48 smithi096 ceph-mon[102928]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:49.432 INFO:teuthology.orchestra.run.smithi096.stdout:446676598793 2024-09-13T14:38:49.433 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T14:38:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:48 smithi204 ceph-mon[87071]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:50.046 INFO:teuthology.orchestra.run.smithi096.stdout:425201762319 2024-09-13T14:38:50.046 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T14:38:50.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:38:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:38:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:50.896 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:50.896 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:51.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:50 smithi096 ceph-mon[102928]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:51.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:50 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:50 smithi096 ceph-mon[98901]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:50 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:51.369 INFO:teuthology.orchestra.run.smithi096.stdout:184683593887 2024-09-13T14:38:51.370 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T14:38:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:50 smithi204 ceph-mon[87071]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:50 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:52.102 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:52.626 INFO:teuthology.orchestra.run.smithi096.stdout:377957122086 2024-09-13T14:38:52.626 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T14:38:53.197 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:52 smithi096 ceph-mon[98901]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:53.198 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:52 smithi096 ceph-mon[102928]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:53.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:52 smithi204 ceph-mon[87071]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:53.795 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:53.808 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:54.793 INFO:teuthology.orchestra.run.smithi096.stdout:210453397659 2024-09-13T14:38:55.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:54 smithi096 ceph-mon[98901]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:55.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:54 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2238727887' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T14:38:55.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:54 smithi096 ceph-mon[102928]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:55.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:54 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2238727887' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T14:38:55.335 INFO:teuthology.orchestra.run.smithi096.stdout:403726925847 2024-09-13T14:38:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:54 smithi204 ceph-mon[87071]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:55.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:54 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2238727887' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T14:38:56.055 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:56.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:55 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1314419831' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T14:38:56.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:55 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1314419831' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T14:38:56.370 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397656 got 210453397659 for osd.7 2024-09-13T14:38:56.370 DEBUG:teuthology.parallel:result is None 2024-09-13T14:38:56.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:55 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1314419831' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T14:38:56.997 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:56 smithi096 ceph-mon[98901]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:57.309 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:56 smithi096 ceph-mon[102928]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:56 smithi204 ceph-mon[87071]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:58.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:38:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:38:57] "GET /metrics HTTP/1.1" 200 34032 "" "Prometheus/2.51.0" 2024-09-13T14:38:59.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:38:58 smithi096 ceph-mon[98901]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:59.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:38:58 smithi096 ceph-mon[102928]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:38:58 smithi204 ceph-mon[87071]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:38:59.554 INFO:teuthology.orchestra.run.smithi096.stdout:133143986347 2024-09-13T14:39:00.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:00 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1902269931' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T14:39:00.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:00 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1902269931' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T14:39:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:00 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1902269931' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T14:39:00.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:39:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:39:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:00.471 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726925845 got 403726925847 for osd.1 2024-09-13T14:39:00.471 DEBUG:teuthology.parallel:result is None 2024-09-13T14:39:01.285 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:01 smithi096 ceph-mon[98901]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:01.285 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:01 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/567131094' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T14:39:01.285 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:01 smithi096 ceph-mon[102928]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:01.286 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:01 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/567131094' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T14:39:01.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:01 smithi204 ceph-mon[87071]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:01.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:01 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/567131094' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T14:39:01.613 INFO:teuthology.orchestra.run.smithi096.stdout:425201762322 2024-09-13T14:39:01.802 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:02.231 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762319 got 425201762322 for osd.2 2024-09-13T14:39:02.231 DEBUG:teuthology.parallel:result is None 2024-09-13T14:39:02.617 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986344 got 133143986347 for osd.4 2024-09-13T14:39:02.618 DEBUG:teuthology.parallel:result is None 2024-09-13T14:39:03.035 INFO:teuthology.orchestra.run.smithi096.stdout:158913790119 2024-09-13T14:39:03.256 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:03 smithi096 ceph-mon[98901]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:03.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:03 smithi096 ceph-mon[102928]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:03.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:03 smithi204 ceph-mon[87071]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:04.197 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:04 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3310785379' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T14:39:04.197 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:04 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3310785379' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T14:39:04.379 INFO:teuthology.orchestra.run.smithi096.stdout:184683593890 2024-09-13T14:39:04.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:04 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3310785379' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T14:39:05.174 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:05 smithi096 ceph-mon[102928]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:05.174 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:05 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/218600895' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T14:39:05.174 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:05 smithi096 ceph-mon[98901]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:05.175 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:05 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/218600895' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T14:39:05.175 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913790116 got 158913790119 for osd.5 2024-09-13T14:39:05.175 DEBUG:teuthology.parallel:result is None 2024-09-13T14:39:05.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:05 smithi204 ceph-mon[87071]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:05.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:05 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/218600895' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T14:39:06.234 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:06.235 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:06.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:06.677 INFO:teuthology.orchestra.run.smithi096.stdout:446676598796 2024-09-13T14:39:06.920 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593887 got 184683593890 for osd.6 2024-09-13T14:39:06.920 DEBUG:teuthology.parallel:result is None 2024-09-13T14:39:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:07 smithi096 ceph-mon[98901]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:07 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4269064165' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T14:39:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:07 smithi096 ceph-mon[102928]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:07 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4269064165' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T14:39:07.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:07 smithi204 ceph-mon[87071]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:07.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:07 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4269064165' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T14:39:07.867 INFO:tasks.cephadm.ceph_manager.ceph:need seq 446676598793 got 446676598796 for osd.3 2024-09-13T14:39:07.868 DEBUG:teuthology.parallel:result is None 2024-09-13T14:39:08.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:39:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:39:07] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.51.0" 2024-09-13T14:39:08.290 INFO:teuthology.orchestra.run.smithi096.stdout:377957122089 2024-09-13T14:39:08.989 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122086 got 377957122089 for osd.0 2024-09-13T14:39:08.989 DEBUG:teuthology.parallel:result is None 2024-09-13T14:39:08.990 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:39:08.990 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:39:09.234 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:09.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:09 smithi096 ceph-mon[98901]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:09.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:09 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1994896407' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T14:39:09.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:09 smithi096 ceph-mon[102928]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:09.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:09 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1994896407' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T14:39:09.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:09 smithi204 ceph-mon[87071]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:09 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1994896407' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T14:39:10.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:39:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:39:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:11.039 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:39:11.040 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:39:11.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:11 smithi096 ceph-mon[98901]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:11.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:11 smithi096 ceph-mon[102928]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:11.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:11 smithi204 ceph-mon[87071]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:11.765 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":223,"stamp":"2024-09-13T14:39:09.614495+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6304,"num_read_kb":13770,"num_write":1056,"num_write_kb":14251,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":595202,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1132,"ondisk_log_size":1132,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1331152,"kb_used_data":23240,"kb_used_omap":35,"kb_used_meta":1307740,"kb_avail":748630064,"statfs":{"total":767960285184,"available":766597185536,"internally_reserved":0,"allocated":23797760,"data_stored":19304672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36830,"internal_metadata":1339125794},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002894"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982673+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:38:13.982673+0000","last_peered":"2024-09-13T14:38:13.982673+0000","last_clean":"2024-09-13T14:38:13.982673+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:38:13.982673+0000","last_undegraded":"2024-09-13T14:38:13.982673+0000","last_fullsized":"2024-09-13T14:38:13.982673+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.967270+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:38:12.967270+0000","last_peered":"2024-09-13T14:38:12.967270+0000","last_clean":"2024-09-13T14:38:12.967270+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:38:12.967270+0000","last_undegraded":"2024-09-13T14:38:12.967270+0000","last_fullsized":"2024-09-13T14:38:12.967270+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979669+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:38:13.979669+0000","last_peered":"2024-09-13T14:38:13.979669+0000","last_clean":"2024-09-13T14:38:13.979669+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:38:13.979669+0000","last_undegraded":"2024-09-13T14:38:13.979669+0000","last_fullsized":"2024-09-13T14:38:13.979669+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":66,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978069+0000","last_change":"2024-09-13T14:38:13.977954+0000","last_active":"2024-09-13T14:38:13.978069+0000","last_peered":"2024-09-13T14:38:13.978069+0000","last_clean":"2024-09-13T14:38:13.978069+0000","last_became_active":"2024-09-13T14:38:13.977619+0000","last_became_peered":"2024-09-13T14:38:13.977619+0000","last_unstale":"2024-09-13T14:38:13.978069+0000","last_undegraded":"2024-09-13T14:38:13.978069+0000","last_fullsized":"2024-09-13T14:38:13.978069+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:58:30.259967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978307+0000","last_change":"2024-09-13T14:38:13.978232+0000","last_active":"2024-09-13T14:38:13.978307+0000","last_peered":"2024-09-13T14:38:13.978307+0000","last_clean":"2024-09-13T14:38:13.978307+0000","last_became_active":"2024-09-13T14:38:13.977921+0000","last_became_peered":"2024-09-13T14:38:13.977921+0000","last_unstale":"2024-09-13T14:38:13.978307+0000","last_undegraded":"2024-09-13T14:38:13.978307+0000","last_fullsized":"2024-09-13T14:38:13.978307+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:10.127626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969015+0000","last_change":"2024-09-13T14:28:10.508894+0000","last_active":"2024-09-13T14:38:12.969015+0000","last_peered":"2024-09-13T14:38:12.969015+0000","last_clean":"2024-09-13T14:38:12.969015+0000","last_became_active":"2024-09-13T14:28:10.508612+0000","last_became_peered":"2024-09-13T14:28:10.508612+0000","last_unstale":"2024-09-13T14:38:12.969015+0000","last_undegraded":"2024-09-13T14:38:12.969015+0000","last_fullsized":"2024-09-13T14:38:12.969015+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:01.774646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":53,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.970970+0000","last_change":"2024-09-13T14:37:40.925691+0000","last_active":"2024-09-13T14:38:13.970970+0000","last_peered":"2024-09-13T14:38:13.970970+0000","last_clean":"2024-09-13T14:38:13.970970+0000","last_became_active":"2024-09-13T14:37:40.925360+0000","last_became_peered":"2024-09-13T14:37:40.925360+0000","last_unstale":"2024-09-13T14:38:13.970970+0000","last_undegraded":"2024-09-13T14:38:13.970970+0000","last_fullsized":"2024-09-13T14:38:13.970970+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:07.337967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.971837+0000","last_change":"2024-09-13T14:35:49.984224+0000","last_active":"2024-09-13T14:38:12.971837+0000","last_peered":"2024-09-13T14:38:12.971837+0000","last_clean":"2024-09-13T14:38:12.971837+0000","last_became_active":"2024-09-13T14:35:49.983815+0000","last_became_peered":"2024-09-13T14:35:49.983815+0000","last_unstale":"2024-09-13T14:38:12.971837+0000","last_undegraded":"2024-09-13T14:38:12.971837+0000","last_fullsized":"2024-09-13T14:38:12.971837+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:27:07.882263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":57,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.973759+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:38:13.973759+0000","last_peered":"2024-09-13T14:38:13.973759+0000","last_clean":"2024-09-13T14:38:13.973759+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:38:13.973759+0000","last_undegraded":"2024-09-13T14:38:13.973759+0000","last_fullsized":"2024-09-13T14:38:13.973759+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968292+0000","last_change":"2024-09-13T14:28:11.507252+0000","last_active":"2024-09-13T14:38:12.968292+0000","last_peered":"2024-09-13T14:38:12.968292+0000","last_clean":"2024-09-13T14:38:12.968292+0000","last_became_active":"2024-09-13T14:28:11.506972+0000","last_became_peered":"2024-09-13T14:28:11.506972+0000","last_unstale":"2024-09-13T14:38:12.968292+0000","last_undegraded":"2024-09-13T14:38:12.968292+0000","last_fullsized":"2024-09-13T14:38:12.968292+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:01.583699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.970091+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:38:12.970091+0000","last_peered":"2024-09-13T14:38:12.970091+0000","last_clean":"2024-09-13T14:38:12.970091+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:38:12.970091+0000","last_undegraded":"2024-09-13T14:38:12.970091+0000","last_fullsized":"2024-09-13T14:38:12.970091+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":96,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979677+0000","last_change":"2024-09-13T14:38:13.979677+0000","last_active":"2024-09-13T14:38:13.979677+0000","last_peered":"2024-09-13T14:38:13.979677+0000","last_clean":"2024-09-13T14:38:13.979677+0000","last_became_active":"2024-09-13T14:38:13.979323+0000","last_became_peered":"2024-09-13T14:38:13.979323+0000","last_unstale":"2024-09-13T14:38:13.979677+0000","last_undegraded":"2024-09-13T14:38:13.979677+0000","last_fullsized":"2024-09-13T14:38:13.979677+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:29:03.491254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969851+0000","last_change":"2024-09-13T14:28:10.504588+0000","last_active":"2024-09-13T14:38:12.969851+0000","last_peered":"2024-09-13T14:38:12.969851+0000","last_clean":"2024-09-13T14:38:12.969851+0000","last_became_active":"2024-09-13T14:28:10.504338+0000","last_became_peered":"2024-09-13T14:28:10.504338+0000","last_unstale":"2024-09-13T14:38:12.969851+0000","last_undegraded":"2024-09-13T14:38:12.969851+0000","last_fullsized":"2024-09-13T14:38:12.969851+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:06:49.774993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.972936+0000","last_change":"2024-09-13T14:37:09.287528+0000","last_active":"2024-09-13T14:38:12.972936+0000","last_peered":"2024-09-13T14:38:12.972936+0000","last_clean":"2024-09-13T14:38:12.972936+0000","last_became_active":"2024-09-13T14:37:09.287182+0000","last_became_peered":"2024-09-13T14:37:09.287182+0000","last_unstale":"2024-09-13T14:38:12.972936+0000","last_undegraded":"2024-09-13T14:38:12.972936+0000","last_fullsized":"2024-09-13T14:38:12.972936+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:15.435336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982477+0000","last_change":"2024-09-13T14:38:13.982477+0000","last_active":"2024-09-13T14:38:13.982477+0000","last_peered":"2024-09-13T14:38:13.982477+0000","last_clean":"2024-09-13T14:38:13.982477+0000","last_became_active":"2024-09-13T14:38:13.982101+0000","last_became_peered":"2024-09-13T14:38:13.982101+0000","last_unstale":"2024-09-13T14:38:13.982477+0000","last_undegraded":"2024-09-13T14:38:13.982477+0000","last_fullsized":"2024-09-13T14:38:13.982477+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:24:07.896516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971326+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:38:13.971326+0000","last_peered":"2024-09-13T14:38:13.971326+0000","last_clean":"2024-09-13T14:38:13.971326+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:38:13.971326+0000","last_undegraded":"2024-09-13T14:38:13.971326+0000","last_fullsized":"2024-09-13T14:38:13.971326+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.983292+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:38:13.983292+0000","last_peered":"2024-09-13T14:38:13.983292+0000","last_clean":"2024-09-13T14:38:13.983292+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:38:13.983292+0000","last_undegraded":"2024-09-13T14:38:13.983292+0000","last_fullsized":"2024-09-13T14:38:13.983292+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982051+0000","last_change":"2024-09-13T14:38:13.982051+0000","last_active":"2024-09-13T14:38:13.982051+0000","last_peered":"2024-09-13T14:38:13.982051+0000","last_clean":"2024-09-13T14:38:13.982051+0000","last_became_active":"2024-09-13T14:38:13.981724+0000","last_became_peered":"2024-09-13T14:38:13.981724+0000","last_unstale":"2024-09-13T14:38:13.982051+0000","last_undegraded":"2024-09-13T14:38:13.982051+0000","last_fullsized":"2024-09-13T14:38:13.982051+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:52:34.815144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.972777+0000","last_change":"2024-09-13T14:37:09.286323+0000","last_active":"2024-09-13T14:38:12.972777+0000","last_peered":"2024-09-13T14:38:12.972777+0000","last_clean":"2024-09-13T14:38:12.972777+0000","last_became_active":"2024-09-13T14:37:09.285954+0000","last_became_peered":"2024-09-13T14:37:09.285954+0000","last_unstale":"2024-09-13T14:38:12.972777+0000","last_undegraded":"2024-09-13T14:38:12.972777+0000","last_fullsized":"2024-09-13T14:38:12.972777+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:23.527100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.967074+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:38:12.967074+0000","last_peered":"2024-09-13T14:38:12.967074+0000","last_clean":"2024-09-13T14:38:12.967074+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:38:12.967074+0000","last_undegraded":"2024-09-13T14:38:12.967074+0000","last_fullsized":"2024-09-13T14:38:12.967074+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":53,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.970670+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:38:13.970670+0000","last_peered":"2024-09-13T14:38:13.970670+0000","last_clean":"2024-09-13T14:38:13.970670+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:38:13.970670+0000","last_undegraded":"2024-09-13T14:38:13.970670+0000","last_fullsized":"2024-09-13T14:38:13.970670+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.966244+0000","last_change":"2024-09-13T14:37:09.289820+0000","last_active":"2024-09-13T14:38:12.966244+0000","last_peered":"2024-09-13T14:38:12.966244+0000","last_clean":"2024-09-13T14:38:12.966244+0000","last_became_active":"2024-09-13T14:37:09.289544+0000","last_became_peered":"2024-09-13T14:37:09.289544+0000","last_unstale":"2024-09-13T14:38:12.966244+0000","last_undegraded":"2024-09-13T14:38:12.966244+0000","last_fullsized":"2024-09-13T14:38:12.966244+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:23.255945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.981925+0000","last_change":"2024-09-13T14:38:13.981576+0000","last_active":"2024-09-13T14:38:13.981925+0000","last_peered":"2024-09-13T14:38:13.981925+0000","last_clean":"2024-09-13T14:38:13.981925+0000","last_became_active":"2024-09-13T14:38:13.981262+0000","last_became_peered":"2024-09-13T14:38:13.981262+0000","last_unstale":"2024-09-13T14:38:13.981925+0000","last_undegraded":"2024-09-13T14:38:13.981925+0000","last_fullsized":"2024-09-13T14:38:13.981925+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:20.284250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":54,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979392+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:38:13.979392+0000","last_peered":"2024-09-13T14:38:13.979392+0000","last_clean":"2024-09-13T14:38:13.979392+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:38:13.979392+0000","last_undegraded":"2024-09-13T14:38:13.979392+0000","last_fullsized":"2024-09-13T14:38:13.979392+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":186,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968558+0000","last_change":"2024-09-13T14:35:49.988378+0000","last_active":"2024-09-13T14:38:12.968558+0000","last_peered":"2024-09-13T14:38:12.968558+0000","last_clean":"2024-09-13T14:38:12.968558+0000","last_became_active":"2024-09-13T14:35:49.987987+0000","last_became_peered":"2024-09-13T14:35:49.987987+0000","last_unstale":"2024-09-13T14:38:12.968558+0000","last_undegraded":"2024-09-13T14:38:12.968558+0000","last_fullsized":"2024-09-13T14:38:12.968558+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:24:20.875473+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:32.753237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":632,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969114+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:38:12.969114+0000","last_peered":"2024-09-13T14:38:12.969114+0000","last_clean":"2024-09-13T14:38:12.969114+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:38:12.969114+0000","last_undegraded":"2024-09-13T14:38:12.969114+0000","last_fullsized":"2024-09-13T14:38:12.969114+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":24,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984733+0000","last_change":"2024-09-13T14:38:13.984643+0000","last_active":"2024-09-13T14:38:13.984733+0000","last_peered":"2024-09-13T14:38:13.984733+0000","last_clean":"2024-09-13T14:38:13.984733+0000","last_became_active":"2024-09-13T14:38:13.984316+0000","last_became_peered":"2024-09-13T14:38:13.984316+0000","last_unstale":"2024-09-13T14:38:13.984733+0000","last_undegraded":"2024-09-13T14:38:13.984733+0000","last_fullsized":"2024-09-13T14:38:13.984733+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:00:43.369510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.973242+0000","last_change":"2024-09-13T14:28:11.532509+0000","last_active":"2024-09-13T14:38:12.973242+0000","last_peered":"2024-09-13T14:38:12.973242+0000","last_clean":"2024-09-13T14:38:12.973242+0000","last_became_active":"2024-09-13T14:28:11.532207+0000","last_became_peered":"2024-09-13T14:28:11.532207+0000","last_unstale":"2024-09-13T14:38:12.973242+0000","last_undegraded":"2024-09-13T14:38:12.973242+0000","last_fullsized":"2024-09-13T14:38:12.973242+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:33.385182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":316,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968438+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:38:12.968438+0000","last_peered":"2024-09-13T14:38:12.968438+0000","last_clean":"2024-09-13T14:38:12.968438+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:38:12.968438+0000","last_undegraded":"2024-09-13T14:38:12.968438+0000","last_fullsized":"2024-09-13T14:38:12.968438+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978052+0000","last_change":"2024-09-13T14:38:13.977962+0000","last_active":"2024-09-13T14:38:13.978052+0000","last_peered":"2024-09-13T14:38:13.978052+0000","last_clean":"2024-09-13T14:38:13.978052+0000","last_became_active":"2024-09-13T14:38:13.977638+0000","last_became_peered":"2024-09-13T14:38:13.977638+0000","last_unstale":"2024-09-13T14:38:13.978052+0000","last_undegraded":"2024-09-13T14:38:13.978052+0000","last_fullsized":"2024-09-13T14:38:13.978052+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:53:16.902157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":78,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971451+0000","last_change":"2024-09-13T14:37:40.927444+0000","last_active":"2024-09-13T14:38:13.971451+0000","last_peered":"2024-09-13T14:38:13.971451+0000","last_clean":"2024-09-13T14:38:13.971451+0000","last_became_active":"2024-09-13T14:37:40.927140+0000","last_became_peered":"2024-09-13T14:37:40.927140+0000","last_unstale":"2024-09-13T14:38:13.971451+0000","last_undegraded":"2024-09-13T14:38:13.971451+0000","last_fullsized":"2024-09-13T14:38:13.971451+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:32:54.207568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":478,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978772+0000","last_change":"2024-09-13T14:38:13.978679+0000","last_active":"2024-09-13T14:38:13.978772+0000","last_peered":"2024-09-13T14:38:13.978772+0000","last_clean":"2024-09-13T14:38:13.978772+0000","last_became_active":"2024-09-13T14:38:13.978171+0000","last_became_peered":"2024-09-13T14:38:13.978171+0000","last_unstale":"2024-09-13T14:38:13.978772+0000","last_undegraded":"2024-09-13T14:38:13.978772+0000","last_fullsized":"2024-09-13T14:38:13.978772+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:57.691198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972597+0000","last_change":"2024-09-13T14:37:40.924713+0000","last_active":"2024-09-13T14:38:13.972597+0000","last_peered":"2024-09-13T14:38:13.972597+0000","last_clean":"2024-09-13T14:38:13.972597+0000","last_became_active":"2024-09-13T14:37:40.924435+0000","last_became_peered":"2024-09-13T14:37:40.924435+0000","last_unstale":"2024-09-13T14:38:13.972597+0000","last_undegraded":"2024-09-13T14:38:13.972597+0000","last_fullsized":"2024-09-13T14:38:13.972597+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:18:43.648821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":87,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971501+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:38:13.971501+0000","last_peered":"2024-09-13T14:38:13.971501+0000","last_clean":"2024-09-13T14:38:13.971501+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:38:13.971501+0000","last_undegraded":"2024-09-13T14:38:13.971501+0000","last_fullsized":"2024-09-13T14:38:13.971501+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":663,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969130+0000","last_change":"2024-09-13T14:37:09.291147+0000","last_active":"2024-09-13T14:38:12.969130+0000","last_peered":"2024-09-13T14:38:12.969130+0000","last_clean":"2024-09-13T14:38:12.969130+0000","last_became_active":"2024-09-13T14:37:09.290585+0000","last_became_peered":"2024-09-13T14:37:09.290585+0000","last_unstale":"2024-09-13T14:38:12.969130+0000","last_undegraded":"2024-09-13T14:38:12.969130+0000","last_fullsized":"2024-09-13T14:38:12.969130+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:32.764839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968123+0000","last_change":"2024-09-13T14:35:49.990342+0000","last_active":"2024-09-13T14:38:12.968123+0000","last_peered":"2024-09-13T14:38:12.968123+0000","last_clean":"2024-09-13T14:38:12.968123+0000","last_became_active":"2024-09-13T14:35:49.990053+0000","last_became_peered":"2024-09-13T14:35:49.990053+0000","last_unstale":"2024-09-13T14:38:12.968123+0000","last_undegraded":"2024-09-13T14:38:12.968123+0000","last_fullsized":"2024-09-13T14:38:12.968123+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:24:54.256709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":65,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.981376+0000","last_change":"2024-09-13T14:38:13.981294+0000","last_active":"2024-09-13T14:38:13.981376+0000","last_peered":"2024-09-13T14:38:13.981376+0000","last_clean":"2024-09-13T14:38:13.981376+0000","last_became_active":"2024-09-13T14:38:13.980975+0000","last_became_peered":"2024-09-13T14:38:13.980975+0000","last_unstale":"2024-09-13T14:38:13.981376+0000","last_undegraded":"2024-09-13T14:38:13.981376+0000","last_fullsized":"2024-09-13T14:38:13.981376+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:17.545179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":87,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.980578+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:38:13.980578+0000","last_peered":"2024-09-13T14:38:13.980578+0000","last_clean":"2024-09-13T14:38:13.980578+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:38:13.980578+0000","last_undegraded":"2024-09-13T14:38:13.980578+0000","last_fullsized":"2024-09-13T14:38:13.980578+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":571,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.973453+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:38:13.973453+0000","last_peered":"2024-09-13T14:38:13.973453+0000","last_clean":"2024-09-13T14:38:13.973453+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:38:13.973453+0000","last_undegraded":"2024-09-13T14:38:13.973453+0000","last_fullsized":"2024-09-13T14:38:13.973453+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":43,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972786+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:38:13.972786+0000","last_peered":"2024-09-13T14:38:13.972786+0000","last_clean":"2024-09-13T14:38:13.972786+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:38:13.972786+0000","last_undegraded":"2024-09-13T14:38:13.972786+0000","last_fullsized":"2024-09-13T14:38:13.972786+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":54,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972952+0000","last_change":"2024-09-13T14:37:40.921861+0000","last_active":"2024-09-13T14:38:13.972952+0000","last_peered":"2024-09-13T14:38:13.972952+0000","last_clean":"2024-09-13T14:38:13.972952+0000","last_became_active":"2024-09-13T14:37:40.921587+0000","last_became_peered":"2024-09-13T14:37:40.921587+0000","last_unstale":"2024-09-13T14:38:13.972952+0000","last_undegraded":"2024-09-13T14:38:13.972952+0000","last_fullsized":"2024-09-13T14:38:13.972952+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:24.619352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":318,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969979+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:38:12.969979+0000","last_peered":"2024-09-13T14:38:12.969979+0000","last_clean":"2024-09-13T14:38:12.969979+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:38:12.969979+0000","last_undegraded":"2024-09-13T14:38:12.969979+0000","last_fullsized":"2024-09-13T14:38:12.969979+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":56,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971289+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:38:13.971289+0000","last_peered":"2024-09-13T14:38:13.971289+0000","last_clean":"2024-09-13T14:38:13.971289+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:38:13.971289+0000","last_undegraded":"2024-09-13T14:38:13.971289+0000","last_fullsized":"2024-09-13T14:38:13.971289+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982751+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:38:13.982751+0000","last_peered":"2024-09-13T14:38:13.982751+0000","last_clean":"2024-09-13T14:38:13.982751+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:38:13.982751+0000","last_undegraded":"2024-09-13T14:38:13.982751+0000","last_fullsized":"2024-09-13T14:38:13.982751+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":358,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.973308+0000","last_change":"2024-09-13T14:37:09.288287+0000","last_active":"2024-09-13T14:38:13.973308+0000","last_peered":"2024-09-13T14:38:13.973308+0000","last_clean":"2024-09-13T14:38:13.973308+0000","last_became_active":"2024-09-13T14:37:09.287720+0000","last_became_peered":"2024-09-13T14:37:09.287720+0000","last_unstale":"2024-09-13T14:38:13.973308+0000","last_undegraded":"2024-09-13T14:38:13.973308+0000","last_fullsized":"2024-09-13T14:38:13.973308+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:09:34.761953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979342+0000","last_change":"2024-09-13T14:38:13.979255+0000","last_active":"2024-09-13T14:38:13.979342+0000","last_peered":"2024-09-13T14:38:13.979342+0000","last_clean":"2024-09-13T14:38:13.979342+0000","last_became_active":"2024-09-13T14:38:13.978868+0000","last_became_peered":"2024-09-13T14:38:13.978868+0000","last_unstale":"2024-09-13T14:38:13.979342+0000","last_undegraded":"2024-09-13T14:38:13.979342+0000","last_fullsized":"2024-09-13T14:38:13.979342+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:15:26.003538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":78,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971331+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:38:13.971331+0000","last_peered":"2024-09-13T14:38:13.971331+0000","last_clean":"2024-09-13T14:38:13.971331+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:38:13.971331+0000","last_undegraded":"2024-09-13T14:38:13.971331+0000","last_fullsized":"2024-09-13T14:38:13.971331+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1026,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.971972+0000","last_change":"2024-09-13T14:37:40.926427+0000","last_active":"2024-09-13T14:38:12.971972+0000","last_peered":"2024-09-13T14:38:12.971972+0000","last_clean":"2024-09-13T14:38:12.971972+0000","last_became_active":"2024-09-13T14:37:40.925922+0000","last_became_peered":"2024-09-13T14:37:40.925922+0000","last_unstale":"2024-09-13T14:38:12.971972+0000","last_undegraded":"2024-09-13T14:38:12.971972+0000","last_fullsized":"2024-09-13T14:38:12.971972+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:56:52.915505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.966678+0000","last_change":"2024-09-13T14:28:10.583355+0000","last_active":"2024-09-13T14:38:12.966678+0000","last_peered":"2024-09-13T14:38:12.966678+0000","last_clean":"2024-09-13T14:38:12.966678+0000","last_became_active":"2024-09-13T14:28:10.583032+0000","last_became_peered":"2024-09-13T14:28:10.583032+0000","last_unstale":"2024-09-13T14:38:12.966678+0000","last_undegraded":"2024-09-13T14:38:12.966678+0000","last_fullsized":"2024-09-13T14:38:12.966678+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:05:15.659274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968035+0000","last_change":"2024-09-13T14:37:09.284106+0000","last_active":"2024-09-13T14:38:12.968035+0000","last_peered":"2024-09-13T14:38:12.968035+0000","last_clean":"2024-09-13T14:38:12.968035+0000","last_became_active":"2024-09-13T14:37:09.283790+0000","last_became_peered":"2024-09-13T14:37:09.283790+0000","last_unstale":"2024-09-13T14:38:12.968035+0000","last_undegraded":"2024-09-13T14:38:12.968035+0000","last_fullsized":"2024-09-13T14:38:12.968035+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:41.476336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.970896+0000","last_change":"2024-09-13T14:35:50.986974+0000","last_active":"2024-09-13T14:38:13.970896+0000","last_peered":"2024-09-13T14:38:13.970896+0000","last_clean":"2024-09-13T14:38:13.970896+0000","last_became_active":"2024-09-13T14:35:50.978777+0000","last_became_peered":"2024-09-13T14:35:50.978777+0000","last_unstale":"2024-09-13T14:38:13.970896+0000","last_undegraded":"2024-09-13T14:38:13.970896+0000","last_fullsized":"2024-09-13T14:38:13.970896+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:03:44.859263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972699+0000","last_change":"2024-09-13T14:37:40.923570+0000","last_active":"2024-09-13T14:38:13.972699+0000","last_peered":"2024-09-13T14:38:13.972699+0000","last_clean":"2024-09-13T14:38:13.972699+0000","last_became_active":"2024-09-13T14:37:40.923133+0000","last_became_peered":"2024-09-13T14:37:40.923133+0000","last_unstale":"2024-09-13T14:38:13.972699+0000","last_undegraded":"2024-09-13T14:38:13.972699+0000","last_fullsized":"2024-09-13T14:38:13.972699+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:06:29.033083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.967638+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:38:12.967638+0000","last_peered":"2024-09-13T14:38:12.967638+0000","last_clean":"2024-09-13T14:38:12.967638+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:38:12.967638+0000","last_undegraded":"2024-09-13T14:38:12.967638+0000","last_fullsized":"2024-09-13T14:38:12.967638+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968772+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:38:12.968772+0000","last_peered":"2024-09-13T14:38:12.968772+0000","last_clean":"2024-09-13T14:38:12.968772+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:38:12.968772+0000","last_undegraded":"2024-09-13T14:38:12.968772+0000","last_fullsized":"2024-09-13T14:38:12.968772+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":24,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984514+0000","last_change":"2024-09-13T14:38:13.984429+0000","last_active":"2024-09-13T14:38:13.984514+0000","last_peered":"2024-09-13T14:38:13.984514+0000","last_clean":"2024-09-13T14:38:13.984514+0000","last_became_active":"2024-09-13T14:38:13.984142+0000","last_became_peered":"2024-09-13T14:38:13.984142+0000","last_unstale":"2024-09-13T14:38:13.984514+0000","last_undegraded":"2024-09-13T14:38:13.984514+0000","last_fullsized":"2024-09-13T14:38:13.984514+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:57.500962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":66,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978227+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:38:13.978227+0000","last_peered":"2024-09-13T14:38:13.978227+0000","last_clean":"2024-09-13T14:38:13.978227+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:38:13.978227+0000","last_undegraded":"2024-09-13T14:38:13.978227+0000","last_fullsized":"2024-09-13T14:38:13.978227+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984578+0000","last_change":"2024-09-13T14:38:13.984506+0000","last_active":"2024-09-13T14:38:13.984578+0000","last_peered":"2024-09-13T14:38:13.984578+0000","last_clean":"2024-09-13T14:38:13.984578+0000","last_became_active":"2024-09-13T14:38:13.984233+0000","last_became_peered":"2024-09-13T14:38:13.984233+0000","last_unstale":"2024-09-13T14:38:13.984578+0000","last_undegraded":"2024-09-13T14:38:13.984578+0000","last_fullsized":"2024-09-13T14:38:13.984578+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:03.382438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.973071+0000","last_change":"2024-09-13T14:37:09.284269+0000","last_active":"2024-09-13T14:38:12.973071+0000","last_peered":"2024-09-13T14:38:12.973071+0000","last_clean":"2024-09-13T14:38:12.973071+0000","last_became_active":"2024-09-13T14:37:09.283958+0000","last_became_peered":"2024-09-13T14:37:09.283958+0000","last_unstale":"2024-09-13T14:38:12.973071+0000","last_undegraded":"2024-09-13T14:38:12.973071+0000","last_fullsized":"2024-09-13T14:38:12.973071+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:48:30.577749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.981961+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:38:13.981961+0000","last_peered":"2024-09-13T14:38:13.981961+0000","last_clean":"2024-09-13T14:38:13.981961+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:38:13.981961+0000","last_undegraded":"2024-09-13T14:38:13.981961+0000","last_fullsized":"2024-09-13T14:38:13.981961+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979687+0000","last_change":"2024-09-13T14:38:13.979604+0000","last_active":"2024-09-13T14:38:13.979687+0000","last_peered":"2024-09-13T14:38:13.979687+0000","last_clean":"2024-09-13T14:38:13.979687+0000","last_became_active":"2024-09-13T14:38:13.979288+0000","last_became_peered":"2024-09-13T14:38:13.979288+0000","last_unstale":"2024-09-13T14:38:13.979687+0000","last_undegraded":"2024-09-13T14:38:13.979687+0000","last_fullsized":"2024-09-13T14:38:13.979687+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:21.031527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979348+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:38:13.979348+0000","last_peered":"2024-09-13T14:38:13.979348+0000","last_clean":"2024-09-13T14:38:13.979348+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:38:13.979348+0000","last_undegraded":"2024-09-13T14:38:13.979348+0000","last_fullsized":"2024-09-13T14:38:13.979348+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":87,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.983710+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:38:13.983710+0000","last_peered":"2024-09-13T14:38:13.983710+0000","last_clean":"2024-09-13T14:38:13.983710+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:38:13.983710+0000","last_undegraded":"2024-09-13T14:38:13.983710+0000","last_fullsized":"2024-09-13T14:38:13.983710+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":88,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.966152+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:38:12.966152+0000","last_peered":"2024-09-13T14:38:12.966152+0000","last_clean":"2024-09-13T14:38:12.966152+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:38:12.966152+0000","last_undegraded":"2024-09-13T14:38:12.966152+0000","last_fullsized":"2024-09-13T14:38:12.966152+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":93,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.977040+0000","last_change":"2024-09-13T14:38:13.976950+0000","last_active":"2024-09-13T14:38:13.977040+0000","last_peered":"2024-09-13T14:38:13.977040+0000","last_clean":"2024-09-13T14:38:13.977040+0000","last_became_active":"2024-09-13T14:38:13.976639+0000","last_became_peered":"2024-09-13T14:38:13.976639+0000","last_unstale":"2024-09-13T14:38:13.977040+0000","last_undegraded":"2024-09-13T14:38:13.977040+0000","last_fullsized":"2024-09-13T14:38:13.977040+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:24.126890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":59,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971216+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:38:13.971216+0000","last_peered":"2024-09-13T14:38:13.971216+0000","last_clean":"2024-09-13T14:38:13.971216+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:38:13.971216+0000","last_undegraded":"2024-09-13T14:38:13.971216+0000","last_fullsized":"2024-09-13T14:38:13.971216+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.980031+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:38:13.980031+0000","last_peered":"2024-09-13T14:38:13.980031+0000","last_clean":"2024-09-13T14:38:13.980031+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:38:13.980031+0000","last_undegraded":"2024-09-13T14:38:13.980031+0000","last_fullsized":"2024-09-13T14:38:13.980031+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969325+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:38:12.969325+0000","last_peered":"2024-09-13T14:38:12.969325+0000","last_clean":"2024-09-13T14:38:12.969325+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:38:12.969325+0000","last_undegraded":"2024-09-13T14:38:12.969325+0000","last_fullsized":"2024-09-13T14:38:12.969325+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.980507+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:38:13.980507+0000","last_peered":"2024-09-13T14:38:13.980507+0000","last_clean":"2024-09-13T14:38:13.980507+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:38:13.980507+0000","last_undegraded":"2024-09-13T14:38:13.980507+0000","last_fullsized":"2024-09-13T14:38:13.980507+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969671+0000","last_change":"2024-09-13T14:35:50.988294+0000","last_active":"2024-09-13T14:38:12.969671+0000","last_peered":"2024-09-13T14:38:12.969671+0000","last_clean":"2024-09-13T14:38:12.969671+0000","last_became_active":"2024-09-13T14:35:50.981513+0000","last_became_peered":"2024-09-13T14:35:50.981513+0000","last_unstale":"2024-09-13T14:38:12.969671+0000","last_undegraded":"2024-09-13T14:38:12.969671+0000","last_fullsized":"2024-09-13T14:38:12.969671+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:33.902687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":61,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968431+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:38:12.968431+0000","last_peered":"2024-09-13T14:38:12.968431+0000","last_clean":"2024-09-13T14:38:12.968431+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:38:12.968431+0000","last_undegraded":"2024-09-13T14:38:12.968431+0000","last_fullsized":"2024-09-13T14:38:12.968431+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":79,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984733+0000","last_change":"2024-09-13T14:38:13.984637+0000","last_active":"2024-09-13T14:38:13.984733+0000","last_peered":"2024-09-13T14:38:13.984733+0000","last_clean":"2024-09-13T14:38:13.984733+0000","last_became_active":"2024-09-13T14:38:13.984287+0000","last_became_peered":"2024-09-13T14:38:13.984287+0000","last_unstale":"2024-09-13T14:38:13.984733+0000","last_undegraded":"2024-09-13T14:38:13.984733+0000","last_fullsized":"2024-09-13T14:38:13.984733+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:47.110224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":62,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.965762+0000","last_change":"2024-09-13T14:28:52.557448+0000","last_active":"2024-09-13T14:38:12.965762+0000","last_peered":"2024-09-13T14:38:12.965762+0000","last_clean":"2024-09-13T14:38:12.965762+0000","last_became_active":"2024-09-13T14:28:52.557090+0000","last_became_peered":"2024-09-13T14:28:52.557090+0000","last_unstale":"2024-09-13T14:38:12.965762+0000","last_undegraded":"2024-09-13T14:38:12.965762+0000","last_fullsized":"2024-09-13T14:38:12.965762+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:59.236430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971028+0000","last_change":"2024-09-13T14:37:40.924860+0000","last_active":"2024-09-13T14:38:13.971028+0000","last_peered":"2024-09-13T14:38:13.971028+0000","last_clean":"2024-09-13T14:38:13.971028+0000","last_became_active":"2024-09-13T14:37:40.924522+0000","last_became_peered":"2024-09-13T14:37:40.924522+0000","last_unstale":"2024-09-13T14:38:13.971028+0000","last_undegraded":"2024-09-13T14:38:13.971028+0000","last_fullsized":"2024-09-13T14:38:13.971028+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:10.340511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4834,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397662,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303264,"kb_used_data":1744,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93441888,"statfs":{"total":95995035648,"available":95684493312,"internally_reserved":0,"allocated":1785856,"data_stored":1212181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593891,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307356,"kb_used_data":5832,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93437796,"statfs":{"total":95995035648,"available":95680303104,"internally_reserved":0,"allocated":5971968,"data_stored":5406057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790120,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302600,"kb_used_data":5244,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93442552,"statfs":{"total":95995035648,"available":95685173248,"internally_reserved":0,"allocated":5369856,"data_stored":4811586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986349,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298516,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446636,"statfs":{"total":95995035648,"available":95689355264,"internally_reserved":0,"allocated":1179648,"data_stored":617013,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598797,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28668,"kb_used_data":1136,"kb_used_omap":6,"kb_used_meta":27513,"kb_avail":93716484,"statfs":{"total":95995035648,"available":95965679616,"internally_reserved":0,"allocated":1163264,"data_stored":616651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":28174304},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762324,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28688,"kb_used_data":1148,"kb_used_omap":8,"kb_used_meta":27511,"kb_avail":93716464,"statfs":{"total":95995035648,"available":95965659136,"internally_reserved":0,"allocated":1175552,"data_stored":617277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":28171722},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726925850,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32780,"kb_used_data":5240,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93712372,"statfs":{"total":95995035648,"available":95961468928,"internally_reserved":0,"allocated":5365760,"data_stored":4811218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":28173001},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122090,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29280,"kb_used_data":1744,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93715872,"statfs":{"total":95995035648,"available":95965052928,"internally_reserved":0,"allocated":1785856,"data_stored":1212689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":28172879},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:39:11.769 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:39:12.011 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:12.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:12 smithi096 ceph-mon[98901]: from='client.34962 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:39:12.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:12 smithi096 ceph-mon[102928]: from='client.34962 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:39:12.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:12 smithi204 ceph-mon[87071]: from='client.34962 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:39:13.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:13 smithi096 ceph-mon[98901]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:13.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:13 smithi096 ceph-mon[102928]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:13.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:13 smithi204 ceph-mon[87071]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:13.797 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:39:13.798 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:39:14.553 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":225,"stamp":"2024-09-13T14:39:13.615232+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6304,"num_read_kb":13770,"num_write":1056,"num_write_kb":14251,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":595202,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1132,"ondisk_log_size":1132,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1331152,"kb_used_data":23240,"kb_used_omap":35,"kb_used_meta":1307740,"kb_avail":748630064,"statfs":{"total":767960285184,"available":766597185536,"internally_reserved":0,"allocated":23797760,"data_stored":19304672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36830,"internal_metadata":1339125794},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002687"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982673+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:38:13.982673+0000","last_peered":"2024-09-13T14:38:13.982673+0000","last_clean":"2024-09-13T14:38:13.982673+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:38:13.982673+0000","last_undegraded":"2024-09-13T14:38:13.982673+0000","last_fullsized":"2024-09-13T14:38:13.982673+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.967270+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:38:12.967270+0000","last_peered":"2024-09-13T14:38:12.967270+0000","last_clean":"2024-09-13T14:38:12.967270+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:38:12.967270+0000","last_undegraded":"2024-09-13T14:38:12.967270+0000","last_fullsized":"2024-09-13T14:38:12.967270+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979669+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:38:13.979669+0000","last_peered":"2024-09-13T14:38:13.979669+0000","last_clean":"2024-09-13T14:38:13.979669+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:38:13.979669+0000","last_undegraded":"2024-09-13T14:38:13.979669+0000","last_fullsized":"2024-09-13T14:38:13.979669+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":66,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978069+0000","last_change":"2024-09-13T14:38:13.977954+0000","last_active":"2024-09-13T14:38:13.978069+0000","last_peered":"2024-09-13T14:38:13.978069+0000","last_clean":"2024-09-13T14:38:13.978069+0000","last_became_active":"2024-09-13T14:38:13.977619+0000","last_became_peered":"2024-09-13T14:38:13.977619+0000","last_unstale":"2024-09-13T14:38:13.978069+0000","last_undegraded":"2024-09-13T14:38:13.978069+0000","last_fullsized":"2024-09-13T14:38:13.978069+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:58:30.259967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978307+0000","last_change":"2024-09-13T14:38:13.978232+0000","last_active":"2024-09-13T14:38:13.978307+0000","last_peered":"2024-09-13T14:38:13.978307+0000","last_clean":"2024-09-13T14:38:13.978307+0000","last_became_active":"2024-09-13T14:38:13.977921+0000","last_became_peered":"2024-09-13T14:38:13.977921+0000","last_unstale":"2024-09-13T14:38:13.978307+0000","last_undegraded":"2024-09-13T14:38:13.978307+0000","last_fullsized":"2024-09-13T14:38:13.978307+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:10.127626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969015+0000","last_change":"2024-09-13T14:28:10.508894+0000","last_active":"2024-09-13T14:38:12.969015+0000","last_peered":"2024-09-13T14:38:12.969015+0000","last_clean":"2024-09-13T14:38:12.969015+0000","last_became_active":"2024-09-13T14:28:10.508612+0000","last_became_peered":"2024-09-13T14:28:10.508612+0000","last_unstale":"2024-09-13T14:38:12.969015+0000","last_undegraded":"2024-09-13T14:38:12.969015+0000","last_fullsized":"2024-09-13T14:38:12.969015+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:01.774646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":53,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.970970+0000","last_change":"2024-09-13T14:37:40.925691+0000","last_active":"2024-09-13T14:38:13.970970+0000","last_peered":"2024-09-13T14:38:13.970970+0000","last_clean":"2024-09-13T14:38:13.970970+0000","last_became_active":"2024-09-13T14:37:40.925360+0000","last_became_peered":"2024-09-13T14:37:40.925360+0000","last_unstale":"2024-09-13T14:38:13.970970+0000","last_undegraded":"2024-09-13T14:38:13.970970+0000","last_fullsized":"2024-09-13T14:38:13.970970+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:07.337967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.971837+0000","last_change":"2024-09-13T14:35:49.984224+0000","last_active":"2024-09-13T14:38:12.971837+0000","last_peered":"2024-09-13T14:38:12.971837+0000","last_clean":"2024-09-13T14:38:12.971837+0000","last_became_active":"2024-09-13T14:35:49.983815+0000","last_became_peered":"2024-09-13T14:35:49.983815+0000","last_unstale":"2024-09-13T14:38:12.971837+0000","last_undegraded":"2024-09-13T14:38:12.971837+0000","last_fullsized":"2024-09-13T14:38:12.971837+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:27:07.882263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":57,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.973759+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:38:13.973759+0000","last_peered":"2024-09-13T14:38:13.973759+0000","last_clean":"2024-09-13T14:38:13.973759+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:38:13.973759+0000","last_undegraded":"2024-09-13T14:38:13.973759+0000","last_fullsized":"2024-09-13T14:38:13.973759+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968292+0000","last_change":"2024-09-13T14:28:11.507252+0000","last_active":"2024-09-13T14:38:12.968292+0000","last_peered":"2024-09-13T14:38:12.968292+0000","last_clean":"2024-09-13T14:38:12.968292+0000","last_became_active":"2024-09-13T14:28:11.506972+0000","last_became_peered":"2024-09-13T14:28:11.506972+0000","last_unstale":"2024-09-13T14:38:12.968292+0000","last_undegraded":"2024-09-13T14:38:12.968292+0000","last_fullsized":"2024-09-13T14:38:12.968292+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:01.583699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.970091+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:38:12.970091+0000","last_peered":"2024-09-13T14:38:12.970091+0000","last_clean":"2024-09-13T14:38:12.970091+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:38:12.970091+0000","last_undegraded":"2024-09-13T14:38:12.970091+0000","last_fullsized":"2024-09-13T14:38:12.970091+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":96,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979677+0000","last_change":"2024-09-13T14:38:13.979677+0000","last_active":"2024-09-13T14:38:13.979677+0000","last_peered":"2024-09-13T14:38:13.979677+0000","last_clean":"2024-09-13T14:38:13.979677+0000","last_became_active":"2024-09-13T14:38:13.979323+0000","last_became_peered":"2024-09-13T14:38:13.979323+0000","last_unstale":"2024-09-13T14:38:13.979677+0000","last_undegraded":"2024-09-13T14:38:13.979677+0000","last_fullsized":"2024-09-13T14:38:13.979677+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:29:03.491254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969851+0000","last_change":"2024-09-13T14:28:10.504588+0000","last_active":"2024-09-13T14:38:12.969851+0000","last_peered":"2024-09-13T14:38:12.969851+0000","last_clean":"2024-09-13T14:38:12.969851+0000","last_became_active":"2024-09-13T14:28:10.504338+0000","last_became_peered":"2024-09-13T14:28:10.504338+0000","last_unstale":"2024-09-13T14:38:12.969851+0000","last_undegraded":"2024-09-13T14:38:12.969851+0000","last_fullsized":"2024-09-13T14:38:12.969851+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:06:49.774993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.972936+0000","last_change":"2024-09-13T14:37:09.287528+0000","last_active":"2024-09-13T14:38:12.972936+0000","last_peered":"2024-09-13T14:38:12.972936+0000","last_clean":"2024-09-13T14:38:12.972936+0000","last_became_active":"2024-09-13T14:37:09.287182+0000","last_became_peered":"2024-09-13T14:37:09.287182+0000","last_unstale":"2024-09-13T14:38:12.972936+0000","last_undegraded":"2024-09-13T14:38:12.972936+0000","last_fullsized":"2024-09-13T14:38:12.972936+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:15.435336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982477+0000","last_change":"2024-09-13T14:38:13.982477+0000","last_active":"2024-09-13T14:38:13.982477+0000","last_peered":"2024-09-13T14:38:13.982477+0000","last_clean":"2024-09-13T14:38:13.982477+0000","last_became_active":"2024-09-13T14:38:13.982101+0000","last_became_peered":"2024-09-13T14:38:13.982101+0000","last_unstale":"2024-09-13T14:38:13.982477+0000","last_undegraded":"2024-09-13T14:38:13.982477+0000","last_fullsized":"2024-09-13T14:38:13.982477+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:24:07.896516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971326+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:38:13.971326+0000","last_peered":"2024-09-13T14:38:13.971326+0000","last_clean":"2024-09-13T14:38:13.971326+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:38:13.971326+0000","last_undegraded":"2024-09-13T14:38:13.971326+0000","last_fullsized":"2024-09-13T14:38:13.971326+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.983292+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:38:13.983292+0000","last_peered":"2024-09-13T14:38:13.983292+0000","last_clean":"2024-09-13T14:38:13.983292+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:38:13.983292+0000","last_undegraded":"2024-09-13T14:38:13.983292+0000","last_fullsized":"2024-09-13T14:38:13.983292+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982051+0000","last_change":"2024-09-13T14:38:13.982051+0000","last_active":"2024-09-13T14:38:13.982051+0000","last_peered":"2024-09-13T14:38:13.982051+0000","last_clean":"2024-09-13T14:38:13.982051+0000","last_became_active":"2024-09-13T14:38:13.981724+0000","last_became_peered":"2024-09-13T14:38:13.981724+0000","last_unstale":"2024-09-13T14:38:13.982051+0000","last_undegraded":"2024-09-13T14:38:13.982051+0000","last_fullsized":"2024-09-13T14:38:13.982051+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:52:34.815144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.972777+0000","last_change":"2024-09-13T14:37:09.286323+0000","last_active":"2024-09-13T14:38:12.972777+0000","last_peered":"2024-09-13T14:38:12.972777+0000","last_clean":"2024-09-13T14:38:12.972777+0000","last_became_active":"2024-09-13T14:37:09.285954+0000","last_became_peered":"2024-09-13T14:37:09.285954+0000","last_unstale":"2024-09-13T14:38:12.972777+0000","last_undegraded":"2024-09-13T14:38:12.972777+0000","last_fullsized":"2024-09-13T14:38:12.972777+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:23.527100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.967074+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:38:12.967074+0000","last_peered":"2024-09-13T14:38:12.967074+0000","last_clean":"2024-09-13T14:38:12.967074+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:38:12.967074+0000","last_undegraded":"2024-09-13T14:38:12.967074+0000","last_fullsized":"2024-09-13T14:38:12.967074+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":53,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.970670+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:38:13.970670+0000","last_peered":"2024-09-13T14:38:13.970670+0000","last_clean":"2024-09-13T14:38:13.970670+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:38:13.970670+0000","last_undegraded":"2024-09-13T14:38:13.970670+0000","last_fullsized":"2024-09-13T14:38:13.970670+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.966244+0000","last_change":"2024-09-13T14:37:09.289820+0000","last_active":"2024-09-13T14:38:12.966244+0000","last_peered":"2024-09-13T14:38:12.966244+0000","last_clean":"2024-09-13T14:38:12.966244+0000","last_became_active":"2024-09-13T14:37:09.289544+0000","last_became_peered":"2024-09-13T14:37:09.289544+0000","last_unstale":"2024-09-13T14:38:12.966244+0000","last_undegraded":"2024-09-13T14:38:12.966244+0000","last_fullsized":"2024-09-13T14:38:12.966244+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:23.255945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.981925+0000","last_change":"2024-09-13T14:38:13.981576+0000","last_active":"2024-09-13T14:38:13.981925+0000","last_peered":"2024-09-13T14:38:13.981925+0000","last_clean":"2024-09-13T14:38:13.981925+0000","last_became_active":"2024-09-13T14:38:13.981262+0000","last_became_peered":"2024-09-13T14:38:13.981262+0000","last_unstale":"2024-09-13T14:38:13.981925+0000","last_undegraded":"2024-09-13T14:38:13.981925+0000","last_fullsized":"2024-09-13T14:38:13.981925+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:20.284250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":54,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979392+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:38:13.979392+0000","last_peered":"2024-09-13T14:38:13.979392+0000","last_clean":"2024-09-13T14:38:13.979392+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:38:13.979392+0000","last_undegraded":"2024-09-13T14:38:13.979392+0000","last_fullsized":"2024-09-13T14:38:13.979392+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":186,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968558+0000","last_change":"2024-09-13T14:35:49.988378+0000","last_active":"2024-09-13T14:38:12.968558+0000","last_peered":"2024-09-13T14:38:12.968558+0000","last_clean":"2024-09-13T14:38:12.968558+0000","last_became_active":"2024-09-13T14:35:49.987987+0000","last_became_peered":"2024-09-13T14:35:49.987987+0000","last_unstale":"2024-09-13T14:38:12.968558+0000","last_undegraded":"2024-09-13T14:38:12.968558+0000","last_fullsized":"2024-09-13T14:38:12.968558+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:24:20.875473+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:32.753237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":632,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969114+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:38:12.969114+0000","last_peered":"2024-09-13T14:38:12.969114+0000","last_clean":"2024-09-13T14:38:12.969114+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:38:12.969114+0000","last_undegraded":"2024-09-13T14:38:12.969114+0000","last_fullsized":"2024-09-13T14:38:12.969114+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":24,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984733+0000","last_change":"2024-09-13T14:38:13.984643+0000","last_active":"2024-09-13T14:38:13.984733+0000","last_peered":"2024-09-13T14:38:13.984733+0000","last_clean":"2024-09-13T14:38:13.984733+0000","last_became_active":"2024-09-13T14:38:13.984316+0000","last_became_peered":"2024-09-13T14:38:13.984316+0000","last_unstale":"2024-09-13T14:38:13.984733+0000","last_undegraded":"2024-09-13T14:38:13.984733+0000","last_fullsized":"2024-09-13T14:38:13.984733+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:00:43.369510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.973242+0000","last_change":"2024-09-13T14:28:11.532509+0000","last_active":"2024-09-13T14:38:12.973242+0000","last_peered":"2024-09-13T14:38:12.973242+0000","last_clean":"2024-09-13T14:38:12.973242+0000","last_became_active":"2024-09-13T14:28:11.532207+0000","last_became_peered":"2024-09-13T14:28:11.532207+0000","last_unstale":"2024-09-13T14:38:12.973242+0000","last_undegraded":"2024-09-13T14:38:12.973242+0000","last_fullsized":"2024-09-13T14:38:12.973242+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:33.385182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":316,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968438+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:38:12.968438+0000","last_peered":"2024-09-13T14:38:12.968438+0000","last_clean":"2024-09-13T14:38:12.968438+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:38:12.968438+0000","last_undegraded":"2024-09-13T14:38:12.968438+0000","last_fullsized":"2024-09-13T14:38:12.968438+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978052+0000","last_change":"2024-09-13T14:38:13.977962+0000","last_active":"2024-09-13T14:38:13.978052+0000","last_peered":"2024-09-13T14:38:13.978052+0000","last_clean":"2024-09-13T14:38:13.978052+0000","last_became_active":"2024-09-13T14:38:13.977638+0000","last_became_peered":"2024-09-13T14:38:13.977638+0000","last_unstale":"2024-09-13T14:38:13.978052+0000","last_undegraded":"2024-09-13T14:38:13.978052+0000","last_fullsized":"2024-09-13T14:38:13.978052+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:53:16.902157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":78,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971451+0000","last_change":"2024-09-13T14:37:40.927444+0000","last_active":"2024-09-13T14:38:13.971451+0000","last_peered":"2024-09-13T14:38:13.971451+0000","last_clean":"2024-09-13T14:38:13.971451+0000","last_became_active":"2024-09-13T14:37:40.927140+0000","last_became_peered":"2024-09-13T14:37:40.927140+0000","last_unstale":"2024-09-13T14:38:13.971451+0000","last_undegraded":"2024-09-13T14:38:13.971451+0000","last_fullsized":"2024-09-13T14:38:13.971451+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:32:54.207568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":478,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978772+0000","last_change":"2024-09-13T14:38:13.978679+0000","last_active":"2024-09-13T14:38:13.978772+0000","last_peered":"2024-09-13T14:38:13.978772+0000","last_clean":"2024-09-13T14:38:13.978772+0000","last_became_active":"2024-09-13T14:38:13.978171+0000","last_became_peered":"2024-09-13T14:38:13.978171+0000","last_unstale":"2024-09-13T14:38:13.978772+0000","last_undegraded":"2024-09-13T14:38:13.978772+0000","last_fullsized":"2024-09-13T14:38:13.978772+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:57.691198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972597+0000","last_change":"2024-09-13T14:37:40.924713+0000","last_active":"2024-09-13T14:38:13.972597+0000","last_peered":"2024-09-13T14:38:13.972597+0000","last_clean":"2024-09-13T14:38:13.972597+0000","last_became_active":"2024-09-13T14:37:40.924435+0000","last_became_peered":"2024-09-13T14:37:40.924435+0000","last_unstale":"2024-09-13T14:38:13.972597+0000","last_undegraded":"2024-09-13T14:38:13.972597+0000","last_fullsized":"2024-09-13T14:38:13.972597+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:18:43.648821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":87,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971501+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:38:13.971501+0000","last_peered":"2024-09-13T14:38:13.971501+0000","last_clean":"2024-09-13T14:38:13.971501+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:38:13.971501+0000","last_undegraded":"2024-09-13T14:38:13.971501+0000","last_fullsized":"2024-09-13T14:38:13.971501+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":663,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969130+0000","last_change":"2024-09-13T14:37:09.291147+0000","last_active":"2024-09-13T14:38:12.969130+0000","last_peered":"2024-09-13T14:38:12.969130+0000","last_clean":"2024-09-13T14:38:12.969130+0000","last_became_active":"2024-09-13T14:37:09.290585+0000","last_became_peered":"2024-09-13T14:37:09.290585+0000","last_unstale":"2024-09-13T14:38:12.969130+0000","last_undegraded":"2024-09-13T14:38:12.969130+0000","last_fullsized":"2024-09-13T14:38:12.969130+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:32.764839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968123+0000","last_change":"2024-09-13T14:35:49.990342+0000","last_active":"2024-09-13T14:38:12.968123+0000","last_peered":"2024-09-13T14:38:12.968123+0000","last_clean":"2024-09-13T14:38:12.968123+0000","last_became_active":"2024-09-13T14:35:49.990053+0000","last_became_peered":"2024-09-13T14:35:49.990053+0000","last_unstale":"2024-09-13T14:38:12.968123+0000","last_undegraded":"2024-09-13T14:38:12.968123+0000","last_fullsized":"2024-09-13T14:38:12.968123+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:24:54.256709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":66,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.981713+0000","last_change":"2024-09-13T14:38:13.981294+0000","last_active":"2024-09-13T14:38:13.981713+0000","last_peered":"2024-09-13T14:38:13.981713+0000","last_clean":"2024-09-13T14:38:13.981713+0000","last_became_active":"2024-09-13T14:38:13.980975+0000","last_became_peered":"2024-09-13T14:38:13.980975+0000","last_unstale":"2024-09-13T14:38:13.981713+0000","last_undegraded":"2024-09-13T14:38:13.981713+0000","last_fullsized":"2024-09-13T14:38:13.981713+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:17.545179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":87,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.980578+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:38:13.980578+0000","last_peered":"2024-09-13T14:38:13.980578+0000","last_clean":"2024-09-13T14:38:13.980578+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:38:13.980578+0000","last_undegraded":"2024-09-13T14:38:13.980578+0000","last_fullsized":"2024-09-13T14:38:13.980578+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":572,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.973497+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:38:13.973497+0000","last_peered":"2024-09-13T14:38:13.973497+0000","last_clean":"2024-09-13T14:38:13.973497+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:38:13.973497+0000","last_undegraded":"2024-09-13T14:38:13.973497+0000","last_fullsized":"2024-09-13T14:38:13.973497+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972824+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:38:13.972824+0000","last_peered":"2024-09-13T14:38:13.972824+0000","last_clean":"2024-09-13T14:38:13.972824+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:38:13.972824+0000","last_undegraded":"2024-09-13T14:38:13.972824+0000","last_fullsized":"2024-09-13T14:38:13.972824+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":55,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972994+0000","last_change":"2024-09-13T14:37:40.921861+0000","last_active":"2024-09-13T14:38:13.972994+0000","last_peered":"2024-09-13T14:38:13.972994+0000","last_clean":"2024-09-13T14:38:13.972994+0000","last_became_active":"2024-09-13T14:37:40.921587+0000","last_became_peered":"2024-09-13T14:37:40.921587+0000","last_unstale":"2024-09-13T14:38:13.972994+0000","last_undegraded":"2024-09-13T14:38:13.972994+0000","last_fullsized":"2024-09-13T14:38:13.972994+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:24.619352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":318,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969979+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:38:12.969979+0000","last_peered":"2024-09-13T14:38:12.969979+0000","last_clean":"2024-09-13T14:38:12.969979+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:38:12.969979+0000","last_undegraded":"2024-09-13T14:38:12.969979+0000","last_fullsized":"2024-09-13T14:38:12.969979+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":56,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971289+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:38:13.971289+0000","last_peered":"2024-09-13T14:38:13.971289+0000","last_clean":"2024-09-13T14:38:13.971289+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:38:13.971289+0000","last_undegraded":"2024-09-13T14:38:13.971289+0000","last_fullsized":"2024-09-13T14:38:13.971289+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982751+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:38:13.982751+0000","last_peered":"2024-09-13T14:38:13.982751+0000","last_clean":"2024-09-13T14:38:13.982751+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:38:13.982751+0000","last_undegraded":"2024-09-13T14:38:13.982751+0000","last_fullsized":"2024-09-13T14:38:13.982751+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":358,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.973308+0000","last_change":"2024-09-13T14:37:09.288287+0000","last_active":"2024-09-13T14:38:13.973308+0000","last_peered":"2024-09-13T14:38:13.973308+0000","last_clean":"2024-09-13T14:38:13.973308+0000","last_became_active":"2024-09-13T14:37:09.287720+0000","last_became_peered":"2024-09-13T14:37:09.287720+0000","last_unstale":"2024-09-13T14:38:13.973308+0000","last_undegraded":"2024-09-13T14:38:13.973308+0000","last_fullsized":"2024-09-13T14:38:13.973308+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:09:34.761953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979342+0000","last_change":"2024-09-13T14:38:13.979255+0000","last_active":"2024-09-13T14:38:13.979342+0000","last_peered":"2024-09-13T14:38:13.979342+0000","last_clean":"2024-09-13T14:38:13.979342+0000","last_became_active":"2024-09-13T14:38:13.978868+0000","last_became_peered":"2024-09-13T14:38:13.978868+0000","last_unstale":"2024-09-13T14:38:13.979342+0000","last_undegraded":"2024-09-13T14:38:13.979342+0000","last_fullsized":"2024-09-13T14:38:13.979342+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:15:26.003538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":78,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971331+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:38:13.971331+0000","last_peered":"2024-09-13T14:38:13.971331+0000","last_clean":"2024-09-13T14:38:13.971331+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:38:13.971331+0000","last_undegraded":"2024-09-13T14:38:13.971331+0000","last_fullsized":"2024-09-13T14:38:13.971331+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1026,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.971972+0000","last_change":"2024-09-13T14:37:40.926427+0000","last_active":"2024-09-13T14:38:12.971972+0000","last_peered":"2024-09-13T14:38:12.971972+0000","last_clean":"2024-09-13T14:38:12.971972+0000","last_became_active":"2024-09-13T14:37:40.925922+0000","last_became_peered":"2024-09-13T14:37:40.925922+0000","last_unstale":"2024-09-13T14:38:12.971972+0000","last_undegraded":"2024-09-13T14:38:12.971972+0000","last_fullsized":"2024-09-13T14:38:12.971972+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:56:52.915505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.966678+0000","last_change":"2024-09-13T14:28:10.583355+0000","last_active":"2024-09-13T14:38:12.966678+0000","last_peered":"2024-09-13T14:38:12.966678+0000","last_clean":"2024-09-13T14:38:12.966678+0000","last_became_active":"2024-09-13T14:28:10.583032+0000","last_became_peered":"2024-09-13T14:28:10.583032+0000","last_unstale":"2024-09-13T14:38:12.966678+0000","last_undegraded":"2024-09-13T14:38:12.966678+0000","last_fullsized":"2024-09-13T14:38:12.966678+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:05:15.659274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968035+0000","last_change":"2024-09-13T14:37:09.284106+0000","last_active":"2024-09-13T14:38:12.968035+0000","last_peered":"2024-09-13T14:38:12.968035+0000","last_clean":"2024-09-13T14:38:12.968035+0000","last_became_active":"2024-09-13T14:37:09.283790+0000","last_became_peered":"2024-09-13T14:37:09.283790+0000","last_unstale":"2024-09-13T14:38:12.968035+0000","last_undegraded":"2024-09-13T14:38:12.968035+0000","last_fullsized":"2024-09-13T14:38:12.968035+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:41.476336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.970896+0000","last_change":"2024-09-13T14:35:50.986974+0000","last_active":"2024-09-13T14:38:13.970896+0000","last_peered":"2024-09-13T14:38:13.970896+0000","last_clean":"2024-09-13T14:38:13.970896+0000","last_became_active":"2024-09-13T14:35:50.978777+0000","last_became_peered":"2024-09-13T14:35:50.978777+0000","last_unstale":"2024-09-13T14:38:13.970896+0000","last_undegraded":"2024-09-13T14:38:13.970896+0000","last_fullsized":"2024-09-13T14:38:13.970896+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:03:44.859263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972699+0000","last_change":"2024-09-13T14:37:40.923570+0000","last_active":"2024-09-13T14:38:13.972699+0000","last_peered":"2024-09-13T14:38:13.972699+0000","last_clean":"2024-09-13T14:38:13.972699+0000","last_became_active":"2024-09-13T14:37:40.923133+0000","last_became_peered":"2024-09-13T14:37:40.923133+0000","last_unstale":"2024-09-13T14:38:13.972699+0000","last_undegraded":"2024-09-13T14:38:13.972699+0000","last_fullsized":"2024-09-13T14:38:13.972699+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:06:29.033083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.967638+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:38:12.967638+0000","last_peered":"2024-09-13T14:38:12.967638+0000","last_clean":"2024-09-13T14:38:12.967638+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:38:12.967638+0000","last_undegraded":"2024-09-13T14:38:12.967638+0000","last_fullsized":"2024-09-13T14:38:12.967638+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968772+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:38:12.968772+0000","last_peered":"2024-09-13T14:38:12.968772+0000","last_clean":"2024-09-13T14:38:12.968772+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:38:12.968772+0000","last_undegraded":"2024-09-13T14:38:12.968772+0000","last_fullsized":"2024-09-13T14:38:12.968772+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":24,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984514+0000","last_change":"2024-09-13T14:38:13.984429+0000","last_active":"2024-09-13T14:38:13.984514+0000","last_peered":"2024-09-13T14:38:13.984514+0000","last_clean":"2024-09-13T14:38:13.984514+0000","last_became_active":"2024-09-13T14:38:13.984142+0000","last_became_peered":"2024-09-13T14:38:13.984142+0000","last_unstale":"2024-09-13T14:38:13.984514+0000","last_undegraded":"2024-09-13T14:38:13.984514+0000","last_fullsized":"2024-09-13T14:38:13.984514+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:57.500962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":66,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978227+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:38:13.978227+0000","last_peered":"2024-09-13T14:38:13.978227+0000","last_clean":"2024-09-13T14:38:13.978227+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:38:13.978227+0000","last_undegraded":"2024-09-13T14:38:13.978227+0000","last_fullsized":"2024-09-13T14:38:13.978227+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984578+0000","last_change":"2024-09-13T14:38:13.984506+0000","last_active":"2024-09-13T14:38:13.984578+0000","last_peered":"2024-09-13T14:38:13.984578+0000","last_clean":"2024-09-13T14:38:13.984578+0000","last_became_active":"2024-09-13T14:38:13.984233+0000","last_became_peered":"2024-09-13T14:38:13.984233+0000","last_unstale":"2024-09-13T14:38:13.984578+0000","last_undegraded":"2024-09-13T14:38:13.984578+0000","last_fullsized":"2024-09-13T14:38:13.984578+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:03.382438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.973071+0000","last_change":"2024-09-13T14:37:09.284269+0000","last_active":"2024-09-13T14:38:12.973071+0000","last_peered":"2024-09-13T14:38:12.973071+0000","last_clean":"2024-09-13T14:38:12.973071+0000","last_became_active":"2024-09-13T14:37:09.283958+0000","last_became_peered":"2024-09-13T14:37:09.283958+0000","last_unstale":"2024-09-13T14:38:12.973071+0000","last_undegraded":"2024-09-13T14:38:12.973071+0000","last_fullsized":"2024-09-13T14:38:12.973071+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:48:30.577749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.981961+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:38:13.981961+0000","last_peered":"2024-09-13T14:38:13.981961+0000","last_clean":"2024-09-13T14:38:13.981961+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:38:13.981961+0000","last_undegraded":"2024-09-13T14:38:13.981961+0000","last_fullsized":"2024-09-13T14:38:13.981961+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979687+0000","last_change":"2024-09-13T14:38:13.979604+0000","last_active":"2024-09-13T14:38:13.979687+0000","last_peered":"2024-09-13T14:38:13.979687+0000","last_clean":"2024-09-13T14:38:13.979687+0000","last_became_active":"2024-09-13T14:38:13.979288+0000","last_became_peered":"2024-09-13T14:38:13.979288+0000","last_unstale":"2024-09-13T14:38:13.979687+0000","last_undegraded":"2024-09-13T14:38:13.979687+0000","last_fullsized":"2024-09-13T14:38:13.979687+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:21.031527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979348+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:38:13.979348+0000","last_peered":"2024-09-13T14:38:13.979348+0000","last_clean":"2024-09-13T14:38:13.979348+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:38:13.979348+0000","last_undegraded":"2024-09-13T14:38:13.979348+0000","last_fullsized":"2024-09-13T14:38:13.979348+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":87,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.983710+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:38:13.983710+0000","last_peered":"2024-09-13T14:38:13.983710+0000","last_clean":"2024-09-13T14:38:13.983710+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:38:13.983710+0000","last_undegraded":"2024-09-13T14:38:13.983710+0000","last_fullsized":"2024-09-13T14:38:13.983710+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":88,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.966152+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:38:12.966152+0000","last_peered":"2024-09-13T14:38:12.966152+0000","last_clean":"2024-09-13T14:38:12.966152+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:38:12.966152+0000","last_undegraded":"2024-09-13T14:38:12.966152+0000","last_fullsized":"2024-09-13T14:38:12.966152+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":93,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.977040+0000","last_change":"2024-09-13T14:38:13.976950+0000","last_active":"2024-09-13T14:38:13.977040+0000","last_peered":"2024-09-13T14:38:13.977040+0000","last_clean":"2024-09-13T14:38:13.977040+0000","last_became_active":"2024-09-13T14:38:13.976639+0000","last_became_peered":"2024-09-13T14:38:13.976639+0000","last_unstale":"2024-09-13T14:38:13.977040+0000","last_undegraded":"2024-09-13T14:38:13.977040+0000","last_fullsized":"2024-09-13T14:38:13.977040+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:24.126890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":59,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971216+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:38:13.971216+0000","last_peered":"2024-09-13T14:38:13.971216+0000","last_clean":"2024-09-13T14:38:13.971216+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:38:13.971216+0000","last_undegraded":"2024-09-13T14:38:13.971216+0000","last_fullsized":"2024-09-13T14:38:13.971216+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.980031+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:38:13.980031+0000","last_peered":"2024-09-13T14:38:13.980031+0000","last_clean":"2024-09-13T14:38:13.980031+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:38:13.980031+0000","last_undegraded":"2024-09-13T14:38:13.980031+0000","last_fullsized":"2024-09-13T14:38:13.980031+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969325+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:38:12.969325+0000","last_peered":"2024-09-13T14:38:12.969325+0000","last_clean":"2024-09-13T14:38:12.969325+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:38:12.969325+0000","last_undegraded":"2024-09-13T14:38:12.969325+0000","last_fullsized":"2024-09-13T14:38:12.969325+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.980507+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:38:13.980507+0000","last_peered":"2024-09-13T14:38:13.980507+0000","last_clean":"2024-09-13T14:38:13.980507+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:38:13.980507+0000","last_undegraded":"2024-09-13T14:38:13.980507+0000","last_fullsized":"2024-09-13T14:38:13.980507+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969671+0000","last_change":"2024-09-13T14:35:50.988294+0000","last_active":"2024-09-13T14:38:12.969671+0000","last_peered":"2024-09-13T14:38:12.969671+0000","last_clean":"2024-09-13T14:38:12.969671+0000","last_became_active":"2024-09-13T14:35:50.981513+0000","last_became_peered":"2024-09-13T14:35:50.981513+0000","last_unstale":"2024-09-13T14:38:12.969671+0000","last_undegraded":"2024-09-13T14:38:12.969671+0000","last_fullsized":"2024-09-13T14:38:12.969671+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:33.902687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":61,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968431+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:38:12.968431+0000","last_peered":"2024-09-13T14:38:12.968431+0000","last_clean":"2024-09-13T14:38:12.968431+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:38:12.968431+0000","last_undegraded":"2024-09-13T14:38:12.968431+0000","last_fullsized":"2024-09-13T14:38:12.968431+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":79,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984733+0000","last_change":"2024-09-13T14:38:13.984637+0000","last_active":"2024-09-13T14:38:13.984733+0000","last_peered":"2024-09-13T14:38:13.984733+0000","last_clean":"2024-09-13T14:38:13.984733+0000","last_became_active":"2024-09-13T14:38:13.984287+0000","last_became_peered":"2024-09-13T14:38:13.984287+0000","last_unstale":"2024-09-13T14:38:13.984733+0000","last_undegraded":"2024-09-13T14:38:13.984733+0000","last_fullsized":"2024-09-13T14:38:13.984733+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:47.110224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":62,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.965762+0000","last_change":"2024-09-13T14:28:52.557448+0000","last_active":"2024-09-13T14:38:12.965762+0000","last_peered":"2024-09-13T14:38:12.965762+0000","last_clean":"2024-09-13T14:38:12.965762+0000","last_became_active":"2024-09-13T14:28:52.557090+0000","last_became_peered":"2024-09-13T14:28:52.557090+0000","last_unstale":"2024-09-13T14:38:12.965762+0000","last_undegraded":"2024-09-13T14:38:12.965762+0000","last_fullsized":"2024-09-13T14:38:12.965762+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:59.236430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971028+0000","last_change":"2024-09-13T14:37:40.924860+0000","last_active":"2024-09-13T14:38:13.971028+0000","last_peered":"2024-09-13T14:38:13.971028+0000","last_clean":"2024-09-13T14:38:13.971028+0000","last_became_active":"2024-09-13T14:37:40.924522+0000","last_became_peered":"2024-09-13T14:37:40.924522+0000","last_unstale":"2024-09-13T14:38:13.971028+0000","last_undegraded":"2024-09-13T14:38:13.971028+0000","last_fullsized":"2024-09-13T14:38:13.971028+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:10.340511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4834,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397663,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303264,"kb_used_data":1744,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93441888,"statfs":{"total":95995035648,"available":95684493312,"internally_reserved":0,"allocated":1785856,"data_stored":1212181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593892,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307356,"kb_used_data":5832,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93437796,"statfs":{"total":95995035648,"available":95680303104,"internally_reserved":0,"allocated":5971968,"data_stored":5406057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790121,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302600,"kb_used_data":5244,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93442552,"statfs":{"total":95995035648,"available":95685173248,"internally_reserved":0,"allocated":5369856,"data_stored":4811586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986350,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298516,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446636,"statfs":{"total":95995035648,"available":95689355264,"internally_reserved":0,"allocated":1179648,"data_stored":617013,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598798,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28668,"kb_used_data":1136,"kb_used_omap":6,"kb_used_meta":27513,"kb_avail":93716484,"statfs":{"total":95995035648,"available":95965679616,"internally_reserved":0,"allocated":1163264,"data_stored":616651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":28174304},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762325,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28688,"kb_used_data":1148,"kb_used_omap":8,"kb_used_meta":27511,"kb_avail":93716464,"statfs":{"total":95995035648,"available":95965659136,"internally_reserved":0,"allocated":1175552,"data_stored":617277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":28171722},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726925851,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32780,"kb_used_data":5240,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93712372,"statfs":{"total":95995035648,"available":95961468928,"internally_reserved":0,"allocated":5365760,"data_stored":4811218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":28173001},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122091,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29280,"kb_used_data":1744,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93715872,"statfs":{"total":95995035648,"available":95965052928,"internally_reserved":0,"allocated":1785856,"data_stored":1212689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":28172879},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:39:14.556 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:39:14.556 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T14:39:14.556 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T14:39:14.556 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T14:39:14.801 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:15.273 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:15 smithi096 ceph-mon[98901]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:15.274 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:15 smithi096 ceph-mon[98901]: from='client.44837 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:39:15.274 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:15 smithi096 ceph-mon[102928]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:15.274 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:15 smithi096 ceph-mon[102928]: from='client.44837 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:39:15.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:15 smithi204 ceph-mon[87071]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:15.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:15 smithi204 ceph-mon[87071]: from='client.44837 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:39:16.558 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:39:16.559 INFO:teuthology.orchestra.run.smithi096.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T14:39:17.268 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T14:39:17.268 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:39:17.278 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2024-09-13T14:39:17.278 INFO:teuthology.run_tasks:Running task parallel... 2024-09-13T14:39:17.288 INFO:teuthology.task.parallel:starting parallel... 2024-09-13T14:39:17.289 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T14:39:17.289 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2024-09-13T14:39:17.291 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-13T14:39:17.292 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'wip-aclamk-cbt-combined'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'PG_AVAILABILITY'], 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']}, 'workunit': {'branch': 'wip-aclamk-cbt-combined', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}} 2024-09-13T14:39:17.292 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-13T14:39:17.292 INFO:tasks.thrashosds:Beginning thrashosds... 2024-09-13T14:39:17.292 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:39:17.293 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:39:17.315 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:17 smithi096 ceph-mon[98901]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:17.315 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:17 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2846231460' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-13T14:39:17.315 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:17 smithi096 ceph-mon[102928]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:17.315 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:17 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2846231460' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-13T14:39:17.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:17 smithi204 ceph-mon[87071]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:17.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:17 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2846231460' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-13T14:39:17.535 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:18.308 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:39:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:39:17] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.51.0" 2024-09-13T14:39:19.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:19 smithi096 ceph-mon[98901]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:19.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:19 smithi096 ceph-mon[102928]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:19.361 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:39:19.361 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:39:19.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:19 smithi204 ceph-mon[87071]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:20.014 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":227,"stamp":"2024-09-13T14:39:17.616118+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6304,"num_read_kb":13770,"num_write":1056,"num_write_kb":14251,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":595202,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1132,"ondisk_log_size":1132,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1331152,"kb_used_data":23240,"kb_used_omap":35,"kb_used_meta":1307740,"kb_avail":748630064,"statfs":{"total":767960285184,"available":766597185536,"internally_reserved":0,"allocated":23797760,"data_stored":19304672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36830,"internal_metadata":1339125794},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002489"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982673+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:38:13.982673+0000","last_peered":"2024-09-13T14:38:13.982673+0000","last_clean":"2024-09-13T14:38:13.982673+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:38:13.982673+0000","last_undegraded":"2024-09-13T14:38:13.982673+0000","last_fullsized":"2024-09-13T14:38:13.982673+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.967270+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:38:12.967270+0000","last_peered":"2024-09-13T14:38:12.967270+0000","last_clean":"2024-09-13T14:38:12.967270+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:38:12.967270+0000","last_undegraded":"2024-09-13T14:38:12.967270+0000","last_fullsized":"2024-09-13T14:38:12.967270+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979669+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:38:13.979669+0000","last_peered":"2024-09-13T14:38:13.979669+0000","last_clean":"2024-09-13T14:38:13.979669+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:38:13.979669+0000","last_undegraded":"2024-09-13T14:38:13.979669+0000","last_fullsized":"2024-09-13T14:38:13.979669+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":66,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978069+0000","last_change":"2024-09-13T14:38:13.977954+0000","last_active":"2024-09-13T14:38:13.978069+0000","last_peered":"2024-09-13T14:38:13.978069+0000","last_clean":"2024-09-13T14:38:13.978069+0000","last_became_active":"2024-09-13T14:38:13.977619+0000","last_became_peered":"2024-09-13T14:38:13.977619+0000","last_unstale":"2024-09-13T14:38:13.978069+0000","last_undegraded":"2024-09-13T14:38:13.978069+0000","last_fullsized":"2024-09-13T14:38:13.978069+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:58:30.259967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978307+0000","last_change":"2024-09-13T14:38:13.978232+0000","last_active":"2024-09-13T14:38:13.978307+0000","last_peered":"2024-09-13T14:38:13.978307+0000","last_clean":"2024-09-13T14:38:13.978307+0000","last_became_active":"2024-09-13T14:38:13.977921+0000","last_became_peered":"2024-09-13T14:38:13.977921+0000","last_unstale":"2024-09-13T14:38:13.978307+0000","last_undegraded":"2024-09-13T14:38:13.978307+0000","last_fullsized":"2024-09-13T14:38:13.978307+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:10.127626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969015+0000","last_change":"2024-09-13T14:28:10.508894+0000","last_active":"2024-09-13T14:38:12.969015+0000","last_peered":"2024-09-13T14:38:12.969015+0000","last_clean":"2024-09-13T14:38:12.969015+0000","last_became_active":"2024-09-13T14:28:10.508612+0000","last_became_peered":"2024-09-13T14:28:10.508612+0000","last_unstale":"2024-09-13T14:38:12.969015+0000","last_undegraded":"2024-09-13T14:38:12.969015+0000","last_fullsized":"2024-09-13T14:38:12.969015+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:01.774646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":53,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.970970+0000","last_change":"2024-09-13T14:37:40.925691+0000","last_active":"2024-09-13T14:38:13.970970+0000","last_peered":"2024-09-13T14:38:13.970970+0000","last_clean":"2024-09-13T14:38:13.970970+0000","last_became_active":"2024-09-13T14:37:40.925360+0000","last_became_peered":"2024-09-13T14:37:40.925360+0000","last_unstale":"2024-09-13T14:38:13.970970+0000","last_undegraded":"2024-09-13T14:38:13.970970+0000","last_fullsized":"2024-09-13T14:38:13.970970+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:07.337967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.971837+0000","last_change":"2024-09-13T14:35:49.984224+0000","last_active":"2024-09-13T14:38:12.971837+0000","last_peered":"2024-09-13T14:38:12.971837+0000","last_clean":"2024-09-13T14:38:12.971837+0000","last_became_active":"2024-09-13T14:35:49.983815+0000","last_became_peered":"2024-09-13T14:35:49.983815+0000","last_unstale":"2024-09-13T14:38:12.971837+0000","last_undegraded":"2024-09-13T14:38:12.971837+0000","last_fullsized":"2024-09-13T14:38:12.971837+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:27:07.882263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":57,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.973759+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:38:13.973759+0000","last_peered":"2024-09-13T14:38:13.973759+0000","last_clean":"2024-09-13T14:38:13.973759+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:38:13.973759+0000","last_undegraded":"2024-09-13T14:38:13.973759+0000","last_fullsized":"2024-09-13T14:38:13.973759+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968292+0000","last_change":"2024-09-13T14:28:11.507252+0000","last_active":"2024-09-13T14:38:12.968292+0000","last_peered":"2024-09-13T14:38:12.968292+0000","last_clean":"2024-09-13T14:38:12.968292+0000","last_became_active":"2024-09-13T14:28:11.506972+0000","last_became_peered":"2024-09-13T14:28:11.506972+0000","last_unstale":"2024-09-13T14:38:12.968292+0000","last_undegraded":"2024-09-13T14:38:12.968292+0000","last_fullsized":"2024-09-13T14:38:12.968292+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:01.583699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.970091+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:38:12.970091+0000","last_peered":"2024-09-13T14:38:12.970091+0000","last_clean":"2024-09-13T14:38:12.970091+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:38:12.970091+0000","last_undegraded":"2024-09-13T14:38:12.970091+0000","last_fullsized":"2024-09-13T14:38:12.970091+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":96,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979677+0000","last_change":"2024-09-13T14:38:13.979677+0000","last_active":"2024-09-13T14:38:13.979677+0000","last_peered":"2024-09-13T14:38:13.979677+0000","last_clean":"2024-09-13T14:38:13.979677+0000","last_became_active":"2024-09-13T14:38:13.979323+0000","last_became_peered":"2024-09-13T14:38:13.979323+0000","last_unstale":"2024-09-13T14:38:13.979677+0000","last_undegraded":"2024-09-13T14:38:13.979677+0000","last_fullsized":"2024-09-13T14:38:13.979677+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:29:03.491254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969851+0000","last_change":"2024-09-13T14:28:10.504588+0000","last_active":"2024-09-13T14:38:12.969851+0000","last_peered":"2024-09-13T14:38:12.969851+0000","last_clean":"2024-09-13T14:38:12.969851+0000","last_became_active":"2024-09-13T14:28:10.504338+0000","last_became_peered":"2024-09-13T14:28:10.504338+0000","last_unstale":"2024-09-13T14:38:12.969851+0000","last_undegraded":"2024-09-13T14:38:12.969851+0000","last_fullsized":"2024-09-13T14:38:12.969851+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:06:49.774993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.972936+0000","last_change":"2024-09-13T14:37:09.287528+0000","last_active":"2024-09-13T14:38:12.972936+0000","last_peered":"2024-09-13T14:38:12.972936+0000","last_clean":"2024-09-13T14:38:12.972936+0000","last_became_active":"2024-09-13T14:37:09.287182+0000","last_became_peered":"2024-09-13T14:37:09.287182+0000","last_unstale":"2024-09-13T14:38:12.972936+0000","last_undegraded":"2024-09-13T14:38:12.972936+0000","last_fullsized":"2024-09-13T14:38:12.972936+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:15.435336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982477+0000","last_change":"2024-09-13T14:38:13.982477+0000","last_active":"2024-09-13T14:38:13.982477+0000","last_peered":"2024-09-13T14:38:13.982477+0000","last_clean":"2024-09-13T14:38:13.982477+0000","last_became_active":"2024-09-13T14:38:13.982101+0000","last_became_peered":"2024-09-13T14:38:13.982101+0000","last_unstale":"2024-09-13T14:38:13.982477+0000","last_undegraded":"2024-09-13T14:38:13.982477+0000","last_fullsized":"2024-09-13T14:38:13.982477+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:24:07.896516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971326+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:38:13.971326+0000","last_peered":"2024-09-13T14:38:13.971326+0000","last_clean":"2024-09-13T14:38:13.971326+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:38:13.971326+0000","last_undegraded":"2024-09-13T14:38:13.971326+0000","last_fullsized":"2024-09-13T14:38:13.971326+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.983292+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:38:13.983292+0000","last_peered":"2024-09-13T14:38:13.983292+0000","last_clean":"2024-09-13T14:38:13.983292+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:38:13.983292+0000","last_undegraded":"2024-09-13T14:38:13.983292+0000","last_fullsized":"2024-09-13T14:38:13.983292+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982051+0000","last_change":"2024-09-13T14:38:13.982051+0000","last_active":"2024-09-13T14:38:13.982051+0000","last_peered":"2024-09-13T14:38:13.982051+0000","last_clean":"2024-09-13T14:38:13.982051+0000","last_became_active":"2024-09-13T14:38:13.981724+0000","last_became_peered":"2024-09-13T14:38:13.981724+0000","last_unstale":"2024-09-13T14:38:13.982051+0000","last_undegraded":"2024-09-13T14:38:13.982051+0000","last_fullsized":"2024-09-13T14:38:13.982051+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:52:34.815144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.972777+0000","last_change":"2024-09-13T14:37:09.286323+0000","last_active":"2024-09-13T14:38:12.972777+0000","last_peered":"2024-09-13T14:38:12.972777+0000","last_clean":"2024-09-13T14:38:12.972777+0000","last_became_active":"2024-09-13T14:37:09.285954+0000","last_became_peered":"2024-09-13T14:37:09.285954+0000","last_unstale":"2024-09-13T14:38:12.972777+0000","last_undegraded":"2024-09-13T14:38:12.972777+0000","last_fullsized":"2024-09-13T14:38:12.972777+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:23.527100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.967074+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:38:12.967074+0000","last_peered":"2024-09-13T14:38:12.967074+0000","last_clean":"2024-09-13T14:38:12.967074+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:38:12.967074+0000","last_undegraded":"2024-09-13T14:38:12.967074+0000","last_fullsized":"2024-09-13T14:38:12.967074+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":53,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.970670+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:38:13.970670+0000","last_peered":"2024-09-13T14:38:13.970670+0000","last_clean":"2024-09-13T14:38:13.970670+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:38:13.970670+0000","last_undegraded":"2024-09-13T14:38:13.970670+0000","last_fullsized":"2024-09-13T14:38:13.970670+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.966244+0000","last_change":"2024-09-13T14:37:09.289820+0000","last_active":"2024-09-13T14:38:12.966244+0000","last_peered":"2024-09-13T14:38:12.966244+0000","last_clean":"2024-09-13T14:38:12.966244+0000","last_became_active":"2024-09-13T14:37:09.289544+0000","last_became_peered":"2024-09-13T14:37:09.289544+0000","last_unstale":"2024-09-13T14:38:12.966244+0000","last_undegraded":"2024-09-13T14:38:12.966244+0000","last_fullsized":"2024-09-13T14:38:12.966244+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:23.255945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.981925+0000","last_change":"2024-09-13T14:38:13.981576+0000","last_active":"2024-09-13T14:38:13.981925+0000","last_peered":"2024-09-13T14:38:13.981925+0000","last_clean":"2024-09-13T14:38:13.981925+0000","last_became_active":"2024-09-13T14:38:13.981262+0000","last_became_peered":"2024-09-13T14:38:13.981262+0000","last_unstale":"2024-09-13T14:38:13.981925+0000","last_undegraded":"2024-09-13T14:38:13.981925+0000","last_fullsized":"2024-09-13T14:38:13.981925+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:20.284250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":54,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979392+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:38:13.979392+0000","last_peered":"2024-09-13T14:38:13.979392+0000","last_clean":"2024-09-13T14:38:13.979392+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:38:13.979392+0000","last_undegraded":"2024-09-13T14:38:13.979392+0000","last_fullsized":"2024-09-13T14:38:13.979392+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":186,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968558+0000","last_change":"2024-09-13T14:35:49.988378+0000","last_active":"2024-09-13T14:38:12.968558+0000","last_peered":"2024-09-13T14:38:12.968558+0000","last_clean":"2024-09-13T14:38:12.968558+0000","last_became_active":"2024-09-13T14:35:49.987987+0000","last_became_peered":"2024-09-13T14:35:49.987987+0000","last_unstale":"2024-09-13T14:38:12.968558+0000","last_undegraded":"2024-09-13T14:38:12.968558+0000","last_fullsized":"2024-09-13T14:38:12.968558+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:24:20.875473+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:32.753237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":632,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969114+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:38:12.969114+0000","last_peered":"2024-09-13T14:38:12.969114+0000","last_clean":"2024-09-13T14:38:12.969114+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:38:12.969114+0000","last_undegraded":"2024-09-13T14:38:12.969114+0000","last_fullsized":"2024-09-13T14:38:12.969114+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":24,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984733+0000","last_change":"2024-09-13T14:38:13.984643+0000","last_active":"2024-09-13T14:38:13.984733+0000","last_peered":"2024-09-13T14:38:13.984733+0000","last_clean":"2024-09-13T14:38:13.984733+0000","last_became_active":"2024-09-13T14:38:13.984316+0000","last_became_peered":"2024-09-13T14:38:13.984316+0000","last_unstale":"2024-09-13T14:38:13.984733+0000","last_undegraded":"2024-09-13T14:38:13.984733+0000","last_fullsized":"2024-09-13T14:38:13.984733+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:00:43.369510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.973242+0000","last_change":"2024-09-13T14:28:11.532509+0000","last_active":"2024-09-13T14:38:12.973242+0000","last_peered":"2024-09-13T14:38:12.973242+0000","last_clean":"2024-09-13T14:38:12.973242+0000","last_became_active":"2024-09-13T14:28:11.532207+0000","last_became_peered":"2024-09-13T14:28:11.532207+0000","last_unstale":"2024-09-13T14:38:12.973242+0000","last_undegraded":"2024-09-13T14:38:12.973242+0000","last_fullsized":"2024-09-13T14:38:12.973242+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:33.385182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":316,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968438+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:38:12.968438+0000","last_peered":"2024-09-13T14:38:12.968438+0000","last_clean":"2024-09-13T14:38:12.968438+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:38:12.968438+0000","last_undegraded":"2024-09-13T14:38:12.968438+0000","last_fullsized":"2024-09-13T14:38:12.968438+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978052+0000","last_change":"2024-09-13T14:38:13.977962+0000","last_active":"2024-09-13T14:38:13.978052+0000","last_peered":"2024-09-13T14:38:13.978052+0000","last_clean":"2024-09-13T14:38:13.978052+0000","last_became_active":"2024-09-13T14:38:13.977638+0000","last_became_peered":"2024-09-13T14:38:13.977638+0000","last_unstale":"2024-09-13T14:38:13.978052+0000","last_undegraded":"2024-09-13T14:38:13.978052+0000","last_fullsized":"2024-09-13T14:38:13.978052+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:53:16.902157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":78,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971451+0000","last_change":"2024-09-13T14:37:40.927444+0000","last_active":"2024-09-13T14:38:13.971451+0000","last_peered":"2024-09-13T14:38:13.971451+0000","last_clean":"2024-09-13T14:38:13.971451+0000","last_became_active":"2024-09-13T14:37:40.927140+0000","last_became_peered":"2024-09-13T14:37:40.927140+0000","last_unstale":"2024-09-13T14:38:13.971451+0000","last_undegraded":"2024-09-13T14:38:13.971451+0000","last_fullsized":"2024-09-13T14:38:13.971451+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:32:54.207568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":478,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978772+0000","last_change":"2024-09-13T14:38:13.978679+0000","last_active":"2024-09-13T14:38:13.978772+0000","last_peered":"2024-09-13T14:38:13.978772+0000","last_clean":"2024-09-13T14:38:13.978772+0000","last_became_active":"2024-09-13T14:38:13.978171+0000","last_became_peered":"2024-09-13T14:38:13.978171+0000","last_unstale":"2024-09-13T14:38:13.978772+0000","last_undegraded":"2024-09-13T14:38:13.978772+0000","last_fullsized":"2024-09-13T14:38:13.978772+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:57.691198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972597+0000","last_change":"2024-09-13T14:37:40.924713+0000","last_active":"2024-09-13T14:38:13.972597+0000","last_peered":"2024-09-13T14:38:13.972597+0000","last_clean":"2024-09-13T14:38:13.972597+0000","last_became_active":"2024-09-13T14:37:40.924435+0000","last_became_peered":"2024-09-13T14:37:40.924435+0000","last_unstale":"2024-09-13T14:38:13.972597+0000","last_undegraded":"2024-09-13T14:38:13.972597+0000","last_fullsized":"2024-09-13T14:38:13.972597+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:18:43.648821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971540+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:38:13.971540+0000","last_peered":"2024-09-13T14:38:13.971540+0000","last_clean":"2024-09-13T14:38:13.971540+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:38:13.971540+0000","last_undegraded":"2024-09-13T14:38:13.971540+0000","last_fullsized":"2024-09-13T14:38:13.971540+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":663,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969130+0000","last_change":"2024-09-13T14:37:09.291147+0000","last_active":"2024-09-13T14:38:12.969130+0000","last_peered":"2024-09-13T14:38:12.969130+0000","last_clean":"2024-09-13T14:38:12.969130+0000","last_became_active":"2024-09-13T14:37:09.290585+0000","last_became_peered":"2024-09-13T14:37:09.290585+0000","last_unstale":"2024-09-13T14:38:12.969130+0000","last_undegraded":"2024-09-13T14:38:12.969130+0000","last_fullsized":"2024-09-13T14:38:12.969130+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:32.764839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968123+0000","last_change":"2024-09-13T14:35:49.990342+0000","last_active":"2024-09-13T14:38:12.968123+0000","last_peered":"2024-09-13T14:38:12.968123+0000","last_clean":"2024-09-13T14:38:12.968123+0000","last_became_active":"2024-09-13T14:35:49.990053+0000","last_became_peered":"2024-09-13T14:35:49.990053+0000","last_unstale":"2024-09-13T14:38:12.968123+0000","last_undegraded":"2024-09-13T14:38:12.968123+0000","last_fullsized":"2024-09-13T14:38:12.968123+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:24:54.256709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":66,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.981713+0000","last_change":"2024-09-13T14:38:13.981294+0000","last_active":"2024-09-13T14:38:13.981713+0000","last_peered":"2024-09-13T14:38:13.981713+0000","last_clean":"2024-09-13T14:38:13.981713+0000","last_became_active":"2024-09-13T14:38:13.980975+0000","last_became_peered":"2024-09-13T14:38:13.980975+0000","last_unstale":"2024-09-13T14:38:13.981713+0000","last_undegraded":"2024-09-13T14:38:13.981713+0000","last_fullsized":"2024-09-13T14:38:13.981713+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:17.545179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":87,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.980578+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:38:13.980578+0000","last_peered":"2024-09-13T14:38:13.980578+0000","last_clean":"2024-09-13T14:38:13.980578+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:38:13.980578+0000","last_undegraded":"2024-09-13T14:38:13.980578+0000","last_fullsized":"2024-09-13T14:38:13.980578+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":572,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.973497+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:38:13.973497+0000","last_peered":"2024-09-13T14:38:13.973497+0000","last_clean":"2024-09-13T14:38:13.973497+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:38:13.973497+0000","last_undegraded":"2024-09-13T14:38:13.973497+0000","last_fullsized":"2024-09-13T14:38:13.973497+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972824+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:38:13.972824+0000","last_peered":"2024-09-13T14:38:13.972824+0000","last_clean":"2024-09-13T14:38:13.972824+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:38:13.972824+0000","last_undegraded":"2024-09-13T14:38:13.972824+0000","last_fullsized":"2024-09-13T14:38:13.972824+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":55,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972994+0000","last_change":"2024-09-13T14:37:40.921861+0000","last_active":"2024-09-13T14:38:13.972994+0000","last_peered":"2024-09-13T14:38:13.972994+0000","last_clean":"2024-09-13T14:38:13.972994+0000","last_became_active":"2024-09-13T14:37:40.921587+0000","last_became_peered":"2024-09-13T14:37:40.921587+0000","last_unstale":"2024-09-13T14:38:13.972994+0000","last_undegraded":"2024-09-13T14:38:13.972994+0000","last_fullsized":"2024-09-13T14:38:13.972994+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:24.619352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":318,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969979+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:38:12.969979+0000","last_peered":"2024-09-13T14:38:12.969979+0000","last_clean":"2024-09-13T14:38:12.969979+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:38:12.969979+0000","last_undegraded":"2024-09-13T14:38:12.969979+0000","last_fullsized":"2024-09-13T14:38:12.969979+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":56,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971289+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:38:13.971289+0000","last_peered":"2024-09-13T14:38:13.971289+0000","last_clean":"2024-09-13T14:38:13.971289+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:38:13.971289+0000","last_undegraded":"2024-09-13T14:38:13.971289+0000","last_fullsized":"2024-09-13T14:38:13.971289+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982751+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:38:13.982751+0000","last_peered":"2024-09-13T14:38:13.982751+0000","last_clean":"2024-09-13T14:38:13.982751+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:38:13.982751+0000","last_undegraded":"2024-09-13T14:38:13.982751+0000","last_fullsized":"2024-09-13T14:38:13.982751+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":359,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.973353+0000","last_change":"2024-09-13T14:37:09.288287+0000","last_active":"2024-09-13T14:38:13.973353+0000","last_peered":"2024-09-13T14:38:13.973353+0000","last_clean":"2024-09-13T14:38:13.973353+0000","last_became_active":"2024-09-13T14:37:09.287720+0000","last_became_peered":"2024-09-13T14:37:09.287720+0000","last_unstale":"2024-09-13T14:38:13.973353+0000","last_undegraded":"2024-09-13T14:38:13.973353+0000","last_fullsized":"2024-09-13T14:38:13.973353+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:09:34.761953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979342+0000","last_change":"2024-09-13T14:38:13.979255+0000","last_active":"2024-09-13T14:38:13.979342+0000","last_peered":"2024-09-13T14:38:13.979342+0000","last_clean":"2024-09-13T14:38:13.979342+0000","last_became_active":"2024-09-13T14:38:13.978868+0000","last_became_peered":"2024-09-13T14:38:13.978868+0000","last_unstale":"2024-09-13T14:38:13.979342+0000","last_undegraded":"2024-09-13T14:38:13.979342+0000","last_fullsized":"2024-09-13T14:38:13.979342+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:15:26.003538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":78,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971331+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:38:13.971331+0000","last_peered":"2024-09-13T14:38:13.971331+0000","last_clean":"2024-09-13T14:38:13.971331+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:38:13.971331+0000","last_undegraded":"2024-09-13T14:38:13.971331+0000","last_fullsized":"2024-09-13T14:38:13.971331+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1026,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.971972+0000","last_change":"2024-09-13T14:37:40.926427+0000","last_active":"2024-09-13T14:38:12.971972+0000","last_peered":"2024-09-13T14:38:12.971972+0000","last_clean":"2024-09-13T14:38:12.971972+0000","last_became_active":"2024-09-13T14:37:40.925922+0000","last_became_peered":"2024-09-13T14:37:40.925922+0000","last_unstale":"2024-09-13T14:38:12.971972+0000","last_undegraded":"2024-09-13T14:38:12.971972+0000","last_fullsized":"2024-09-13T14:38:12.971972+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:56:52.915505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.966678+0000","last_change":"2024-09-13T14:28:10.583355+0000","last_active":"2024-09-13T14:38:12.966678+0000","last_peered":"2024-09-13T14:38:12.966678+0000","last_clean":"2024-09-13T14:38:12.966678+0000","last_became_active":"2024-09-13T14:28:10.583032+0000","last_became_peered":"2024-09-13T14:28:10.583032+0000","last_unstale":"2024-09-13T14:38:12.966678+0000","last_undegraded":"2024-09-13T14:38:12.966678+0000","last_fullsized":"2024-09-13T14:38:12.966678+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:05:15.659274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968035+0000","last_change":"2024-09-13T14:37:09.284106+0000","last_active":"2024-09-13T14:38:12.968035+0000","last_peered":"2024-09-13T14:38:12.968035+0000","last_clean":"2024-09-13T14:38:12.968035+0000","last_became_active":"2024-09-13T14:37:09.283790+0000","last_became_peered":"2024-09-13T14:37:09.283790+0000","last_unstale":"2024-09-13T14:38:12.968035+0000","last_undegraded":"2024-09-13T14:38:12.968035+0000","last_fullsized":"2024-09-13T14:38:12.968035+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:41.476336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.970896+0000","last_change":"2024-09-13T14:35:50.986974+0000","last_active":"2024-09-13T14:38:13.970896+0000","last_peered":"2024-09-13T14:38:13.970896+0000","last_clean":"2024-09-13T14:38:13.970896+0000","last_became_active":"2024-09-13T14:35:50.978777+0000","last_became_peered":"2024-09-13T14:35:50.978777+0000","last_unstale":"2024-09-13T14:38:13.970896+0000","last_undegraded":"2024-09-13T14:38:13.970896+0000","last_fullsized":"2024-09-13T14:38:13.970896+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:03:44.859263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972699+0000","last_change":"2024-09-13T14:37:40.923570+0000","last_active":"2024-09-13T14:38:13.972699+0000","last_peered":"2024-09-13T14:38:13.972699+0000","last_clean":"2024-09-13T14:38:13.972699+0000","last_became_active":"2024-09-13T14:37:40.923133+0000","last_became_peered":"2024-09-13T14:37:40.923133+0000","last_unstale":"2024-09-13T14:38:13.972699+0000","last_undegraded":"2024-09-13T14:38:13.972699+0000","last_fullsized":"2024-09-13T14:38:13.972699+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:06:29.033083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.967638+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:38:12.967638+0000","last_peered":"2024-09-13T14:38:12.967638+0000","last_clean":"2024-09-13T14:38:12.967638+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:38:12.967638+0000","last_undegraded":"2024-09-13T14:38:12.967638+0000","last_fullsized":"2024-09-13T14:38:12.967638+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968772+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:38:12.968772+0000","last_peered":"2024-09-13T14:38:12.968772+0000","last_clean":"2024-09-13T14:38:12.968772+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:38:12.968772+0000","last_undegraded":"2024-09-13T14:38:12.968772+0000","last_fullsized":"2024-09-13T14:38:12.968772+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":24,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984514+0000","last_change":"2024-09-13T14:38:13.984429+0000","last_active":"2024-09-13T14:38:13.984514+0000","last_peered":"2024-09-13T14:38:13.984514+0000","last_clean":"2024-09-13T14:38:13.984514+0000","last_became_active":"2024-09-13T14:38:13.984142+0000","last_became_peered":"2024-09-13T14:38:13.984142+0000","last_unstale":"2024-09-13T14:38:13.984514+0000","last_undegraded":"2024-09-13T14:38:13.984514+0000","last_fullsized":"2024-09-13T14:38:13.984514+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:57.500962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":66,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978227+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:38:13.978227+0000","last_peered":"2024-09-13T14:38:13.978227+0000","last_clean":"2024-09-13T14:38:13.978227+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:38:13.978227+0000","last_undegraded":"2024-09-13T14:38:13.978227+0000","last_fullsized":"2024-09-13T14:38:13.978227+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984578+0000","last_change":"2024-09-13T14:38:13.984506+0000","last_active":"2024-09-13T14:38:13.984578+0000","last_peered":"2024-09-13T14:38:13.984578+0000","last_clean":"2024-09-13T14:38:13.984578+0000","last_became_active":"2024-09-13T14:38:13.984233+0000","last_became_peered":"2024-09-13T14:38:13.984233+0000","last_unstale":"2024-09-13T14:38:13.984578+0000","last_undegraded":"2024-09-13T14:38:13.984578+0000","last_fullsized":"2024-09-13T14:38:13.984578+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:03.382438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.973071+0000","last_change":"2024-09-13T14:37:09.284269+0000","last_active":"2024-09-13T14:38:12.973071+0000","last_peered":"2024-09-13T14:38:12.973071+0000","last_clean":"2024-09-13T14:38:12.973071+0000","last_became_active":"2024-09-13T14:37:09.283958+0000","last_became_peered":"2024-09-13T14:37:09.283958+0000","last_unstale":"2024-09-13T14:38:12.973071+0000","last_undegraded":"2024-09-13T14:38:12.973071+0000","last_fullsized":"2024-09-13T14:38:12.973071+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:48:30.577749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.981961+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:38:13.981961+0000","last_peered":"2024-09-13T14:38:13.981961+0000","last_clean":"2024-09-13T14:38:13.981961+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:38:13.981961+0000","last_undegraded":"2024-09-13T14:38:13.981961+0000","last_fullsized":"2024-09-13T14:38:13.981961+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979687+0000","last_change":"2024-09-13T14:38:13.979604+0000","last_active":"2024-09-13T14:38:13.979687+0000","last_peered":"2024-09-13T14:38:13.979687+0000","last_clean":"2024-09-13T14:38:13.979687+0000","last_became_active":"2024-09-13T14:38:13.979288+0000","last_became_peered":"2024-09-13T14:38:13.979288+0000","last_unstale":"2024-09-13T14:38:13.979687+0000","last_undegraded":"2024-09-13T14:38:13.979687+0000","last_fullsized":"2024-09-13T14:38:13.979687+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:21.031527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979348+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:38:13.979348+0000","last_peered":"2024-09-13T14:38:13.979348+0000","last_clean":"2024-09-13T14:38:13.979348+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:38:13.979348+0000","last_undegraded":"2024-09-13T14:38:13.979348+0000","last_fullsized":"2024-09-13T14:38:13.979348+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":87,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.983710+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:38:13.983710+0000","last_peered":"2024-09-13T14:38:13.983710+0000","last_clean":"2024-09-13T14:38:13.983710+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:38:13.983710+0000","last_undegraded":"2024-09-13T14:38:13.983710+0000","last_fullsized":"2024-09-13T14:38:13.983710+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":88,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.966152+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:38:12.966152+0000","last_peered":"2024-09-13T14:38:12.966152+0000","last_clean":"2024-09-13T14:38:12.966152+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:38:12.966152+0000","last_undegraded":"2024-09-13T14:38:12.966152+0000","last_fullsized":"2024-09-13T14:38:12.966152+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":93,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.977040+0000","last_change":"2024-09-13T14:38:13.976950+0000","last_active":"2024-09-13T14:38:13.977040+0000","last_peered":"2024-09-13T14:38:13.977040+0000","last_clean":"2024-09-13T14:38:13.977040+0000","last_became_active":"2024-09-13T14:38:13.976639+0000","last_became_peered":"2024-09-13T14:38:13.976639+0000","last_unstale":"2024-09-13T14:38:13.977040+0000","last_undegraded":"2024-09-13T14:38:13.977040+0000","last_fullsized":"2024-09-13T14:38:13.977040+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:24.126890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":59,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971216+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:38:13.971216+0000","last_peered":"2024-09-13T14:38:13.971216+0000","last_clean":"2024-09-13T14:38:13.971216+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:38:13.971216+0000","last_undegraded":"2024-09-13T14:38:13.971216+0000","last_fullsized":"2024-09-13T14:38:13.971216+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.980031+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:38:13.980031+0000","last_peered":"2024-09-13T14:38:13.980031+0000","last_clean":"2024-09-13T14:38:13.980031+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:38:13.980031+0000","last_undegraded":"2024-09-13T14:38:13.980031+0000","last_fullsized":"2024-09-13T14:38:13.980031+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969325+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:38:12.969325+0000","last_peered":"2024-09-13T14:38:12.969325+0000","last_clean":"2024-09-13T14:38:12.969325+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:38:12.969325+0000","last_undegraded":"2024-09-13T14:38:12.969325+0000","last_fullsized":"2024-09-13T14:38:12.969325+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.980507+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:38:13.980507+0000","last_peered":"2024-09-13T14:38:13.980507+0000","last_clean":"2024-09-13T14:38:13.980507+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:38:13.980507+0000","last_undegraded":"2024-09-13T14:38:13.980507+0000","last_fullsized":"2024-09-13T14:38:13.980507+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969671+0000","last_change":"2024-09-13T14:35:50.988294+0000","last_active":"2024-09-13T14:38:12.969671+0000","last_peered":"2024-09-13T14:38:12.969671+0000","last_clean":"2024-09-13T14:38:12.969671+0000","last_became_active":"2024-09-13T14:35:50.981513+0000","last_became_peered":"2024-09-13T14:35:50.981513+0000","last_unstale":"2024-09-13T14:38:12.969671+0000","last_undegraded":"2024-09-13T14:38:12.969671+0000","last_fullsized":"2024-09-13T14:38:12.969671+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:33.902687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":61,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968431+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:38:12.968431+0000","last_peered":"2024-09-13T14:38:12.968431+0000","last_clean":"2024-09-13T14:38:12.968431+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:38:12.968431+0000","last_undegraded":"2024-09-13T14:38:12.968431+0000","last_fullsized":"2024-09-13T14:38:12.968431+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":79,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984733+0000","last_change":"2024-09-13T14:38:13.984637+0000","last_active":"2024-09-13T14:38:13.984733+0000","last_peered":"2024-09-13T14:38:13.984733+0000","last_clean":"2024-09-13T14:38:13.984733+0000","last_became_active":"2024-09-13T14:38:13.984287+0000","last_became_peered":"2024-09-13T14:38:13.984287+0000","last_unstale":"2024-09-13T14:38:13.984733+0000","last_undegraded":"2024-09-13T14:38:13.984733+0000","last_fullsized":"2024-09-13T14:38:13.984733+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:47.110224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":62,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.965762+0000","last_change":"2024-09-13T14:28:52.557448+0000","last_active":"2024-09-13T14:38:12.965762+0000","last_peered":"2024-09-13T14:38:12.965762+0000","last_clean":"2024-09-13T14:38:12.965762+0000","last_became_active":"2024-09-13T14:28:52.557090+0000","last_became_peered":"2024-09-13T14:28:52.557090+0000","last_unstale":"2024-09-13T14:38:12.965762+0000","last_undegraded":"2024-09-13T14:38:12.965762+0000","last_fullsized":"2024-09-13T14:38:12.965762+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:59.236430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971028+0000","last_change":"2024-09-13T14:37:40.924860+0000","last_active":"2024-09-13T14:38:13.971028+0000","last_peered":"2024-09-13T14:38:13.971028+0000","last_clean":"2024-09-13T14:38:13.971028+0000","last_became_active":"2024-09-13T14:37:40.924522+0000","last_became_peered":"2024-09-13T14:37:40.924522+0000","last_unstale":"2024-09-13T14:38:13.971028+0000","last_undegraded":"2024-09-13T14:38:13.971028+0000","last_fullsized":"2024-09-13T14:38:13.971028+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:10.340511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4834,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397664,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303264,"kb_used_data":1744,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93441888,"statfs":{"total":95995035648,"available":95684493312,"internally_reserved":0,"allocated":1785856,"data_stored":1212181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593893,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307356,"kb_used_data":5832,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93437796,"statfs":{"total":95995035648,"available":95680303104,"internally_reserved":0,"allocated":5971968,"data_stored":5406057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790122,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302600,"kb_used_data":5244,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93442552,"statfs":{"total":95995035648,"available":95685173248,"internally_reserved":0,"allocated":5369856,"data_stored":4811586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986351,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298516,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446636,"statfs":{"total":95995035648,"available":95689355264,"internally_reserved":0,"allocated":1179648,"data_stored":617013,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598799,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28668,"kb_used_data":1136,"kb_used_omap":6,"kb_used_meta":27513,"kb_avail":93716484,"statfs":{"total":95995035648,"available":95965679616,"internally_reserved":0,"allocated":1163264,"data_stored":616651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":28174304},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762325,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28688,"kb_used_data":1148,"kb_used_omap":8,"kb_used_meta":27511,"kb_avail":93716464,"statfs":{"total":95995035648,"available":95965659136,"internally_reserved":0,"allocated":1175552,"data_stored":617277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":28171722},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726925852,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32780,"kb_used_data":5240,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93712372,"statfs":{"total":95995035648,"available":95961468928,"internally_reserved":0,"allocated":5365760,"data_stored":4811218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":28173001},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122091,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29280,"kb_used_data":1744,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93715872,"statfs":{"total":95995035648,"available":95965052928,"internally_reserved":0,"allocated":1785856,"data_stored":1212689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":28172879},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:39:20.017 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:39:20.296 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:20.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:20 smithi096 ceph-mon[98901]: from='client.44849 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:39:20.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:20 smithi096 ceph-mon[102928]: from='client.44849 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:39:20.371 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:20 smithi204 ceph-mon[87071]: from='client.44849 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:39:20.371 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:39:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:39:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:21.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:21 smithi096 ceph-mon[102928]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:21.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:21.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:21 smithi096 ceph-mon[98901]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:21.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:21.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:21 smithi204 ceph-mon[87071]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:21.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:22.023 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:39:22.023 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:39:22.617 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":229,"stamp":"2024-09-13T14:39:21.617144+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6304,"num_read_kb":13770,"num_write":1056,"num_write_kb":14251,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":595202,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1132,"ondisk_log_size":1132,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1331152,"kb_used_data":23240,"kb_used_omap":35,"kb_used_meta":1307740,"kb_avail":748630064,"statfs":{"total":767960285184,"available":766597185536,"internally_reserved":0,"allocated":23797760,"data_stored":19304672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36830,"internal_metadata":1339125794},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002648"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982673+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:38:13.982673+0000","last_peered":"2024-09-13T14:38:13.982673+0000","last_clean":"2024-09-13T14:38:13.982673+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:38:13.982673+0000","last_undegraded":"2024-09-13T14:38:13.982673+0000","last_fullsized":"2024-09-13T14:38:13.982673+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.967270+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:38:12.967270+0000","last_peered":"2024-09-13T14:38:12.967270+0000","last_clean":"2024-09-13T14:38:12.967270+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:38:12.967270+0000","last_undegraded":"2024-09-13T14:38:12.967270+0000","last_fullsized":"2024-09-13T14:38:12.967270+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979669+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:38:13.979669+0000","last_peered":"2024-09-13T14:38:13.979669+0000","last_clean":"2024-09-13T14:38:13.979669+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:38:13.979669+0000","last_undegraded":"2024-09-13T14:38:13.979669+0000","last_fullsized":"2024-09-13T14:38:13.979669+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":66,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978069+0000","last_change":"2024-09-13T14:38:13.977954+0000","last_active":"2024-09-13T14:38:13.978069+0000","last_peered":"2024-09-13T14:38:13.978069+0000","last_clean":"2024-09-13T14:38:13.978069+0000","last_became_active":"2024-09-13T14:38:13.977619+0000","last_became_peered":"2024-09-13T14:38:13.977619+0000","last_unstale":"2024-09-13T14:38:13.978069+0000","last_undegraded":"2024-09-13T14:38:13.978069+0000","last_fullsized":"2024-09-13T14:38:13.978069+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:58:30.259967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978307+0000","last_change":"2024-09-13T14:38:13.978232+0000","last_active":"2024-09-13T14:38:13.978307+0000","last_peered":"2024-09-13T14:38:13.978307+0000","last_clean":"2024-09-13T14:38:13.978307+0000","last_became_active":"2024-09-13T14:38:13.977921+0000","last_became_peered":"2024-09-13T14:38:13.977921+0000","last_unstale":"2024-09-13T14:38:13.978307+0000","last_undegraded":"2024-09-13T14:38:13.978307+0000","last_fullsized":"2024-09-13T14:38:13.978307+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:10.127626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969015+0000","last_change":"2024-09-13T14:28:10.508894+0000","last_active":"2024-09-13T14:38:12.969015+0000","last_peered":"2024-09-13T14:38:12.969015+0000","last_clean":"2024-09-13T14:38:12.969015+0000","last_became_active":"2024-09-13T14:28:10.508612+0000","last_became_peered":"2024-09-13T14:28:10.508612+0000","last_unstale":"2024-09-13T14:38:12.969015+0000","last_undegraded":"2024-09-13T14:38:12.969015+0000","last_fullsized":"2024-09-13T14:38:12.969015+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:01.774646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":53,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.970970+0000","last_change":"2024-09-13T14:37:40.925691+0000","last_active":"2024-09-13T14:38:13.970970+0000","last_peered":"2024-09-13T14:38:13.970970+0000","last_clean":"2024-09-13T14:38:13.970970+0000","last_became_active":"2024-09-13T14:37:40.925360+0000","last_became_peered":"2024-09-13T14:37:40.925360+0000","last_unstale":"2024-09-13T14:38:13.970970+0000","last_undegraded":"2024-09-13T14:38:13.970970+0000","last_fullsized":"2024-09-13T14:38:13.970970+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:07.337967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.971837+0000","last_change":"2024-09-13T14:35:49.984224+0000","last_active":"2024-09-13T14:38:12.971837+0000","last_peered":"2024-09-13T14:38:12.971837+0000","last_clean":"2024-09-13T14:38:12.971837+0000","last_became_active":"2024-09-13T14:35:49.983815+0000","last_became_peered":"2024-09-13T14:35:49.983815+0000","last_unstale":"2024-09-13T14:38:12.971837+0000","last_undegraded":"2024-09-13T14:38:12.971837+0000","last_fullsized":"2024-09-13T14:38:12.971837+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:27:07.882263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":57,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.973759+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:38:13.973759+0000","last_peered":"2024-09-13T14:38:13.973759+0000","last_clean":"2024-09-13T14:38:13.973759+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:38:13.973759+0000","last_undegraded":"2024-09-13T14:38:13.973759+0000","last_fullsized":"2024-09-13T14:38:13.973759+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968292+0000","last_change":"2024-09-13T14:28:11.507252+0000","last_active":"2024-09-13T14:38:12.968292+0000","last_peered":"2024-09-13T14:38:12.968292+0000","last_clean":"2024-09-13T14:38:12.968292+0000","last_became_active":"2024-09-13T14:28:11.506972+0000","last_became_peered":"2024-09-13T14:28:11.506972+0000","last_unstale":"2024-09-13T14:38:12.968292+0000","last_undegraded":"2024-09-13T14:38:12.968292+0000","last_fullsized":"2024-09-13T14:38:12.968292+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:01.583699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.970091+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:38:12.970091+0000","last_peered":"2024-09-13T14:38:12.970091+0000","last_clean":"2024-09-13T14:38:12.970091+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:38:12.970091+0000","last_undegraded":"2024-09-13T14:38:12.970091+0000","last_fullsized":"2024-09-13T14:38:12.970091+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":96,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979677+0000","last_change":"2024-09-13T14:38:13.979677+0000","last_active":"2024-09-13T14:38:13.979677+0000","last_peered":"2024-09-13T14:38:13.979677+0000","last_clean":"2024-09-13T14:38:13.979677+0000","last_became_active":"2024-09-13T14:38:13.979323+0000","last_became_peered":"2024-09-13T14:38:13.979323+0000","last_unstale":"2024-09-13T14:38:13.979677+0000","last_undegraded":"2024-09-13T14:38:13.979677+0000","last_fullsized":"2024-09-13T14:38:13.979677+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:29:03.491254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969851+0000","last_change":"2024-09-13T14:28:10.504588+0000","last_active":"2024-09-13T14:38:12.969851+0000","last_peered":"2024-09-13T14:38:12.969851+0000","last_clean":"2024-09-13T14:38:12.969851+0000","last_became_active":"2024-09-13T14:28:10.504338+0000","last_became_peered":"2024-09-13T14:28:10.504338+0000","last_unstale":"2024-09-13T14:38:12.969851+0000","last_undegraded":"2024-09-13T14:38:12.969851+0000","last_fullsized":"2024-09-13T14:38:12.969851+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:06:49.774993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.972936+0000","last_change":"2024-09-13T14:37:09.287528+0000","last_active":"2024-09-13T14:38:12.972936+0000","last_peered":"2024-09-13T14:38:12.972936+0000","last_clean":"2024-09-13T14:38:12.972936+0000","last_became_active":"2024-09-13T14:37:09.287182+0000","last_became_peered":"2024-09-13T14:37:09.287182+0000","last_unstale":"2024-09-13T14:38:12.972936+0000","last_undegraded":"2024-09-13T14:38:12.972936+0000","last_fullsized":"2024-09-13T14:38:12.972936+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:15.435336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982477+0000","last_change":"2024-09-13T14:38:13.982477+0000","last_active":"2024-09-13T14:38:13.982477+0000","last_peered":"2024-09-13T14:38:13.982477+0000","last_clean":"2024-09-13T14:38:13.982477+0000","last_became_active":"2024-09-13T14:38:13.982101+0000","last_became_peered":"2024-09-13T14:38:13.982101+0000","last_unstale":"2024-09-13T14:38:13.982477+0000","last_undegraded":"2024-09-13T14:38:13.982477+0000","last_fullsized":"2024-09-13T14:38:13.982477+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:24:07.896516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971326+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:38:13.971326+0000","last_peered":"2024-09-13T14:38:13.971326+0000","last_clean":"2024-09-13T14:38:13.971326+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:38:13.971326+0000","last_undegraded":"2024-09-13T14:38:13.971326+0000","last_fullsized":"2024-09-13T14:38:13.971326+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.983292+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:38:13.983292+0000","last_peered":"2024-09-13T14:38:13.983292+0000","last_clean":"2024-09-13T14:38:13.983292+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:38:13.983292+0000","last_undegraded":"2024-09-13T14:38:13.983292+0000","last_fullsized":"2024-09-13T14:38:13.983292+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982051+0000","last_change":"2024-09-13T14:38:13.982051+0000","last_active":"2024-09-13T14:38:13.982051+0000","last_peered":"2024-09-13T14:38:13.982051+0000","last_clean":"2024-09-13T14:38:13.982051+0000","last_became_active":"2024-09-13T14:38:13.981724+0000","last_became_peered":"2024-09-13T14:38:13.981724+0000","last_unstale":"2024-09-13T14:38:13.982051+0000","last_undegraded":"2024-09-13T14:38:13.982051+0000","last_fullsized":"2024-09-13T14:38:13.982051+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:52:34.815144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.972777+0000","last_change":"2024-09-13T14:37:09.286323+0000","last_active":"2024-09-13T14:38:12.972777+0000","last_peered":"2024-09-13T14:38:12.972777+0000","last_clean":"2024-09-13T14:38:12.972777+0000","last_became_active":"2024-09-13T14:37:09.285954+0000","last_became_peered":"2024-09-13T14:37:09.285954+0000","last_unstale":"2024-09-13T14:38:12.972777+0000","last_undegraded":"2024-09-13T14:38:12.972777+0000","last_fullsized":"2024-09-13T14:38:12.972777+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:23.527100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.967074+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:38:12.967074+0000","last_peered":"2024-09-13T14:38:12.967074+0000","last_clean":"2024-09-13T14:38:12.967074+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:38:12.967074+0000","last_undegraded":"2024-09-13T14:38:12.967074+0000","last_fullsized":"2024-09-13T14:38:12.967074+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":53,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.970670+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:38:13.970670+0000","last_peered":"2024-09-13T14:38:13.970670+0000","last_clean":"2024-09-13T14:38:13.970670+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:38:13.970670+0000","last_undegraded":"2024-09-13T14:38:13.970670+0000","last_fullsized":"2024-09-13T14:38:13.970670+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.966244+0000","last_change":"2024-09-13T14:37:09.289820+0000","last_active":"2024-09-13T14:38:12.966244+0000","last_peered":"2024-09-13T14:38:12.966244+0000","last_clean":"2024-09-13T14:38:12.966244+0000","last_became_active":"2024-09-13T14:37:09.289544+0000","last_became_peered":"2024-09-13T14:37:09.289544+0000","last_unstale":"2024-09-13T14:38:12.966244+0000","last_undegraded":"2024-09-13T14:38:12.966244+0000","last_fullsized":"2024-09-13T14:38:12.966244+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:23.255945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.981925+0000","last_change":"2024-09-13T14:38:13.981576+0000","last_active":"2024-09-13T14:38:13.981925+0000","last_peered":"2024-09-13T14:38:13.981925+0000","last_clean":"2024-09-13T14:38:13.981925+0000","last_became_active":"2024-09-13T14:38:13.981262+0000","last_became_peered":"2024-09-13T14:38:13.981262+0000","last_unstale":"2024-09-13T14:38:13.981925+0000","last_undegraded":"2024-09-13T14:38:13.981925+0000","last_fullsized":"2024-09-13T14:38:13.981925+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:20.284250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":54,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979392+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:38:13.979392+0000","last_peered":"2024-09-13T14:38:13.979392+0000","last_clean":"2024-09-13T14:38:13.979392+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:38:13.979392+0000","last_undegraded":"2024-09-13T14:38:13.979392+0000","last_fullsized":"2024-09-13T14:38:13.979392+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":186,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968558+0000","last_change":"2024-09-13T14:35:49.988378+0000","last_active":"2024-09-13T14:38:12.968558+0000","last_peered":"2024-09-13T14:38:12.968558+0000","last_clean":"2024-09-13T14:38:12.968558+0000","last_became_active":"2024-09-13T14:35:49.987987+0000","last_became_peered":"2024-09-13T14:35:49.987987+0000","last_unstale":"2024-09-13T14:38:12.968558+0000","last_undegraded":"2024-09-13T14:38:12.968558+0000","last_fullsized":"2024-09-13T14:38:12.968558+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:24:20.875473+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:32.753237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":632,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969114+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:38:12.969114+0000","last_peered":"2024-09-13T14:38:12.969114+0000","last_clean":"2024-09-13T14:38:12.969114+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:38:12.969114+0000","last_undegraded":"2024-09-13T14:38:12.969114+0000","last_fullsized":"2024-09-13T14:38:12.969114+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":24,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984733+0000","last_change":"2024-09-13T14:38:13.984643+0000","last_active":"2024-09-13T14:38:13.984733+0000","last_peered":"2024-09-13T14:38:13.984733+0000","last_clean":"2024-09-13T14:38:13.984733+0000","last_became_active":"2024-09-13T14:38:13.984316+0000","last_became_peered":"2024-09-13T14:38:13.984316+0000","last_unstale":"2024-09-13T14:38:13.984733+0000","last_undegraded":"2024-09-13T14:38:13.984733+0000","last_fullsized":"2024-09-13T14:38:13.984733+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:00:43.369510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.973242+0000","last_change":"2024-09-13T14:28:11.532509+0000","last_active":"2024-09-13T14:38:12.973242+0000","last_peered":"2024-09-13T14:38:12.973242+0000","last_clean":"2024-09-13T14:38:12.973242+0000","last_became_active":"2024-09-13T14:28:11.532207+0000","last_became_peered":"2024-09-13T14:28:11.532207+0000","last_unstale":"2024-09-13T14:38:12.973242+0000","last_undegraded":"2024-09-13T14:38:12.973242+0000","last_fullsized":"2024-09-13T14:38:12.973242+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:33.385182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":316,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968438+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:38:12.968438+0000","last_peered":"2024-09-13T14:38:12.968438+0000","last_clean":"2024-09-13T14:38:12.968438+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:38:12.968438+0000","last_undegraded":"2024-09-13T14:38:12.968438+0000","last_fullsized":"2024-09-13T14:38:12.968438+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978052+0000","last_change":"2024-09-13T14:38:13.977962+0000","last_active":"2024-09-13T14:38:13.978052+0000","last_peered":"2024-09-13T14:38:13.978052+0000","last_clean":"2024-09-13T14:38:13.978052+0000","last_became_active":"2024-09-13T14:38:13.977638+0000","last_became_peered":"2024-09-13T14:38:13.977638+0000","last_unstale":"2024-09-13T14:38:13.978052+0000","last_undegraded":"2024-09-13T14:38:13.978052+0000","last_fullsized":"2024-09-13T14:38:13.978052+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:53:16.902157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":78,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971451+0000","last_change":"2024-09-13T14:37:40.927444+0000","last_active":"2024-09-13T14:38:13.971451+0000","last_peered":"2024-09-13T14:38:13.971451+0000","last_clean":"2024-09-13T14:38:13.971451+0000","last_became_active":"2024-09-13T14:37:40.927140+0000","last_became_peered":"2024-09-13T14:37:40.927140+0000","last_unstale":"2024-09-13T14:38:13.971451+0000","last_undegraded":"2024-09-13T14:38:13.971451+0000","last_fullsized":"2024-09-13T14:38:13.971451+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:32:54.207568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":478,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978772+0000","last_change":"2024-09-13T14:38:13.978679+0000","last_active":"2024-09-13T14:38:13.978772+0000","last_peered":"2024-09-13T14:38:13.978772+0000","last_clean":"2024-09-13T14:38:13.978772+0000","last_became_active":"2024-09-13T14:38:13.978171+0000","last_became_peered":"2024-09-13T14:38:13.978171+0000","last_unstale":"2024-09-13T14:38:13.978772+0000","last_undegraded":"2024-09-13T14:38:13.978772+0000","last_fullsized":"2024-09-13T14:38:13.978772+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:57.691198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972597+0000","last_change":"2024-09-13T14:37:40.924713+0000","last_active":"2024-09-13T14:38:13.972597+0000","last_peered":"2024-09-13T14:38:13.972597+0000","last_clean":"2024-09-13T14:38:13.972597+0000","last_became_active":"2024-09-13T14:37:40.924435+0000","last_became_peered":"2024-09-13T14:37:40.924435+0000","last_unstale":"2024-09-13T14:38:13.972597+0000","last_undegraded":"2024-09-13T14:38:13.972597+0000","last_fullsized":"2024-09-13T14:38:13.972597+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:18:43.648821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971540+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:38:13.971540+0000","last_peered":"2024-09-13T14:38:13.971540+0000","last_clean":"2024-09-13T14:38:13.971540+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:38:13.971540+0000","last_undegraded":"2024-09-13T14:38:13.971540+0000","last_fullsized":"2024-09-13T14:38:13.971540+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":663,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969130+0000","last_change":"2024-09-13T14:37:09.291147+0000","last_active":"2024-09-13T14:38:12.969130+0000","last_peered":"2024-09-13T14:38:12.969130+0000","last_clean":"2024-09-13T14:38:12.969130+0000","last_became_active":"2024-09-13T14:37:09.290585+0000","last_became_peered":"2024-09-13T14:37:09.290585+0000","last_unstale":"2024-09-13T14:38:12.969130+0000","last_undegraded":"2024-09-13T14:38:12.969130+0000","last_fullsized":"2024-09-13T14:38:12.969130+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:32.764839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968123+0000","last_change":"2024-09-13T14:35:49.990342+0000","last_active":"2024-09-13T14:38:12.968123+0000","last_peered":"2024-09-13T14:38:12.968123+0000","last_clean":"2024-09-13T14:38:12.968123+0000","last_became_active":"2024-09-13T14:35:49.990053+0000","last_became_peered":"2024-09-13T14:35:49.990053+0000","last_unstale":"2024-09-13T14:38:12.968123+0000","last_undegraded":"2024-09-13T14:38:12.968123+0000","last_fullsized":"2024-09-13T14:38:12.968123+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:24:54.256709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":66,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.981713+0000","last_change":"2024-09-13T14:38:13.981294+0000","last_active":"2024-09-13T14:38:13.981713+0000","last_peered":"2024-09-13T14:38:13.981713+0000","last_clean":"2024-09-13T14:38:13.981713+0000","last_became_active":"2024-09-13T14:38:13.980975+0000","last_became_peered":"2024-09-13T14:38:13.980975+0000","last_unstale":"2024-09-13T14:38:13.981713+0000","last_undegraded":"2024-09-13T14:38:13.981713+0000","last_fullsized":"2024-09-13T14:38:13.981713+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:17.545179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":87,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.980578+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:38:13.980578+0000","last_peered":"2024-09-13T14:38:13.980578+0000","last_clean":"2024-09-13T14:38:13.980578+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:38:13.980578+0000","last_undegraded":"2024-09-13T14:38:13.980578+0000","last_fullsized":"2024-09-13T14:38:13.980578+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":572,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.973497+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:38:13.973497+0000","last_peered":"2024-09-13T14:38:13.973497+0000","last_clean":"2024-09-13T14:38:13.973497+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:38:13.973497+0000","last_undegraded":"2024-09-13T14:38:13.973497+0000","last_fullsized":"2024-09-13T14:38:13.973497+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972824+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:38:13.972824+0000","last_peered":"2024-09-13T14:38:13.972824+0000","last_clean":"2024-09-13T14:38:13.972824+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:38:13.972824+0000","last_undegraded":"2024-09-13T14:38:13.972824+0000","last_fullsized":"2024-09-13T14:38:13.972824+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":55,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972994+0000","last_change":"2024-09-13T14:37:40.921861+0000","last_active":"2024-09-13T14:38:13.972994+0000","last_peered":"2024-09-13T14:38:13.972994+0000","last_clean":"2024-09-13T14:38:13.972994+0000","last_became_active":"2024-09-13T14:37:40.921587+0000","last_became_peered":"2024-09-13T14:37:40.921587+0000","last_unstale":"2024-09-13T14:38:13.972994+0000","last_undegraded":"2024-09-13T14:38:13.972994+0000","last_fullsized":"2024-09-13T14:38:13.972994+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:24.619352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":318,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969979+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:38:12.969979+0000","last_peered":"2024-09-13T14:38:12.969979+0000","last_clean":"2024-09-13T14:38:12.969979+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:38:12.969979+0000","last_undegraded":"2024-09-13T14:38:12.969979+0000","last_fullsized":"2024-09-13T14:38:12.969979+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":56,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971289+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:38:13.971289+0000","last_peered":"2024-09-13T14:38:13.971289+0000","last_clean":"2024-09-13T14:38:13.971289+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:38:13.971289+0000","last_undegraded":"2024-09-13T14:38:13.971289+0000","last_fullsized":"2024-09-13T14:38:13.971289+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.982751+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:38:13.982751+0000","last_peered":"2024-09-13T14:38:13.982751+0000","last_clean":"2024-09-13T14:38:13.982751+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:38:13.982751+0000","last_undegraded":"2024-09-13T14:38:13.982751+0000","last_fullsized":"2024-09-13T14:38:13.982751+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":359,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.973353+0000","last_change":"2024-09-13T14:37:09.288287+0000","last_active":"2024-09-13T14:38:13.973353+0000","last_peered":"2024-09-13T14:38:13.973353+0000","last_clean":"2024-09-13T14:38:13.973353+0000","last_became_active":"2024-09-13T14:37:09.287720+0000","last_became_peered":"2024-09-13T14:37:09.287720+0000","last_unstale":"2024-09-13T14:38:13.973353+0000","last_undegraded":"2024-09-13T14:38:13.973353+0000","last_fullsized":"2024-09-13T14:38:13.973353+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:09:34.761953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979342+0000","last_change":"2024-09-13T14:38:13.979255+0000","last_active":"2024-09-13T14:38:13.979342+0000","last_peered":"2024-09-13T14:38:13.979342+0000","last_clean":"2024-09-13T14:38:13.979342+0000","last_became_active":"2024-09-13T14:38:13.978868+0000","last_became_peered":"2024-09-13T14:38:13.978868+0000","last_unstale":"2024-09-13T14:38:13.979342+0000","last_undegraded":"2024-09-13T14:38:13.979342+0000","last_fullsized":"2024-09-13T14:38:13.979342+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:15:26.003538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":78,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971331+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:38:13.971331+0000","last_peered":"2024-09-13T14:38:13.971331+0000","last_clean":"2024-09-13T14:38:13.971331+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:38:13.971331+0000","last_undegraded":"2024-09-13T14:38:13.971331+0000","last_fullsized":"2024-09-13T14:38:13.971331+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1026,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.971972+0000","last_change":"2024-09-13T14:37:40.926427+0000","last_active":"2024-09-13T14:38:12.971972+0000","last_peered":"2024-09-13T14:38:12.971972+0000","last_clean":"2024-09-13T14:38:12.971972+0000","last_became_active":"2024-09-13T14:37:40.925922+0000","last_became_peered":"2024-09-13T14:37:40.925922+0000","last_unstale":"2024-09-13T14:38:12.971972+0000","last_undegraded":"2024-09-13T14:38:12.971972+0000","last_fullsized":"2024-09-13T14:38:12.971972+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:56:52.915505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.966678+0000","last_change":"2024-09-13T14:28:10.583355+0000","last_active":"2024-09-13T14:38:12.966678+0000","last_peered":"2024-09-13T14:38:12.966678+0000","last_clean":"2024-09-13T14:38:12.966678+0000","last_became_active":"2024-09-13T14:28:10.583032+0000","last_became_peered":"2024-09-13T14:28:10.583032+0000","last_unstale":"2024-09-13T14:38:12.966678+0000","last_undegraded":"2024-09-13T14:38:12.966678+0000","last_fullsized":"2024-09-13T14:38:12.966678+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:05:15.659274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968035+0000","last_change":"2024-09-13T14:37:09.284106+0000","last_active":"2024-09-13T14:38:12.968035+0000","last_peered":"2024-09-13T14:38:12.968035+0000","last_clean":"2024-09-13T14:38:12.968035+0000","last_became_active":"2024-09-13T14:37:09.283790+0000","last_became_peered":"2024-09-13T14:37:09.283790+0000","last_unstale":"2024-09-13T14:38:12.968035+0000","last_undegraded":"2024-09-13T14:38:12.968035+0000","last_fullsized":"2024-09-13T14:38:12.968035+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:41.476336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.970896+0000","last_change":"2024-09-13T14:35:50.986974+0000","last_active":"2024-09-13T14:38:13.970896+0000","last_peered":"2024-09-13T14:38:13.970896+0000","last_clean":"2024-09-13T14:38:13.970896+0000","last_became_active":"2024-09-13T14:35:50.978777+0000","last_became_peered":"2024-09-13T14:35:50.978777+0000","last_unstale":"2024-09-13T14:38:13.970896+0000","last_undegraded":"2024-09-13T14:38:13.970896+0000","last_fullsized":"2024-09-13T14:38:13.970896+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:03:44.859263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.972699+0000","last_change":"2024-09-13T14:37:40.923570+0000","last_active":"2024-09-13T14:38:13.972699+0000","last_peered":"2024-09-13T14:38:13.972699+0000","last_clean":"2024-09-13T14:38:13.972699+0000","last_became_active":"2024-09-13T14:37:40.923133+0000","last_became_peered":"2024-09-13T14:37:40.923133+0000","last_unstale":"2024-09-13T14:38:13.972699+0000","last_undegraded":"2024-09-13T14:38:13.972699+0000","last_fullsized":"2024-09-13T14:38:13.972699+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:06:29.033083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.967638+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:38:12.967638+0000","last_peered":"2024-09-13T14:38:12.967638+0000","last_clean":"2024-09-13T14:38:12.967638+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:38:12.967638+0000","last_undegraded":"2024-09-13T14:38:12.967638+0000","last_fullsized":"2024-09-13T14:38:12.967638+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968772+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:38:12.968772+0000","last_peered":"2024-09-13T14:38:12.968772+0000","last_clean":"2024-09-13T14:38:12.968772+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:38:12.968772+0000","last_undegraded":"2024-09-13T14:38:12.968772+0000","last_fullsized":"2024-09-13T14:38:12.968772+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":24,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984514+0000","last_change":"2024-09-13T14:38:13.984429+0000","last_active":"2024-09-13T14:38:13.984514+0000","last_peered":"2024-09-13T14:38:13.984514+0000","last_clean":"2024-09-13T14:38:13.984514+0000","last_became_active":"2024-09-13T14:38:13.984142+0000","last_became_peered":"2024-09-13T14:38:13.984142+0000","last_unstale":"2024-09-13T14:38:13.984514+0000","last_undegraded":"2024-09-13T14:38:13.984514+0000","last_fullsized":"2024-09-13T14:38:13.984514+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:57.500962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":66,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.978227+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:38:13.978227+0000","last_peered":"2024-09-13T14:38:13.978227+0000","last_clean":"2024-09-13T14:38:13.978227+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:38:13.978227+0000","last_undegraded":"2024-09-13T14:38:13.978227+0000","last_fullsized":"2024-09-13T14:38:13.978227+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984578+0000","last_change":"2024-09-13T14:38:13.984506+0000","last_active":"2024-09-13T14:38:13.984578+0000","last_peered":"2024-09-13T14:38:13.984578+0000","last_clean":"2024-09-13T14:38:13.984578+0000","last_became_active":"2024-09-13T14:38:13.984233+0000","last_became_peered":"2024-09-13T14:38:13.984233+0000","last_unstale":"2024-09-13T14:38:13.984578+0000","last_undegraded":"2024-09-13T14:38:13.984578+0000","last_fullsized":"2024-09-13T14:38:13.984578+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:03.382438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.973071+0000","last_change":"2024-09-13T14:37:09.284269+0000","last_active":"2024-09-13T14:38:12.973071+0000","last_peered":"2024-09-13T14:38:12.973071+0000","last_clean":"2024-09-13T14:38:12.973071+0000","last_became_active":"2024-09-13T14:37:09.283958+0000","last_became_peered":"2024-09-13T14:37:09.283958+0000","last_unstale":"2024-09-13T14:38:12.973071+0000","last_undegraded":"2024-09-13T14:38:12.973071+0000","last_fullsized":"2024-09-13T14:38:12.973071+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:48:30.577749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.981961+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:38:13.981961+0000","last_peered":"2024-09-13T14:38:13.981961+0000","last_clean":"2024-09-13T14:38:13.981961+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:38:13.981961+0000","last_undegraded":"2024-09-13T14:38:13.981961+0000","last_fullsized":"2024-09-13T14:38:13.981961+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":67,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979687+0000","last_change":"2024-09-13T14:38:13.979604+0000","last_active":"2024-09-13T14:38:13.979687+0000","last_peered":"2024-09-13T14:38:13.979687+0000","last_clean":"2024-09-13T14:38:13.979687+0000","last_became_active":"2024-09-13T14:38:13.979288+0000","last_became_peered":"2024-09-13T14:38:13.979288+0000","last_unstale":"2024-09-13T14:38:13.979687+0000","last_undegraded":"2024-09-13T14:38:13.979687+0000","last_fullsized":"2024-09-13T14:38:13.979687+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:21.031527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.979348+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:38:13.979348+0000","last_peered":"2024-09-13T14:38:13.979348+0000","last_clean":"2024-09-13T14:38:13.979348+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:38:13.979348+0000","last_undegraded":"2024-09-13T14:38:13.979348+0000","last_fullsized":"2024-09-13T14:38:13.979348+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":87,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.983710+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:38:13.983710+0000","last_peered":"2024-09-13T14:38:13.983710+0000","last_clean":"2024-09-13T14:38:13.983710+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:38:13.983710+0000","last_undegraded":"2024-09-13T14:38:13.983710+0000","last_fullsized":"2024-09-13T14:38:13.983710+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":88,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.966152+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:38:12.966152+0000","last_peered":"2024-09-13T14:38:12.966152+0000","last_clean":"2024-09-13T14:38:12.966152+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:38:12.966152+0000","last_undegraded":"2024-09-13T14:38:12.966152+0000","last_fullsized":"2024-09-13T14:38:12.966152+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":93,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.977040+0000","last_change":"2024-09-13T14:38:13.976950+0000","last_active":"2024-09-13T14:38:13.977040+0000","last_peered":"2024-09-13T14:38:13.977040+0000","last_clean":"2024-09-13T14:38:13.977040+0000","last_became_active":"2024-09-13T14:38:13.976639+0000","last_became_peered":"2024-09-13T14:38:13.976639+0000","last_unstale":"2024-09-13T14:38:13.977040+0000","last_undegraded":"2024-09-13T14:38:13.977040+0000","last_fullsized":"2024-09-13T14:38:13.977040+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:24.126890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":59,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971216+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:38:13.971216+0000","last_peered":"2024-09-13T14:38:13.971216+0000","last_clean":"2024-09-13T14:38:13.971216+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:38:13.971216+0000","last_undegraded":"2024-09-13T14:38:13.971216+0000","last_fullsized":"2024-09-13T14:38:13.971216+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.980031+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:38:13.980031+0000","last_peered":"2024-09-13T14:38:13.980031+0000","last_clean":"2024-09-13T14:38:13.980031+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:38:13.980031+0000","last_undegraded":"2024-09-13T14:38:13.980031+0000","last_fullsized":"2024-09-13T14:38:13.980031+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969325+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:38:12.969325+0000","last_peered":"2024-09-13T14:38:12.969325+0000","last_clean":"2024-09-13T14:38:12.969325+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:38:12.969325+0000","last_undegraded":"2024-09-13T14:38:12.969325+0000","last_fullsized":"2024-09-13T14:38:12.969325+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.980507+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:38:13.980507+0000","last_peered":"2024-09-13T14:38:13.980507+0000","last_clean":"2024-09-13T14:38:13.980507+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:38:13.980507+0000","last_undegraded":"2024-09-13T14:38:13.980507+0000","last_fullsized":"2024-09-13T14:38:13.980507+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.969671+0000","last_change":"2024-09-13T14:35:50.988294+0000","last_active":"2024-09-13T14:38:12.969671+0000","last_peered":"2024-09-13T14:38:12.969671+0000","last_clean":"2024-09-13T14:38:12.969671+0000","last_became_active":"2024-09-13T14:35:50.981513+0000","last_became_peered":"2024-09-13T14:35:50.981513+0000","last_unstale":"2024-09-13T14:38:12.969671+0000","last_undegraded":"2024-09-13T14:38:12.969671+0000","last_fullsized":"2024-09-13T14:38:12.969671+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:33.902687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":61,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.968431+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:38:12.968431+0000","last_peered":"2024-09-13T14:38:12.968431+0000","last_clean":"2024-09-13T14:38:12.968431+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:38:12.968431+0000","last_undegraded":"2024-09-13T14:38:12.968431+0000","last_fullsized":"2024-09-13T14:38:12.968431+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":79,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.984733+0000","last_change":"2024-09-13T14:38:13.984637+0000","last_active":"2024-09-13T14:38:13.984733+0000","last_peered":"2024-09-13T14:38:13.984733+0000","last_clean":"2024-09-13T14:38:13.984733+0000","last_became_active":"2024-09-13T14:38:13.984287+0000","last_became_peered":"2024-09-13T14:38:13.984287+0000","last_unstale":"2024-09-13T14:38:13.984733+0000","last_undegraded":"2024-09-13T14:38:13.984733+0000","last_fullsized":"2024-09-13T14:38:13.984733+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:47.110224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":62,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-09-13T14:38:12.965762+0000","last_change":"2024-09-13T14:28:52.557448+0000","last_active":"2024-09-13T14:38:12.965762+0000","last_peered":"2024-09-13T14:38:12.965762+0000","last_clean":"2024-09-13T14:38:12.965762+0000","last_became_active":"2024-09-13T14:28:52.557090+0000","last_became_peered":"2024-09-13T14:28:52.557090+0000","last_unstale":"2024-09-13T14:38:12.965762+0000","last_undegraded":"2024-09-13T14:38:12.965762+0000","last_fullsized":"2024-09-13T14:38:12.965762+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:59.236430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-09-13T14:38:13.971028+0000","last_change":"2024-09-13T14:37:40.924860+0000","last_active":"2024-09-13T14:38:13.971028+0000","last_peered":"2024-09-13T14:38:13.971028+0000","last_clean":"2024-09-13T14:38:13.971028+0000","last_became_active":"2024-09-13T14:37:40.924522+0000","last_became_peered":"2024-09-13T14:37:40.924522+0000","last_unstale":"2024-09-13T14:38:13.971028+0000","last_undegraded":"2024-09-13T14:38:13.971028+0000","last_fullsized":"2024-09-13T14:38:13.971028+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:10.340511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4834,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397665,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303264,"kb_used_data":1744,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93441888,"statfs":{"total":95995035648,"available":95684493312,"internally_reserved":0,"allocated":1785856,"data_stored":1212181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593894,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307356,"kb_used_data":5832,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93437796,"statfs":{"total":95995035648,"available":95680303104,"internally_reserved":0,"allocated":5971968,"data_stored":5406057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790123,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302600,"kb_used_data":5244,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93442552,"statfs":{"total":95995035648,"available":95685173248,"internally_reserved":0,"allocated":5369856,"data_stored":4811586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986352,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298516,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446636,"statfs":{"total":95995035648,"available":95689355264,"internally_reserved":0,"allocated":1179648,"data_stored":617013,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598800,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28668,"kb_used_data":1136,"kb_used_omap":6,"kb_used_meta":27513,"kb_avail":93716484,"statfs":{"total":95995035648,"available":95965679616,"internally_reserved":0,"allocated":1163264,"data_stored":616651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":28174304},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762326,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28688,"kb_used_data":1148,"kb_used_omap":8,"kb_used_meta":27511,"kb_avail":93716464,"statfs":{"total":95995035648,"available":95965659136,"internally_reserved":0,"allocated":1175552,"data_stored":617277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":28171722},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726925852,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32780,"kb_used_data":5240,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93712372,"statfs":{"total":95995035648,"available":95961468928,"internally_reserved":0,"allocated":5365760,"data_stored":4811218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":28173001},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122092,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29280,"kb_used_data":1744,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93715872,"statfs":{"total":95995035648,"available":95965052928,"internally_reserved":0,"allocated":1785856,"data_stored":1212689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":28172879},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:39:22.620 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:39:22.621 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd dump 2024-09-13T14:39:22.866 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:23.285 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:23 smithi096 ceph-mon[98901]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:23.285 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:23 smithi096 ceph-mon[98901]: from='client.34986 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:39:23.285 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:23 smithi096 ceph-mon[102928]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:23.286 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:23 smithi096 ceph-mon[102928]: from='client.34986 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:39:23.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:23 smithi204 ceph-mon[87071]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:23.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:23 smithi204 ceph-mon[87071]: from='client.34986 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:39:24.665 INFO:teuthology.orchestra.run.smithi096.stdout:epoch 105 2024-09-13T14:39:24.665 INFO:teuthology.orchestra.run.smithi096.stdout:fsid 51e50ed6-71db-11ef-bceb-c7b262605968 2024-09-13T14:39:24.665 INFO:teuthology.orchestra.run.smithi096.stdout:created 2024-09-13T14:21:16.367777+0000 2024-09-13T14:39:24.665 INFO:teuthology.orchestra.run.smithi096.stdout:modified 2024-09-13T14:38:13.960806+0000 2024-09-13T14:39:24.665 INFO:teuthology.orchestra.run.smithi096.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-09-13T14:39:24.666 INFO:teuthology.orchestra.run.smithi096.stdout:crush_version 22 2024-09-13T14:39:24.666 INFO:teuthology.orchestra.run.smithi096.stdout:full_ratio 0.95 2024-09-13T14:39:24.666 INFO:teuthology.orchestra.run.smithi096.stdout:backfillfull_ratio 0.9 2024-09-13T14:39:24.666 INFO:teuthology.orchestra.run.smithi096.stdout:nearfull_ratio 0.85 2024-09-13T14:39:24.666 INFO:teuthology.orchestra.run.smithi096.stdout:require_min_compat_client luminous 2024-09-13T14:39:24.666 INFO:teuthology.orchestra.run.smithi096.stdout:min_compat_client luminous 2024-09-13T14:39:24.666 INFO:teuthology.orchestra.run.smithi096.stdout:require_osd_release quincy 2024-09-13T14:39:24.666 INFO:teuthology.orchestra.run.smithi096.stdout:stretch_mode_enabled false 2024-09-13T14:39:24.666 INFO:teuthology.orchestra.run.smithi096.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2024-09-13T14:39:24.666 INFO:teuthology.orchestra.run.smithi096.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 78 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2024-09-13T14:39:24.666 INFO:teuthology.orchestra.run.smithi096.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 63 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 2.00 2024-09-13T14:39:24.666 INFO:teuthology.orchestra.run.smithi096.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 64 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2024-09-13T14:39:24.666 INFO:teuthology.orchestra.run.smithi096.stdout:max_osd 8 2024-09-13T14:39:24.666 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0 up in weight 1 up_from 88 up_thru 104 down_at 83 last_clean_interval [8,82) [v2:172.21.15.96:6802/2849588101,v1:172.21.15.96:6803/2849588101] [v2:172.21.15.96:6804/2849588101,v1:172.21.15.96:6805/2849588101] exists,up eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed 2024-09-13T14:39:24.667 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1 up in weight 1 up_from 94 up_thru 104 down_at 91 last_clean_interval [13,90) [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923] [v2:172.21.15.96:6812/289441923,v1:172.21.15.96:6813/289441923] exists,up 1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7 2024-09-13T14:39:24.667 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2 up in weight 1 up_from 99 up_thru 104 down_at 96 last_clean_interval [18,95) [v2:172.21.15.96:6818/4202655894,v1:172.21.15.96:6819/4202655894] [v2:172.21.15.96:6820/4202655894,v1:172.21.15.96:6821/4202655894] exists,up 6a92a5b3-bdc7-499a-8b0f-2b0936fd9598 2024-09-13T14:39:24.667 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3 up in weight 1 up_from 104 up_thru 104 down_at 101 last_clean_interval [26,100) [v2:172.21.15.96:6826/3452335742,v1:172.21.15.96:6827/3452335742] [v2:172.21.15.96:6828/3452335742,v1:172.21.15.96:6829/3452335742] exists,up 1230646c-934e-4eb4-b4b9-16be52ffba66 2024-09-13T14:39:24.667 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4 up in weight 1 up_from 31 up_thru 104 down_at 0 last_clean_interval [0,0) [v2:172.21.15.204:6800/1188704632,v1:172.21.15.204:6801/1188704632] [v2:172.21.15.204:6802/1188704632,v1:172.21.15.204:6803/1188704632] exists,up 14115269-3d41-4ebf-8613-71ed59ec0f29 2024-09-13T14:39:24.667 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5 up in weight 1 up_from 37 up_thru 104 down_at 0 last_clean_interval [0,0) [v2:172.21.15.204:6808/580729752,v1:172.21.15.204:6809/580729752] [v2:172.21.15.204:6810/580729752,v1:172.21.15.204:6811/580729752] exists,up 15148a0c-df44-419e-b0f0-2b1600c207f4 2024-09-13T14:39:24.667 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6 up in weight 1 up_from 43 up_thru 101 down_at 0 last_clean_interval [0,0) [v2:172.21.15.204:6816/118365497,v1:172.21.15.204:6817/118365497] [v2:172.21.15.204:6818/118365497,v1:172.21.15.204:6819/118365497] exists,up 8404e088-3523-4c21-b7c2-0a31b22e0279 2024-09-13T14:39:24.667 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7 up in weight 1 up_from 49 up_thru 101 down_at 0 last_clean_interval [0,0) [v2:172.21.15.204:6824/3529639424,v1:172.21.15.204:6825/3529639424] [v2:172.21.15.204:6826/3529639424,v1:172.21.15.204:6827/3529639424] exists,up 93b23a35-0059-4e03-a2ba-28fee6431892 2024-09-13T14:39:24.667 INFO:teuthology.orchestra.run.smithi096.stdout:pg_upmap_items 3.1 [4,2] 2024-09-13T14:39:24.667 INFO:teuthology.orchestra.run.smithi096.stdout:pg_upmap_items 3.6 [4,2] 2024-09-13T14:39:24.667 INFO:teuthology.orchestra.run.smithi096.stdout:pg_upmap_items 3.9 [4,0] 2024-09-13T14:39:24.667 INFO:teuthology.orchestra.run.smithi096.stdout:pg_upmap_items 3.f [7,2] 2024-09-13T14:39:24.667 INFO:teuthology.orchestra.run.smithi096.stdout:pg_upmap_items 3.14 [4,0] 2024-09-13T14:39:24.667 INFO:teuthology.orchestra.run.smithi096.stdout:pg_upmap_items 3.15 [3,0] 2024-09-13T14:39:24.668 INFO:teuthology.orchestra.run.smithi096.stdout:pg_upmap_items 3.1d [4,0] 2024-09-13T14:39:24.668 INFO:teuthology.orchestra.run.smithi096.stdout:pg_upmap_items 4.9 [1,2] 2024-09-13T14:39:24.668 INFO:teuthology.orchestra.run.smithi096.stdout:pg_upmap_items 4.17 [1,2,3,0] 2024-09-13T14:39:24.668 INFO:teuthology.orchestra.run.smithi096.stdout:pg_upmap_items 4.18 [1,5] 2024-09-13T14:39:24.668 INFO:teuthology.orchestra.run.smithi096.stdout:pg_upmap_items 4.19 [3,5] 2024-09-13T14:39:24.668 INFO:teuthology.orchestra.run.smithi096.stdout:pg_upmap_items 4.1d [1,2] 2024-09-13T14:39:24.668 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/2744909732 expires 2024-09-14T14:32:35.504695+0000 2024-09-13T14:39:24.668 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/2862942348 expires 2024-09-14T14:32:35.504695+0000 2024-09-13T14:39:24.668 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:6800/2724610080 expires 2024-09-14T14:32:35.504695+0000 2024-09-13T14:39:24.668 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/1761759058 expires 2024-09-14T14:32:35.504695+0000 2024-09-13T14:39:24.668 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/3329894409 expires 2024-09-14T14:32:35.504695+0000 2024-09-13T14:39:24.668 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:6800/776211071 expires 2024-09-14T14:30:47.049718+0000 2024-09-13T14:39:24.668 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:6801/776211071 expires 2024-09-14T14:30:47.049718+0000 2024-09-13T14:39:24.669 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:6800/2749644659 expires 2024-09-14T14:26:50.514276+0000 2024-09-13T14:39:24.669 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/902309444 expires 2024-09-14T14:30:47.049718+0000 2024-09-13T14:39:24.669 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/752011643 expires 2024-09-14T14:26:50.514276+0000 2024-09-13T14:39:24.669 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/1494034928 expires 2024-09-14T14:22:08.207226+0000 2024-09-13T14:39:24.669 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/1652886381 expires 2024-09-14T14:30:47.049718+0000 2024-09-13T14:39:24.669 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.204:0/3779551551 expires 2024-09-14T14:31:56.307623+0000 2024-09-13T14:39:24.669 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/19119516 expires 2024-09-14T14:22:08.207226+0000 2024-09-13T14:39:24.669 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:6801/2724610080 expires 2024-09-14T14:32:35.504695+0000 2024-09-13T14:39:24.669 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/3050864675 expires 2024-09-14T14:30:47.049718+0000 2024-09-13T14:39:24.669 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:6801/3509914111 expires 2024-09-14T14:21:45.592521+0000 2024-09-13T14:39:24.669 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.204:0/4067568101 expires 2024-09-14T14:31:56.307623+0000 2024-09-13T14:39:24.669 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/2292899363 expires 2024-09-14T14:21:45.592521+0000 2024-09-13T14:39:24.669 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.204:0/4099954218 expires 2024-09-14T14:31:56.307623+0000 2024-09-13T14:39:24.669 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/960238088 expires 2024-09-14T14:26:50.514276+0000 2024-09-13T14:39:24.670 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/260172418 expires 2024-09-14T14:26:50.514276+0000 2024-09-13T14:39:24.670 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:6800/3509914111 expires 2024-09-14T14:21:45.592521+0000 2024-09-13T14:39:24.670 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/1644412166 expires 2024-09-14T14:22:08.207226+0000 2024-09-13T14:39:24.670 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:6801/2749644659 expires 2024-09-14T14:26:50.514276+0000 2024-09-13T14:39:24.670 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/3782790019 expires 2024-09-14T14:30:47.049718+0000 2024-09-13T14:39:24.670 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.204:0/3780556706 expires 2024-09-14T14:31:56.307623+0000 2024-09-13T14:39:24.670 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/376655558 expires 2024-09-14T14:21:45.592521+0000 2024-09-13T14:39:24.670 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/378263291 expires 2024-09-14T14:26:50.514276+0000 2024-09-13T14:39:24.670 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.204:6834/2474101951 expires 2024-09-14T14:31:56.307623+0000 2024-09-13T14:39:24.670 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.204:6835/2474101951 expires 2024-09-14T14:31:56.307623+0000 2024-09-13T14:39:24.670 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.204:0/1453713144 expires 2024-09-14T14:31:56.307623+0000 2024-09-13T14:39:24.670 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.204:0/2380512971 expires 2024-09-14T14:31:56.307623+0000 2024-09-13T14:39:24.670 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:6801/2875623667 expires 2024-09-14T14:22:08.207226+0000 2024-09-13T14:39:24.670 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/3047961062 expires 2024-09-14T14:32:35.504695+0000 2024-09-13T14:39:24.671 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:6800/2875623667 expires 2024-09-14T14:22:08.207226+0000 2024-09-13T14:39:24.671 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/667264267 expires 2024-09-14T14:32:35.504695+0000 2024-09-13T14:39:24.671 INFO:teuthology.orchestra.run.smithi096.stdout:blocklist 172.21.15.96:0/1790922234 expires 2024-09-14T14:21:45.592521+0000 2024-09-13T14:39:25.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:25 smithi096 ceph-mon[98901]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:25.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:25 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1923747392' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-13T14:39:25.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:25 smithi096 ceph-mon[102928]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:25.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:25 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1923747392' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-13T14:39:25.406 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 88 up_thru 104 down_at 83 last_clean_interval [8,82) [v2:172.21.15.96:6802/2849588101,v1:172.21.15.96:6803/2849588101] [v2:172.21.15.96:6804/2849588101,v1:172.21.15.96:6805/2849588101] exists,up eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed', 'osd.1 up in weight 1 up_from 94 up_thru 104 down_at 91 last_clean_interval [13,90) [v2:172.21.15.96:6810/289441923,v1:172.21.15.96:6811/289441923] [v2:172.21.15.96:6812/289441923,v1:172.21.15.96:6813/289441923] exists,up 1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7', 'osd.2 up in weight 1 up_from 99 up_thru 104 down_at 96 last_clean_interval [18,95) [v2:172.21.15.96:6818/4202655894,v1:172.21.15.96:6819/4202655894] [v2:172.21.15.96:6820/4202655894,v1:172.21.15.96:6821/4202655894] exists,up 6a92a5b3-bdc7-499a-8b0f-2b0936fd9598', 'osd.3 up in weight 1 up_from 104 up_thru 104 down_at 101 last_clean_interval [26,100) [v2:172.21.15.96:6826/3452335742,v1:172.21.15.96:6827/3452335742] [v2:172.21.15.96:6828/3452335742,v1:172.21.15.96:6829/3452335742] exists,up 1230646c-934e-4eb4-b4b9-16be52ffba66', 'osd.4 up in weight 1 up_from 31 up_thru 104 down_at 0 last_clean_interval [0,0) [v2:172.21.15.204:6800/1188704632,v1:172.21.15.204:6801/1188704632] [v2:172.21.15.204:6802/1188704632,v1:172.21.15.204:6803/1188704632] exists,up 14115269-3d41-4ebf-8613-71ed59ec0f29', 'osd.5 up in weight 1 up_from 37 up_thru 104 down_at 0 last_clean_interval [0,0) [v2:172.21.15.204:6808/580729752,v1:172.21.15.204:6809/580729752] [v2:172.21.15.204:6810/580729752,v1:172.21.15.204:6811/580729752] exists,up 15148a0c-df44-419e-b0f0-2b1600c207f4', 'osd.6 up in weight 1 up_from 43 up_thru 101 down_at 0 last_clean_interval [0,0) [v2:172.21.15.204:6816/118365497,v1:172.21.15.204:6817/118365497] [v2:172.21.15.204:6818/118365497,v1:172.21.15.204:6819/118365497] exists,up 8404e088-3523-4c21-b7c2-0a31b22e0279', 'osd.7 up in weight 1 up_from 49 up_thru 101 down_at 0 last_clean_interval [0,0) [v2:172.21.15.204:6824/3529639424,v1:172.21.15.204:6825/3529639424] [v2:172.21.15.204:6826/3529639424,v1:172.21.15.204:6827/3529639424] exists,up 93b23a35-0059-4e03-a2ba-28fee6431892'] 2024-09-13T14:39:25.406 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon mon.a config show 2024-09-13T14:39:25.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:25 smithi204 ceph-mon[87071]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:25.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:25 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1923747392' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-13T14:39:25.649 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:27.060 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:39:27.106 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "mon.a", 2024-09-13T14:39:27.106 INFO:teuthology.orchestra.run.smithi096.stdout: "cluster": "ceph", 2024-09-13T14:39:27.106 INFO:teuthology.orchestra.run.smithi096.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2024-09-13T14:39:27.106 INFO:teuthology.orchestra.run.smithi096.stdout: "admin_socket_mode": "", 2024-09-13T14:39:27.107 INFO:teuthology.orchestra.run.smithi096.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2024-09-13T14:39:27.107 INFO:teuthology.orchestra.run.smithi096.stdout: "auth_client_required": "cephx, none", 2024-09-13T14:39:27.107 INFO:teuthology.orchestra.run.smithi096.stdout: "auth_cluster_required": "cephx", 2024-09-13T14:39:27.107 INFO:teuthology.orchestra.run.smithi096.stdout: "auth_debug": "true", 2024-09-13T14:39:27.107 INFO:teuthology.orchestra.run.smithi096.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2024-09-13T14:39:27.107 INFO:teuthology.orchestra.run.smithi096.stdout: "auth_mon_ticket_ttl": "660.000000", 2024-09-13T14:39:27.107 INFO:teuthology.orchestra.run.smithi096.stdout: "auth_service_required": "cephx", 2024-09-13T14:39:27.107 INFO:teuthology.orchestra.run.smithi096.stdout: "auth_service_ticket_ttl": "240.000000", 2024-09-13T14:39:27.107 INFO:teuthology.orchestra.run.smithi096.stdout: "auth_supported": "", 2024-09-13T14:39:27.107 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_aio": "true", 2024-09-13T14:39:27.107 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_aio_max_queue_depth": "1024", 2024-09-13T14:39:27.107 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_aio_poll_ms": "250", 2024-09-13T14:39:27.107 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_aio_reap_max": "16", 2024-09-13T14:39:27.107 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_async_discard_threads": "0", 2024-09-13T14:39:27.108 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_block_size": "4096", 2024-09-13T14:39:27.108 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_debug_aio": "false", 2024-09-13T14:39:27.108 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_debug_aio_log_age": "5.000000", 2024-09-13T14:39:27.108 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2024-09-13T14:39:27.108 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_debug_inflight_ios": "false", 2024-09-13T14:39:27.108 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_enable_discard": "false", 2024-09-13T14:39:27.108 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_flock_retry": "3", 2024-09-13T14:39:27.108 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_flock_retry_interval": "0.100000", 2024-09-13T14:39:27.108 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_inject_crash": "0", 2024-09-13T14:39:27.108 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_inject_crash_flush_delay": "2", 2024-09-13T14:39:27.108 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_ioring": "false", 2024-09-13T14:39:27.108 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_ioring_hipri": "false", 2024-09-13T14:39:27.108 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_ioring_sqthread_poll": "false", 2024-09-13T14:39:27.109 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_nvme_unbind_from_kernel": "false", 2024-09-13T14:39:27.109 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_read_buffer_alignment": "4096", 2024-09-13T14:39:27.109 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_read_preallocated_huge_buffers": "", 2024-09-13T14:39:27.109 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_stalled_read_warn_lifetime": "86400", 2024-09-13T14:39:27.109 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_stalled_read_warn_threshold": "1", 2024-09-13T14:39:27.109 INFO:teuthology.orchestra.run.smithi096.stdout: "bdev_type": "", 2024-09-13T14:39:27.109 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_alloc_size": "1048576", 2024-09-13T14:39:27.109 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_allocator": "hybrid", 2024-09-13T14:39:27.109 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_buffered_io": "true", 2024-09-13T14:39:27.109 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_check_for_zeros": "false", 2024-09-13T14:39:27.109 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_check_volume_selector_often": "false", 2024-09-13T14:39:27.109 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_check_volume_selector_on_umount": "false", 2024-09-13T14:39:27.109 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_compact_log_sync": "false", 2024-09-13T14:39:27.110 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2024-09-13T14:39:27.110 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2024-09-13T14:39:27.110 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_log_compact_min_size": "16777216", 2024-09-13T14:39:27.110 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_log_replay_check_allocations": "true", 2024-09-13T14:39:27.110 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_max_log_runway": "4194304", 2024-09-13T14:39:27.110 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_max_prefetch": "1048576", 2024-09-13T14:39:27.110 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_min_flush_size": "524288", 2024-09-13T14:39:27.110 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_min_log_runway": "1048576", 2024-09-13T14:39:27.110 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_replay_recovery": "false", 2024-09-13T14:39:27.110 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_replay_recovery_disable_compact": "false", 2024-09-13T14:39:27.110 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_shared_alloc_size": "65536", 2024-09-13T14:39:27.110 INFO:teuthology.orchestra.run.smithi096.stdout: "bluefs_sync_write": "false", 2024-09-13T14:39:27.110 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2024-09-13T14:39:27.110 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2024-09-13T14:39:27.111 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2024-09-13T14:39:27.111 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_allocation_from_file": "true", 2024-09-13T14:39:27.111 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_allocator": "hybrid", 2024-09-13T14:39:27.111 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_async_db_compaction": "true", 2024-09-13T14:39:27.111 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2024-09-13T14:39:27.111 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2024-09-13T14:39:27.111 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2024-09-13T14:39:27.111 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2024-09-13T14:39:27.111 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_bdev_label_multi": "true", 2024-09-13T14:39:27.112 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_bdev_label_multi_upgrade": "false", 2024-09-13T14:39:27.112 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_bdev_label_require_all": "true", 2024-09-13T14:39:27.112 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2024-09-13T14:39:27.112 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_bitmapallocator_span_size": "1024", 2024-09-13T14:39:27.112 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_blobid_prealloc": "10240", 2024-09-13T14:39:27.112 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_block_create": "true", 2024-09-13T14:39:27.112 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_block_db_create": "false", 2024-09-13T14:39:27.112 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_block_db_path": "", 2024-09-13T14:39:27.112 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_block_db_size": "0", 2024-09-13T14:39:27.112 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_block_path": "", 2024-09-13T14:39:27.112 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_block_preallocate_file": "false", 2024-09-13T14:39:27.112 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_block_size": "107374182400", 2024-09-13T14:39:27.112 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_block_wal_create": "false", 2024-09-13T14:39:27.113 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_block_wal_path": "", 2024-09-13T14:39:27.113 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_block_wal_size": "100663296", 2024-09-13T14:39:27.113 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_bluefs": "true", 2024-09-13T14:39:27.113 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2024-09-13T14:39:27.113 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_bluefs_env_mirror": "false", 2024-09-13T14:39:27.113 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_bluefs_max_free": "10737418240", 2024-09-13T14:39:27.113 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_btree2_alloc_weight_factor": "2.000000", 2024-09-13T14:39:27.113 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2024-09-13T14:39:27.113 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2024-09-13T14:39:27.113 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2024-09-13T14:39:27.113 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2024-09-13T14:39:27.113 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2024-09-13T14:39:27.113 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_autotune": "true", 2024-09-13T14:39:27.113 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_autotune_interval": "5.000000", 2024-09-13T14:39:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2024-09-13T14:39:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_kv_ratio": "0.450000", 2024-09-13T14:39:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_meta_ratio": "0.450000", 2024-09-13T14:39:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_size": "0", 2024-09-13T14:39:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_size_hdd": "1073741824", 2024-09-13T14:39:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_size_ssd": "3221225472", 2024-09-13T14:39:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_trim_interval": "0.050000", 2024-09-13T14:39:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2024-09-13T14:39:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cache_type": "2q", 2024-09-13T14:39:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2024-09-13T14:39:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_clone_cow": "true", 2024-09-13T14:39:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_compression_algorithm": "snappy", 2024-09-13T14:39:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_compression_max_blob_size": "0", 2024-09-13T14:39:27.115 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2024-09-13T14:39:27.115 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2024-09-13T14:39:27.115 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_compression_min_blob_size": "0", 2024-09-13T14:39:27.115 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2024-09-13T14:39:27.115 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2024-09-13T14:39:27.115 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_compression_mode": "none", 2024-09-13T14:39:27.115 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_compression_required_ratio": "0.875000", 2024-09-13T14:39:27.115 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_csum_type": "crc32c", 2024-09-13T14:39:27.115 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_enforce_min_alloc_size": "0", 2024-09-13T14:39:27.115 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_enforce_settings": "default", 2024-09-13T14:39:27.115 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_freelist": "false", 2024-09-13T14:39:27.115 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_fsck_abort": "false", 2024-09-13T14:39:27.115 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2024-09-13T14:39:27.115 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2024-09-13T14:39:27.116 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_inject_read_err": "false", 2024-09-13T14:39:27.116 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_legacy_omap": "false", 2024-09-13T14:39:27.116 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_no_reuse_blocks": "false", 2024-09-13T14:39:27.116 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_omit_block_device_write": "false", 2024-09-13T14:39:27.116 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_omit_kv_commit": "false", 2024-09-13T14:39:27.116 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2024-09-13T14:39:27.116 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_prefragment_max": "1048576", 2024-09-13T14:39:27.116 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_random_read_err": "0.000000", 2024-09-13T14:39:27.116 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2024-09-13T14:39:27.116 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_small_allocations": "0", 2024-09-13T14:39:27.116 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2024-09-13T14:39:27.116 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_default_buffered_read": "true", 2024-09-13T14:39:27.116 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_default_buffered_write": "false", 2024-09-13T14:39:27.116 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_deferred_batch_ops": "0", 2024-09-13T14:39:27.117 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2024-09-13T14:39:27.117 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2024-09-13T14:39:27.117 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_elastic_shared_blobs": "true", 2024-09-13T14:39:27.117 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2024-09-13T14:39:27.117 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_extent_map_shard_max_size": "1200", 2024-09-13T14:39:27.117 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_extent_map_shard_min_size": "150", 2024-09-13T14:39:27.117 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_extent_map_shard_target_size": "500", 2024-09-13T14:39:27.117 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2024-09-13T14:39:27.117 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_fail_eio": "false", 2024-09-13T14:39:27.117 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_freelist_blocks_per_key": "128", 2024-09-13T14:39:27.117 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2024-09-13T14:39:27.117 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2024-09-13T14:39:27.117 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2024-09-13T14:39:27.119 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_fsck_on_mkfs": "true", 2024-09-13T14:39:27.119 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2024-09-13T14:39:27.119 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_fsck_on_mount": "false", 2024-09-13T14:39:27.119 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_fsck_on_mount_deep": "false", 2024-09-13T14:39:27.119 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_fsck_on_umount": "false", 2024-09-13T14:39:27.119 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_fsck_on_umount_deep": "false", 2024-09-13T14:39:27.119 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2024-09-13T14:39:27.119 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_fsck_quick_fix_threads": "2", 2024-09-13T14:39:27.120 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2024-09-13T14:39:27.120 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2024-09-13T14:39:27.120 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_gc_enable_blob_threshold": "0", 2024-09-13T14:39:27.120 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_gc_enable_total_threshold": "0", 2024-09-13T14:39:27.120 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2024-09-13T14:39:27.120 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_ignore_data_csum": "false", 2024-09-13T14:39:27.120 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2024-09-13T14:39:27.120 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_kvbackend": "rocksdb", 2024-09-13T14:39:27.121 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_log_collection_list_age": "60.000000", 2024-09-13T14:39:27.121 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2024-09-13T14:39:27.121 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_log_op_age": "5.000000", 2024-09-13T14:39:27.121 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_max_alloc_size": "0", 2024-09-13T14:39:27.121 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_max_blob_size": "0", 2024-09-13T14:39:27.121 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_max_blob_size_hdd": "65536", 2024-09-13T14:39:27.121 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_max_blob_size_ssd": "65536", 2024-09-13T14:39:27.121 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_max_defer_interval": "3.000000", 2024-09-13T14:39:27.121 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_max_deferred_txc": "32", 2024-09-13T14:39:27.122 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_min_alloc_size": "0", 2024-09-13T14:39:27.122 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_min_alloc_size_hdd": "4096", 2024-09-13T14:39:27.122 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_min_alloc_size_ssd": "4096", 2024-09-13T14:39:27.122 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_nid_prealloc": "1024", 2024-09-13T14:39:27.122 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_prefer_deferred_size": "0", 2024-09-13T14:39:27.122 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2024-09-13T14:39:27.122 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2024-09-13T14:39:27.122 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_qfsck_on_mount": "true", 2024-09-13T14:39:27.122 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_retry_disk_reads": "3", 2024-09-13T14:39:27.123 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_rocksdb_cf": "true", 2024-09-13T14:39:27.123 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2024-09-13T14:39:27.123 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2024-09-13T14:39:27.123 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_rocksdb_options_annex": "", 2024-09-13T14:39:27.123 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_slow_ops_warn_lifetime": "86400", 2024-09-13T14:39:27.123 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_slow_ops_warn_threshold": "1", 2024-09-13T14:39:27.123 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_spdk_coremask": "0x1", 2024-09-13T14:39:27.123 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_spdk_io_sleep": "5", 2024-09-13T14:39:27.123 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_spdk_max_io_completion": "0", 2024-09-13T14:39:27.123 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_spdk_mem": "512", 2024-09-13T14:39:27.124 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_sync_submit_transaction": "false", 2024-09-13T14:39:27.124 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_throttle_bytes": "67108864", 2024-09-13T14:39:27.124 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_throttle_cost_per_io": "0", 2024-09-13T14:39:27.124 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2024-09-13T14:39:27.124 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2024-09-13T14:39:27.124 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2024-09-13T14:39:27.124 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_throttle_trace_rate": "0.000000", 2024-09-13T14:39:27.124 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_tracing": "false", 2024-09-13T14:39:27.124 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2024-09-13T14:39:27.125 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2024-09-13T14:39:27.125 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_volume_selection_reserved": "0", 2024-09-13T14:39:27.125 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2024-09-13T14:39:27.125 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2024-09-13T14:39:27.125 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_warn_on_legacy_statfs": "true", 2024-09-13T14:39:27.125 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2024-09-13T14:39:27.125 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2024-09-13T14:39:27.125 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2024-09-13T14:39:27.125 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_write_v2": "false", 2024-09-13T14:39:27.126 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_write_v2_random": "false", 2024-09-13T14:39:27.126 INFO:teuthology.orchestra.run.smithi096.stdout: "bluestore_zero_block_detection": "false", 2024-09-13T14:39:27.126 INFO:teuthology.orchestra.run.smithi096.stdout: "cephadm_path": "/usr/sbin/cephadm", 2024-09-13T14:39:27.126 INFO:teuthology.orchestra.run.smithi096.stdout: "cephfs_mirror_action_update_interval": "2", 2024-09-13T14:39:27.126 INFO:teuthology.orchestra.run.smithi096.stdout: "cephfs_mirror_directory_scan_interval": "10", 2024-09-13T14:39:27.126 INFO:teuthology.orchestra.run.smithi096.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2024-09-13T14:39:27.127 INFO:teuthology.orchestra.run.smithi096.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2024-09-13T14:39:27.127 INFO:teuthology.orchestra.run.smithi096.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2024-09-13T14:39:27.127 INFO:teuthology.orchestra.run.smithi096.stdout: "cephfs_mirror_mount_timeout": "10", 2024-09-13T14:39:27.127 INFO:teuthology.orchestra.run.smithi096.stdout: "cephfs_mirror_perf_stats_prio": "5", 2024-09-13T14:39:27.127 INFO:teuthology.orchestra.run.smithi096.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2024-09-13T14:39:27.128 INFO:teuthology.orchestra.run.smithi096.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2024-09-13T14:39:27.128 INFO:teuthology.orchestra.run.smithi096.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2024-09-13T14:39:27.128 INFO:teuthology.orchestra.run.smithi096.stdout: "cephsqlite_blocklist_dead_locker": "true", 2024-09-13T14:39:27.128 INFO:teuthology.orchestra.run.smithi096.stdout: "cephsqlite_lock_renewal_interval": "2000", 2024-09-13T14:39:27.128 INFO:teuthology.orchestra.run.smithi096.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2024-09-13T14:39:27.128 INFO:teuthology.orchestra.run.smithi096.stdout: "cephx_cluster_require_signatures": "false", 2024-09-13T14:39:27.129 INFO:teuthology.orchestra.run.smithi096.stdout: "cephx_cluster_require_version": "2", 2024-09-13T14:39:27.129 INFO:teuthology.orchestra.run.smithi096.stdout: "cephx_require_signatures": "false", 2024-09-13T14:39:27.129 INFO:teuthology.orchestra.run.smithi096.stdout: "cephx_require_version": "2", 2024-09-13T14:39:27.129 INFO:teuthology.orchestra.run.smithi096.stdout: "cephx_service_require_signatures": "false", 2024-09-13T14:39:27.129 INFO:teuthology.orchestra.run.smithi096.stdout: "cephx_service_require_version": "2", 2024-09-13T14:39:27.130 INFO:teuthology.orchestra.run.smithi096.stdout: "cephx_sign_messages": "true", 2024-09-13T14:39:27.130 INFO:teuthology.orchestra.run.smithi096.stdout: "chdir": "", 2024-09-13T14:39:27.130 INFO:teuthology.orchestra.run.smithi096.stdout: "client_acl_type": "", 2024-09-13T14:39:27.130 INFO:teuthology.orchestra.run.smithi096.stdout: "client_asio_thread_count": "2", 2024-09-13T14:39:27.130 INFO:teuthology.orchestra.run.smithi096.stdout: "client_cache_mid": "0.750000", 2024-09-13T14:39:27.131 INFO:teuthology.orchestra.run.smithi096.stdout: "client_cache_size": "16384", 2024-09-13T14:39:27.131 INFO:teuthology.orchestra.run.smithi096.stdout: "client_caps_release_delay": "5", 2024-09-13T14:39:27.131 INFO:teuthology.orchestra.run.smithi096.stdout: "client_check_pool_perm": "true", 2024-09-13T14:39:27.131 INFO:teuthology.orchestra.run.smithi096.stdout: "client_collect_and_send_global_metrics": "false", 2024-09-13T14:39:27.131 INFO:teuthology.orchestra.run.smithi096.stdout: "client_debug_force_sync_read": "false", 2024-09-13T14:39:27.131 INFO:teuthology.orchestra.run.smithi096.stdout: "client_debug_getattr_caps": "false", 2024-09-13T14:39:27.131 INFO:teuthology.orchestra.run.smithi096.stdout: "client_debug_inject_features": "", 2024-09-13T14:39:27.132 INFO:teuthology.orchestra.run.smithi096.stdout: "client_debug_inject_tick_delay": "0", 2024-09-13T14:39:27.132 INFO:teuthology.orchestra.run.smithi096.stdout: "client_die_on_failed_dentry_invalidate": "true", 2024-09-13T14:39:27.132 INFO:teuthology.orchestra.run.smithi096.stdout: "client_die_on_failed_remount": "false", 2024-09-13T14:39:27.132 INFO:teuthology.orchestra.run.smithi096.stdout: "client_dirsize_rbytes": "true", 2024-09-13T14:39:27.132 INFO:teuthology.orchestra.run.smithi096.stdout: "client_force_lazyio": "false", 2024-09-13T14:39:27.132 INFO:teuthology.orchestra.run.smithi096.stdout: "client_fs": "", 2024-09-13T14:39:27.133 INFO:teuthology.orchestra.run.smithi096.stdout: "client_inject_fixed_oldest_tid": "false", 2024-09-13T14:39:27.133 INFO:teuthology.orchestra.run.smithi096.stdout: "client_inject_release_failure": "false", 2024-09-13T14:39:27.133 INFO:teuthology.orchestra.run.smithi096.stdout: "client_max_inline_size": "4096", 2024-09-13T14:39:27.133 INFO:teuthology.orchestra.run.smithi096.stdout: "client_max_retries_on_remount_failure": "5", 2024-09-13T14:39:27.133 INFO:teuthology.orchestra.run.smithi096.stdout: "client_mds_namespace": "", 2024-09-13T14:39:27.134 INFO:teuthology.orchestra.run.smithi096.stdout: "client_metadata": "", 2024-09-13T14:39:27.134 INFO:teuthology.orchestra.run.smithi096.stdout: "client_mount_gid": "-1", 2024-09-13T14:39:27.134 INFO:teuthology.orchestra.run.smithi096.stdout: "client_mount_timeout": "300", 2024-09-13T14:39:27.134 INFO:teuthology.orchestra.run.smithi096.stdout: "client_mount_uid": "-1", 2024-09-13T14:39:27.134 INFO:teuthology.orchestra.run.smithi096.stdout: "client_mountpoint": "/", 2024-09-13T14:39:27.134 INFO:teuthology.orchestra.run.smithi096.stdout: "client_notify_timeout": "10", 2024-09-13T14:39:27.135 INFO:teuthology.orchestra.run.smithi096.stdout: "client_oc": "true", 2024-09-13T14:39:27.135 INFO:teuthology.orchestra.run.smithi096.stdout: "client_oc_max_dirty": "104857600", 2024-09-13T14:39:27.135 INFO:teuthology.orchestra.run.smithi096.stdout: "client_oc_max_dirty_age": "5.000000", 2024-09-13T14:39:27.135 INFO:teuthology.orchestra.run.smithi096.stdout: "client_oc_max_objects": "1000", 2024-09-13T14:39:27.136 INFO:teuthology.orchestra.run.smithi096.stdout: "client_oc_size": "209715200", 2024-09-13T14:39:27.136 INFO:teuthology.orchestra.run.smithi096.stdout: "client_oc_target_dirty": "8388608", 2024-09-13T14:39:27.136 INFO:teuthology.orchestra.run.smithi096.stdout: "client_permissions": "true", 2024-09-13T14:39:27.136 INFO:teuthology.orchestra.run.smithi096.stdout: "client_quota": "true", 2024-09-13T14:39:27.136 INFO:teuthology.orchestra.run.smithi096.stdout: "client_quota_df": "true", 2024-09-13T14:39:27.136 INFO:teuthology.orchestra.run.smithi096.stdout: "client_readahead_max_bytes": "0", 2024-09-13T14:39:27.136 INFO:teuthology.orchestra.run.smithi096.stdout: "client_readahead_max_periods": "4", 2024-09-13T14:39:27.137 INFO:teuthology.orchestra.run.smithi096.stdout: "client_readahead_min": "131072", 2024-09-13T14:39:27.137 INFO:teuthology.orchestra.run.smithi096.stdout: "client_reconnect_stale": "false", 2024-09-13T14:39:27.137 INFO:teuthology.orchestra.run.smithi096.stdout: "client_shutdown_timeout": "30", 2024-09-13T14:39:27.137 INFO:teuthology.orchestra.run.smithi096.stdout: "client_snapdir": ".snap", 2024-09-13T14:39:27.137 INFO:teuthology.orchestra.run.smithi096.stdout: "client_tick_interval": "1", 2024-09-13T14:39:27.137 INFO:teuthology.orchestra.run.smithi096.stdout: "client_trace": "", 2024-09-13T14:39:27.138 INFO:teuthology.orchestra.run.smithi096.stdout: "client_try_dentry_invalidate": "false", 2024-09-13T14:39:27.138 INFO:teuthology.orchestra.run.smithi096.stdout: "client_use_faked_inos": "false", 2024-09-13T14:39:27.138 INFO:teuthology.orchestra.run.smithi096.stdout: "client_use_random_mds": "false", 2024-09-13T14:39:27.138 INFO:teuthology.orchestra.run.smithi096.stdout: "clog_to_graylog": "false", 2024-09-13T14:39:27.138 INFO:teuthology.orchestra.run.smithi096.stdout: "clog_to_graylog_host": "127.0.0.1", 2024-09-13T14:39:27.138 INFO:teuthology.orchestra.run.smithi096.stdout: "clog_to_graylog_port": "12201", 2024-09-13T14:39:27.138 INFO:teuthology.orchestra.run.smithi096.stdout: "clog_to_monitors": "default=true", 2024-09-13T14:39:27.139 INFO:teuthology.orchestra.run.smithi096.stdout: "clog_to_syslog": "false", 2024-09-13T14:39:27.139 INFO:teuthology.orchestra.run.smithi096.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2024-09-13T14:39:27.139 INFO:teuthology.orchestra.run.smithi096.stdout: "clog_to_syslog_level": "info", 2024-09-13T14:39:27.139 INFO:teuthology.orchestra.run.smithi096.stdout: "cluster_addr": "-", 2024-09-13T14:39:27.139 INFO:teuthology.orchestra.run.smithi096.stdout: "cluster_network": "", 2024-09-13T14:39:27.139 INFO:teuthology.orchestra.run.smithi096.stdout: "cluster_network_interface": "", 2024-09-13T14:39:27.140 INFO:teuthology.orchestra.run.smithi096.stdout: "compressor_zlib_isal": "false", 2024-09-13T14:39:27.140 INFO:teuthology.orchestra.run.smithi096.stdout: "compressor_zlib_level": "5", 2024-09-13T14:39:27.140 INFO:teuthology.orchestra.run.smithi096.stdout: "compressor_zlib_winsize": "-15", 2024-09-13T14:39:27.140 INFO:teuthology.orchestra.run.smithi096.stdout: "compressor_zstd_level": "1", 2024-09-13T14:39:27.140 INFO:teuthology.orchestra.run.smithi096.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:39:27.140 INFO:teuthology.orchestra.run.smithi096.stdout: "crash_dir": "/var/lib/ceph/crash", 2024-09-13T14:39:27.141 INFO:teuthology.orchestra.run.smithi096.stdout: "crimson_alien_op_num_threads": "6", 2024-09-13T14:39:27.141 INFO:teuthology.orchestra.run.smithi096.stdout: "crimson_alien_thread_cpu_cores": "", 2024-09-13T14:39:27.141 INFO:teuthology.orchestra.run.smithi096.stdout: "crimson_osd_obc_lru_size": "512", 2024-09-13T14:39:27.141 INFO:teuthology.orchestra.run.smithi096.stdout: "crimson_osd_scheduler_concurrency": "0", 2024-09-13T14:39:27.141 INFO:teuthology.orchestra.run.smithi096.stdout: "crimson_osd_stat_interval": "0", 2024-09-13T14:39:27.141 INFO:teuthology.orchestra.run.smithi096.stdout: "crimson_seastar_cpu_cores": "", 2024-09-13T14:39:27.142 INFO:teuthology.orchestra.run.smithi096.stdout: "crimson_seastar_num_threads": "0", 2024-09-13T14:39:27.142 INFO:teuthology.orchestra.run.smithi096.stdout: "crush_location": "", 2024-09-13T14:39:27.142 INFO:teuthology.orchestra.run.smithi096.stdout: "crush_location_hook": "", 2024-09-13T14:39:27.142 INFO:teuthology.orchestra.run.smithi096.stdout: "crush_location_hook_timeout": "10", 2024-09-13T14:39:27.142 INFO:teuthology.orchestra.run.smithi096.stdout: "daemonize": "false", 2024-09-13T14:39:27.142 INFO:teuthology.orchestra.run.smithi096.stdout: "daos_pool": "tank", 2024-09-13T14:39:27.142 INFO:teuthology.orchestra.run.smithi096.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2024-09-13T14:39:27.143 INFO:teuthology.orchestra.run.smithi096.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2024-09-13T14:39:27.143 INFO:teuthology.orchestra.run.smithi096.stdout: "dbstore_db_name_prefix": "dbstore", 2024-09-13T14:39:27.143 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_alienstore": "0/5", 2024-09-13T14:39:27.143 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_allow_any_pool_priority": "false", 2024-09-13T14:39:27.143 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_asok": "1/5", 2024-09-13T14:39:27.143 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_asok_assert_abort": "false", 2024-09-13T14:39:27.144 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_asserts_on_shutdown": "true", 2024-09-13T14:39:27.144 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_auth": "1/5", 2024-09-13T14:39:27.144 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_bdev": "1/3", 2024-09-13T14:39:27.144 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_bluefs": "1/5", 2024-09-13T14:39:27.146 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_bluestore": "1/5", 2024-09-13T14:39:27.146 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_buffer": "0/1", 2024-09-13T14:39:27.146 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_ceph_dedup": "0/5", 2024-09-13T14:39:27.146 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_ceph_exporter": "1/5", 2024-09-13T14:39:27.146 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_cephfs_mirror": "0/5", 2024-09-13T14:39:27.146 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_cephsqlite": "0/5", 2024-09-13T14:39:27.146 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_client": "0/5", 2024-09-13T14:39:27.146 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_compressor": "1/5", 2024-09-13T14:39:27.146 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_context": "0/1", 2024-09-13T14:39:27.146 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_crimson_interrupt": "0/5", 2024-09-13T14:39:27.147 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_crush": "1/1", 2024-09-13T14:39:27.147 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_crypto": "1/5", 2024-09-13T14:39:27.147 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_cyanstore": "0/5", 2024-09-13T14:39:27.147 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_deliberately_leak_memory": "false", 2024-09-13T14:39:27.147 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_disable_randomized_ping": "false", 2024-09-13T14:39:27.147 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_dpdk": "1/5", 2024-09-13T14:39:27.147 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_eventtrace": "1/5", 2024-09-13T14:39:27.147 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_filer": "0/1", 2024-09-13T14:39:27.147 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_filestore": "1/3", 2024-09-13T14:39:27.147 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_finisher": "1/1", 2024-09-13T14:39:27.147 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_fuse": "1/5", 2024-09-13T14:39:27.147 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_heartbeat_testing_span": "0", 2024-09-13T14:39:27.147 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_heartbeatmap": "1/5", 2024-09-13T14:39:27.148 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_immutable_obj_cache": "0/5", 2024-09-13T14:39:27.148 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_javaclient": "1/5", 2024-09-13T14:39:27.148 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_journal": "1/3", 2024-09-13T14:39:27.148 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_journaler": "0/5", 2024-09-13T14:39:27.148 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_kstore": "1/5", 2024-09-13T14:39:27.148 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_lockdep": "0/1", 2024-09-13T14:39:27.148 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_mclock": "1/5", 2024-09-13T14:39:27.148 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_mds": "1/5", 2024-09-13T14:39:27.148 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_mds_balancer": "1/5", 2024-09-13T14:39:27.148 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_mds_locker": "1/5", 2024-09-13T14:39:27.148 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_mds_log": "1/5", 2024-09-13T14:39:27.148 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_mds_log_expire": "1/5", 2024-09-13T14:39:27.148 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_mds_migrator": "1/5", 2024-09-13T14:39:27.149 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_mds_quiesce": "3/5", 2024-09-13T14:39:27.149 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_memstore": "1/5", 2024-09-13T14:39:27.149 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_mgr": "2/5", 2024-09-13T14:39:27.149 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_mgrc": "1/5", 2024-09-13T14:39:27.149 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_mon": "20/20", 2024-09-13T14:39:27.149 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_monc": "0/10", 2024-09-13T14:39:27.149 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_ms": "1/1", 2024-09-13T14:39:27.149 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_none": "0/5", 2024-09-13T14:39:27.149 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_objclass": "0/5", 2024-09-13T14:39:27.149 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_objectcacher": "0/5", 2024-09-13T14:39:27.149 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_objecter": "0/1", 2024-09-13T14:39:27.149 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_optracker": "0/5", 2024-09-13T14:39:27.149 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_osd": "1/5", 2024-09-13T14:39:27.149 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_paxos": "20/20", 2024-09-13T14:39:27.150 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_perfcounter": "1/5", 2024-09-13T14:39:27.150 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_prioritycache": "1/5", 2024-09-13T14:39:27.150 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_rados": "0/5", 2024-09-13T14:39:27.150 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_rbd": "0/5", 2024-09-13T14:39:27.150 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_rbd_mirror": "0/5", 2024-09-13T14:39:27.150 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_rbd_pwl": "0/5", 2024-09-13T14:39:27.150 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_rbd_replay": "0/5", 2024-09-13T14:39:27.150 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_refs": "0/0", 2024-09-13T14:39:27.150 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_reserver": "1/1", 2024-09-13T14:39:27.150 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_rgw": "1/5", 2024-09-13T14:39:27.150 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_rgw_access": "1/5", 2024-09-13T14:39:27.150 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_rgw_datacache": "1/5", 2024-09-13T14:39:27.150 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_rgw_dbstore": "1/5", 2024-09-13T14:39:27.151 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_rgw_flight": "1/5", 2024-09-13T14:39:27.151 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_rgw_lifecycle": "1/5", 2024-09-13T14:39:27.151 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_rgw_notification": "1/5", 2024-09-13T14:39:27.151 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_rgw_sync": "1/5", 2024-09-13T14:39:27.151 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_rocksdb": "4/5", 2024-09-13T14:39:27.151 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_seastore": "0/5", 2024-09-13T14:39:27.151 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_seastore_backref": "0/5", 2024-09-13T14:39:27.151 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_seastore_cache": "0/5", 2024-09-13T14:39:27.151 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_seastore_cleaner": "0/5", 2024-09-13T14:39:27.151 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_seastore_device": "0/5", 2024-09-13T14:39:27.151 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_seastore_epm": "0/5", 2024-09-13T14:39:27.151 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_seastore_fixedkv_tree": "0/5", 2024-09-13T14:39:27.152 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_seastore_journal": "0/5", 2024-09-13T14:39:27.152 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_seastore_lba": "0/5", 2024-09-13T14:39:27.152 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_seastore_odata": "0/5", 2024-09-13T14:39:27.152 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_seastore_omap": "0/5", 2024-09-13T14:39:27.152 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_seastore_onode": "0/5", 2024-09-13T14:39:27.152 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_seastore_t": "0/5", 2024-09-13T14:39:27.152 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_seastore_tm": "0/5", 2024-09-13T14:39:27.152 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_striper": "0/1", 2024-09-13T14:39:27.152 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_test": "0/5", 2024-09-13T14:39:27.153 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_throttle": "1/1", 2024-09-13T14:39:27.153 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_timer": "0/1", 2024-09-13T14:39:27.153 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_tp": "0/5", 2024-09-13T14:39:27.153 INFO:teuthology.orchestra.run.smithi096.stdout: "debug_trace": "1/5", 2024-09-13T14:39:27.153 INFO:teuthology.orchestra.run.smithi096.stdout: "defer_client_eviction_on_laggy_osds": "false", 2024-09-13T14:39:27.153 INFO:teuthology.orchestra.run.smithi096.stdout: "device_failure_prediction_mode": "none", 2024-09-13T14:39:27.153 INFO:teuthology.orchestra.run.smithi096.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2024-09-13T14:39:27.153 INFO:teuthology.orchestra.run.smithi096.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2024-09-13T14:39:27.153 INFO:teuthology.orchestra.run.smithi096.stdout: "err_to_graylog": "false", 2024-09-13T14:39:27.153 INFO:teuthology.orchestra.run.smithi096.stdout: "err_to_journald": "false", 2024-09-13T14:39:27.153 INFO:teuthology.orchestra.run.smithi096.stdout: "err_to_stderr": "true", 2024-09-13T14:39:27.153 INFO:teuthology.orchestra.run.smithi096.stdout: "err_to_syslog": "false", 2024-09-13T14:39:27.153 INFO:teuthology.orchestra.run.smithi096.stdout: "event_tracing": "false", 2024-09-13T14:39:27.154 INFO:teuthology.orchestra.run.smithi096.stdout: "exporter_addr": "0.0.0.0", 2024-09-13T14:39:27.154 INFO:teuthology.orchestra.run.smithi096.stdout: "exporter_cert_file": "", 2024-09-13T14:39:27.154 INFO:teuthology.orchestra.run.smithi096.stdout: "exporter_http_port": "9926", 2024-09-13T14:39:27.154 INFO:teuthology.orchestra.run.smithi096.stdout: "exporter_key_file": "", 2024-09-13T14:39:27.154 INFO:teuthology.orchestra.run.smithi096.stdout: "exporter_prio_limit": "5", 2024-09-13T14:39:27.154 INFO:teuthology.orchestra.run.smithi096.stdout: "exporter_sock_dir": "/var/run/ceph/", 2024-09-13T14:39:27.154 INFO:teuthology.orchestra.run.smithi096.stdout: "exporter_sort_metrics": "true", 2024-09-13T14:39:27.154 INFO:teuthology.orchestra.run.smithi096.stdout: "exporter_stats_period": "5", 2024-09-13T14:39:27.154 INFO:teuthology.orchestra.run.smithi096.stdout: "fake_statfs_for_testing": "0", 2024-09-13T14:39:27.154 INFO:teuthology.orchestra.run.smithi096.stdout: "fatal_signal_handlers": "true", 2024-09-13T14:39:27.154 INFO:teuthology.orchestra.run.smithi096.stdout: "filer_max_purge_ops": "10", 2024-09-13T14:39:27.154 INFO:teuthology.orchestra.run.smithi096.stdout: "filer_max_truncate_ops": "128", 2024-09-13T14:39:27.154 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_apply_finisher_threads": "1", 2024-09-13T14:39:27.155 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_blackhole": "false", 2024-09-13T14:39:27.155 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_btrfs_clone_range": "true", 2024-09-13T14:39:27.155 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_btrfs_snap": "true", 2024-09-13T14:39:27.155 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_caller_concurrency": "10", 2024-09-13T14:39:27.155 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_collect_device_partition_information": "true", 2024-09-13T14:39:27.155 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_commit_timeout": "600.000000", 2024-09-13T14:39:27.155 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_debug_inject_read_err": "false", 2024-09-13T14:39:27.155 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_debug_omap_check": "false", 2024-09-13T14:39:27.155 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_debug_verify_split": "false", 2024-09-13T14:39:27.155 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_dump_file": "", 2024-09-13T14:39:27.155 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2024-09-13T14:39:27.155 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_expected_throughput_ops": "200.000000", 2024-09-13T14:39:27.155 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_fadvise": "true", 2024-09-13T14:39:27.156 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_fail_eio": "true", 2024-09-13T14:39:27.156 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_fd_cache_shards": "16", 2024-09-13T14:39:27.156 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_fd_cache_size": "128", 2024-09-13T14:39:27.156 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_fiemap": "false", 2024-09-13T14:39:27.156 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_fiemap_threshold": "4096", 2024-09-13T14:39:27.156 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_fsync_flushes_journal_data": "false", 2024-09-13T14:39:27.156 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_index_retry_probability": "0.000000", 2024-09-13T14:39:27.156 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_inject_stall": "0", 2024-09-13T14:39:27.156 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_journal_parallel": "false", 2024-09-13T14:39:27.156 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_journal_trailing": "false", 2024-09-13T14:39:27.156 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_journal_writeahead": "false", 2024-09-13T14:39:27.156 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_kill_at": "0", 2024-09-13T14:39:27.156 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_max_alloc_hint_size": "1048576", 2024-09-13T14:39:27.157 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_max_inline_xattr_size": "0", 2024-09-13T14:39:27.157 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2024-09-13T14:39:27.157 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_max_inline_xattr_size_other": "512", 2024-09-13T14:39:27.157 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2024-09-13T14:39:27.157 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_max_inline_xattrs": "0", 2024-09-13T14:39:27.157 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2024-09-13T14:39:27.157 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_max_inline_xattrs_other": "2", 2024-09-13T14:39:27.157 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_max_inline_xattrs_xfs": "10", 2024-09-13T14:39:27.157 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_max_sync_interval": "5.000000", 2024-09-13T14:39:27.157 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_max_xattr_value_size": "0", 2024-09-13T14:39:27.157 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2024-09-13T14:39:27.157 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_max_xattr_value_size_other": "1024", 2024-09-13T14:39:27.157 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2024-09-13T14:39:27.158 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_merge_threshold": "-10", 2024-09-13T14:39:27.158 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_min_sync_interval": "0.010000", 2024-09-13T14:39:27.158 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_odsync_write": "false", 2024-09-13T14:39:27.158 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_omap_backend": "rocksdb", 2024-09-13T14:39:27.158 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_omap_backend_path": "", 2024-09-13T14:39:27.158 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_omap_header_cache_size": "1024", 2024-09-13T14:39:27.158 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_ondisk_finisher_threads": "1", 2024-09-13T14:39:27.158 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_op_thread_suicide_timeout": "180", 2024-09-13T14:39:27.158 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_op_thread_timeout": "60", 2024-09-13T14:39:27.158 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_op_threads": "2", 2024-09-13T14:39:27.158 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_punch_hole": "false", 2024-09-13T14:39:27.158 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2024-09-13T14:39:27.158 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2024-09-13T14:39:27.159 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2024-09-13T14:39:27.159 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_queue_high_threshhold": "0.900000", 2024-09-13T14:39:27.159 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_queue_low_threshhold": "0.300000", 2024-09-13T14:39:27.159 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_queue_max_bytes": "104857600", 2024-09-13T14:39:27.159 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2024-09-13T14:39:27.159 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2024-09-13T14:39:27.159 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2024-09-13T14:39:27.159 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_queue_max_ops": "50", 2024-09-13T14:39:27.159 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2024-09-13T14:39:27.159 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_seek_data_hole": "false", 2024-09-13T14:39:27.159 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_sloppy_crc": "false", 2024-09-13T14:39:27.159 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_sloppy_crc_block_size": "65536", 2024-09-13T14:39:27.159 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_splice": "false", 2024-09-13T14:39:27.159 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_split_multiple": "2", 2024-09-13T14:39:27.160 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_split_rand_factor": "20", 2024-09-13T14:39:27.160 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_update_to": "1000", 2024-09-13T14:39:27.160 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2024-09-13T14:39:27.160 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2024-09-13T14:39:27.160 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2024-09-13T14:39:27.160 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2024-09-13T14:39:27.160 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2024-09-13T14:39:27.160 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2024-09-13T14:39:27.160 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_wbthrottle_enable": "true", 2024-09-13T14:39:27.160 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2024-09-13T14:39:27.160 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2024-09-13T14:39:27.161 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2024-09-13T14:39:27.161 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2024-09-13T14:39:27.161 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2024-09-13T14:39:27.161 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2024-09-13T14:39:27.161 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_xfs_extsize": "false", 2024-09-13T14:39:27.161 INFO:teuthology.orchestra.run.smithi096.stdout: "filestore_zfs_snap": "false", 2024-09-13T14:39:27.161 INFO:teuthology.orchestra.run.smithi096.stdout: "fio_dir": "/tmp/fio", 2024-09-13T14:39:27.161 INFO:teuthology.orchestra.run.smithi096.stdout: "fsid": "51e50ed6-71db-11ef-bceb-c7b262605968", 2024-09-13T14:39:27.161 INFO:teuthology.orchestra.run.smithi096.stdout: "fuse_allow_other": "true", 2024-09-13T14:39:27.161 INFO:teuthology.orchestra.run.smithi096.stdout: "fuse_atomic_o_trunc": "true", 2024-09-13T14:39:27.162 INFO:teuthology.orchestra.run.smithi096.stdout: "fuse_big_writes": "true", 2024-09-13T14:39:27.162 INFO:teuthology.orchestra.run.smithi096.stdout: "fuse_debug": "false", 2024-09-13T14:39:27.162 INFO:teuthology.orchestra.run.smithi096.stdout: "fuse_default_permissions": "false", 2024-09-13T14:39:27.162 INFO:teuthology.orchestra.run.smithi096.stdout: "fuse_disable_pagecache": "false", 2024-09-13T14:39:27.162 INFO:teuthology.orchestra.run.smithi096.stdout: "fuse_max_write": "0", 2024-09-13T14:39:27.162 INFO:teuthology.orchestra.run.smithi096.stdout: "fuse_multithreaded": "true", 2024-09-13T14:39:27.162 INFO:teuthology.orchestra.run.smithi096.stdout: "fuse_require_active_mds": "true", 2024-09-13T14:39:27.162 INFO:teuthology.orchestra.run.smithi096.stdout: "fuse_set_user_groups": "true", 2024-09-13T14:39:27.162 INFO:teuthology.orchestra.run.smithi096.stdout: "fuse_splice_move": "true", 2024-09-13T14:39:27.162 INFO:teuthology.orchestra.run.smithi096.stdout: "fuse_splice_read": "true", 2024-09-13T14:39:27.162 INFO:teuthology.orchestra.run.smithi096.stdout: "fuse_splice_write": "true", 2024-09-13T14:39:27.162 INFO:teuthology.orchestra.run.smithi096.stdout: "fuse_syncfs_on_mksnap": "true", 2024-09-13T14:39:27.162 INFO:teuthology.orchestra.run.smithi096.stdout: "fuse_use_invalidate_cb": "true", 2024-09-13T14:39:27.163 INFO:teuthology.orchestra.run.smithi096.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2024-09-13T14:39:27.163 INFO:teuthology.orchestra.run.smithi096.stdout: "gss_target_name": "ceph", 2024-09-13T14:39:27.163 INFO:teuthology.orchestra.run.smithi096.stdout: "heartbeat_file": "", 2024-09-13T14:39:27.163 INFO:teuthology.orchestra.run.smithi096.stdout: "heartbeat_inject_failure": "0", 2024-09-13T14:39:27.163 INFO:teuthology.orchestra.run.smithi096.stdout: "heartbeat_interval": "5", 2024-09-13T14:39:27.163 INFO:teuthology.orchestra.run.smithi096.stdout: "host": "smithi096", 2024-09-13T14:39:27.163 INFO:teuthology.orchestra.run.smithi096.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2024-09-13T14:39:27.163 INFO:teuthology.orchestra.run.smithi096.stdout: "immutable_object_cache_max_inflight_ops": "128", 2024-09-13T14:39:27.163 INFO:teuthology.orchestra.run.smithi096.stdout: "immutable_object_cache_max_size": "1073741824", 2024-09-13T14:39:27.163 INFO:teuthology.orchestra.run.smithi096.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2024-09-13T14:39:27.163 INFO:teuthology.orchestra.run.smithi096.stdout: "immutable_object_cache_qos_bps_burst": "0", 2024-09-13T14:39:27.163 INFO:teuthology.orchestra.run.smithi096.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2024-09-13T14:39:27.163 INFO:teuthology.orchestra.run.smithi096.stdout: "immutable_object_cache_qos_bps_limit": "0", 2024-09-13T14:39:27.163 INFO:teuthology.orchestra.run.smithi096.stdout: "immutable_object_cache_qos_iops_burst": "0", 2024-09-13T14:39:27.164 INFO:teuthology.orchestra.run.smithi096.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2024-09-13T14:39:27.164 INFO:teuthology.orchestra.run.smithi096.stdout: "immutable_object_cache_qos_iops_limit": "0", 2024-09-13T14:39:27.164 INFO:teuthology.orchestra.run.smithi096.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2024-09-13T14:39:27.164 INFO:teuthology.orchestra.run.smithi096.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2024-09-13T14:39:27.164 INFO:teuthology.orchestra.run.smithi096.stdout: "immutable_object_cache_watermark": "0.900000", 2024-09-13T14:39:27.164 INFO:teuthology.orchestra.run.smithi096.stdout: "inject_early_sigterm": "false", 2024-09-13T14:39:27.164 INFO:teuthology.orchestra.run.smithi096.stdout: "jaeger_agent_port": "6799", 2024-09-13T14:39:27.164 INFO:teuthology.orchestra.run.smithi096.stdout: "jaeger_tracing_enable": "false", 2024-09-13T14:39:27.164 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_aio": "true", 2024-09-13T14:39:27.164 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_align_min_size": "65536", 2024-09-13T14:39:27.164 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_block_align": "true", 2024-09-13T14:39:27.164 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_block_size": "4096", 2024-09-13T14:39:27.164 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_dio": "true", 2024-09-13T14:39:27.165 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_discard": "false", 2024-09-13T14:39:27.165 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_force_aio": "false", 2024-09-13T14:39:27.165 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_ignore_corruption": "false", 2024-09-13T14:39:27.165 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_max_write_bytes": "10485760", 2024-09-13T14:39:27.165 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_max_write_entries": "100", 2024-09-13T14:39:27.165 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_replay_from": "0", 2024-09-13T14:39:27.165 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_throttle_high_multiple": "0.000000", 2024-09-13T14:39:27.165 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_throttle_high_threshhold": "0.900000", 2024-09-13T14:39:27.165 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_throttle_low_threshhold": "0.600000", 2024-09-13T14:39:27.165 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_throttle_max_multiple": "0.000000", 2024-09-13T14:39:27.165 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_write_header_frequency": "0", 2024-09-13T14:39:27.165 INFO:teuthology.orchestra.run.smithi096.stdout: "journal_zero_on_create": "false", 2024-09-13T14:39:27.165 INFO:teuthology.orchestra.run.smithi096.stdout: "journaler_prefetch_periods": "10", 2024-09-13T14:39:27.165 INFO:teuthology.orchestra.run.smithi096.stdout: "journaler_prezero_periods": "5", 2024-09-13T14:39:27.166 INFO:teuthology.orchestra.run.smithi096.stdout: "journaler_write_head_interval": "15", 2024-09-13T14:39:27.166 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "", 2024-09-13T14:39:27.166 INFO:teuthology.orchestra.run.smithi096.stdout: "keyfile": "", 2024-09-13T14:39:27.166 INFO:teuthology.orchestra.run.smithi096.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2024-09-13T14:39:27.166 INFO:teuthology.orchestra.run.smithi096.stdout: "kstore_backend": "rocksdb", 2024-09-13T14:39:27.166 INFO:teuthology.orchestra.run.smithi096.stdout: "kstore_default_stripe_size": "65536", 2024-09-13T14:39:27.166 INFO:teuthology.orchestra.run.smithi096.stdout: "kstore_fsck_on_mount": "false", 2024-09-13T14:39:27.166 INFO:teuthology.orchestra.run.smithi096.stdout: "kstore_fsck_on_mount_deep": "true", 2024-09-13T14:39:27.166 INFO:teuthology.orchestra.run.smithi096.stdout: "kstore_max_bytes": "67108864", 2024-09-13T14:39:27.166 INFO:teuthology.orchestra.run.smithi096.stdout: "kstore_max_ops": "512", 2024-09-13T14:39:27.166 INFO:teuthology.orchestra.run.smithi096.stdout: "kstore_nid_prealloc": "1024", 2024-09-13T14:39:27.166 INFO:teuthology.orchestra.run.smithi096.stdout: "kstore_onode_map_size": "1024", 2024-09-13T14:39:27.167 INFO:teuthology.orchestra.run.smithi096.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2024-09-13T14:39:27.167 INFO:teuthology.orchestra.run.smithi096.stdout: "kstore_sync_submit_transaction": "false", 2024-09-13T14:39:27.167 INFO:teuthology.orchestra.run.smithi096.stdout: "kstore_sync_transaction": "false", 2024-09-13T14:39:27.167 INFO:teuthology.orchestra.run.smithi096.stdout: "librados_thread_count": "2", 2024-09-13T14:39:27.167 INFO:teuthology.orchestra.run.smithi096.stdout: "lockdep": "false", 2024-09-13T14:39:27.167 INFO:teuthology.orchestra.run.smithi096.stdout: "lockdep_force_backtrace": "false", 2024-09-13T14:39:27.167 INFO:teuthology.orchestra.run.smithi096.stdout: "log_coarse_timestamps": "true", 2024-09-13T14:39:27.167 INFO:teuthology.orchestra.run.smithi096.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2024-09-13T14:39:27.167 INFO:teuthology.orchestra.run.smithi096.stdout: "log_flush_on_exit": "false", 2024-09-13T14:39:27.167 INFO:teuthology.orchestra.run.smithi096.stdout: "log_graylog_host": "127.0.0.1", 2024-09-13T14:39:27.167 INFO:teuthology.orchestra.run.smithi096.stdout: "log_graylog_port": "12201", 2024-09-13T14:39:27.167 INFO:teuthology.orchestra.run.smithi096.stdout: "log_max_new": "1000", 2024-09-13T14:39:27.167 INFO:teuthology.orchestra.run.smithi096.stdout: "log_max_recent": "10000", 2024-09-13T14:39:27.167 INFO:teuthology.orchestra.run.smithi096.stdout: "log_stderr_prefix": "", 2024-09-13T14:39:27.168 INFO:teuthology.orchestra.run.smithi096.stdout: "log_stop_at_utilization": "0.970000", 2024-09-13T14:39:27.168 INFO:teuthology.orchestra.run.smithi096.stdout: "log_to_file": "true", 2024-09-13T14:39:27.168 INFO:teuthology.orchestra.run.smithi096.stdout: "log_to_graylog": "false", 2024-09-13T14:39:27.168 INFO:teuthology.orchestra.run.smithi096.stdout: "log_to_journald": "false", 2024-09-13T14:39:27.168 INFO:teuthology.orchestra.run.smithi096.stdout: "log_to_stderr": "false", 2024-09-13T14:39:27.168 INFO:teuthology.orchestra.run.smithi096.stdout: "log_to_syslog": "false", 2024-09-13T14:39:27.168 INFO:teuthology.orchestra.run.smithi096.stdout: "max_rotating_auth_attempts": "10", 2024-09-13T14:39:27.168 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2024-09-13T14:39:27.168 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_action_on_write_error": "1", 2024-09-13T14:39:27.168 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_alternate_name_max": "8192", 2024-09-13T14:39:27.168 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_asio_thread_count": "2", 2024-09-13T14:39:27.168 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_export_pin": "true", 2024-09-13T14:39:27.168 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_fragment_dirs": "true", 2024-09-13T14:39:27.168 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2024-09-13T14:39:27.169 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_fragment_interval": "5", 2024-09-13T14:39:27.169 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_fragment_size_max": "100000", 2024-09-13T14:39:27.169 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_idle_threshold": "0.000000", 2024-09-13T14:39:27.169 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_interval": "10", 2024-09-13T14:39:27.169 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_max": "-1", 2024-09-13T14:39:27.169 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_max_until": "-1", 2024-09-13T14:39:27.169 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_merge_size": "50", 2024-09-13T14:39:27.169 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_midchunk": "0.300000", 2024-09-13T14:39:27.169 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_min_rebalance": "0.100000", 2024-09-13T14:39:27.169 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_min_start": "0.200000", 2024-09-13T14:39:27.169 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_minchunk": "0.001000", 2024-09-13T14:39:27.169 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_mode": "0", 2024-09-13T14:39:27.169 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_need_max": "1.200000", 2024-09-13T14:39:27.170 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_need_min": "0.800000", 2024-09-13T14:39:27.170 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_overload_epochs": "2", 2024-09-13T14:39:27.170 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_replicate_threshold": "8000.000000", 2024-09-13T14:39:27.170 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_sample_interval": "3.000000", 2024-09-13T14:39:27.170 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_split_bits": "3", 2024-09-13T14:39:27.170 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_split_rd": "25000.000000", 2024-09-13T14:39:27.170 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_split_size": "10000", 2024-09-13T14:39:27.170 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_split_wr": "10000.000000", 2024-09-13T14:39:27.170 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_target_decay": "10.000000", 2024-09-13T14:39:27.170 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2024-09-13T14:39:27.170 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_beacon_grace": "15.000000", 2024-09-13T14:39:27.171 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_beacon_interval": "4.000000", 2024-09-13T14:39:27.171 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_beacon_mon_down_grace": "60", 2024-09-13T14:39:27.171 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_cache_memory_limit": "4294967296", 2024-09-13T14:39:27.171 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_cache_mid": "0.700000", 2024-09-13T14:39:27.171 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_cache_quiesce_decay_rate": "1.000000", 2024-09-13T14:39:27.171 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_cache_quiesce_delay": "0", 2024-09-13T14:39:27.171 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_cache_quiesce_sleep": "200", 2024-09-13T14:39:27.171 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_cache_quiesce_splitauth": "true", 2024-09-13T14:39:27.171 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_cache_quiesce_threshold": "524288", 2024-09-13T14:39:27.171 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_cache_release_free_interval": "10", 2024-09-13T14:39:27.171 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_cache_reservation": "0.050000", 2024-09-13T14:39:27.171 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_cache_trim_decay_rate": "1.000000", 2024-09-13T14:39:27.171 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_cache_trim_interval": "1", 2024-09-13T14:39:27.171 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_cache_trim_threshold": "262144", 2024-09-13T14:39:27.172 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2024-09-13T14:39:27.172 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2024-09-13T14:39:27.172 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_client_delegate_inos_pct": "50", 2024-09-13T14:39:27.172 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_client_prealloc_inos": "1000", 2024-09-13T14:39:27.172 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2024-09-13T14:39:27.172 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_connect_bootstrapping": "false", 2024-09-13T14:39:27.172 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_damage_table_max_entries": "10000", 2024-09-13T14:39:27.172 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2024-09-13T14:39:27.172 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_debug_auth_pins": "false", 2024-09-13T14:39:27.172 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_debug_frag": "false", 2024-09-13T14:39:27.172 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_debug_scatterstat": "false", 2024-09-13T14:39:27.172 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_debug_subtrees": "false", 2024-09-13T14:39:27.173 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_decay_halflife": "5.000000", 2024-09-13T14:39:27.173 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_default_dir_hash": "2", 2024-09-13T14:39:27.173 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_defer_session_stale": "true", 2024-09-13T14:39:27.173 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_deny_all_reconnect": "false", 2024-09-13T14:39:27.173 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_dir_keys_per_op": "16384", 2024-09-13T14:39:27.173 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_dir_max_commit_size": "10", 2024-09-13T14:39:27.173 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_dir_max_entries": "0", 2024-09-13T14:39:27.173 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_dir_prefetch": "true", 2024-09-13T14:39:27.173 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_dirstat_min_interval": "1.000000", 2024-09-13T14:39:27.173 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_dump_cache_after_rejoin": "false", 2024-09-13T14:39:27.173 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_dump_cache_on_map": "false", 2024-09-13T14:39:27.173 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_dump_cache_threshold_file": "0", 2024-09-13T14:39:27.173 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2024-09-13T14:39:27.173 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_early_reply": "true", 2024-09-13T14:39:27.174 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_enable_op_tracker": "true", 2024-09-13T14:39:27.174 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_enforce_unique_name": "true", 2024-09-13T14:39:27.174 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_export_ephemeral_distributed": "true", 2024-09-13T14:39:27.174 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2024-09-13T14:39:27.174 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_export_ephemeral_random": "true", 2024-09-13T14:39:27.174 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_export_ephemeral_random_max": "0.010000", 2024-09-13T14:39:27.174 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_extraordinary_events_dump_interval": "0", 2024-09-13T14:39:27.174 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_forward_all_requests_to_auth": "false", 2024-09-13T14:39:27.174 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_freeze_tree_timeout": "30.000000", 2024-09-13T14:39:27.174 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_fscrypt_last_block_max_size": "4096", 2024-09-13T14:39:27.174 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_go_bad_corrupt_dentry": "true", 2024-09-13T14:39:27.174 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2024-09-13T14:39:27.174 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_health_cache_threshold": "1.500000", 2024-09-13T14:39:27.175 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_health_summarize_threshold": "10", 2024-09-13T14:39:27.175 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_heartbeat_grace": "15.000000", 2024-09-13T14:39:27.175 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_heartbeat_reset_grace": "1000", 2024-09-13T14:39:27.175 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_inject_health_dummy": "false", 2024-09-13T14:39:27.175 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2024-09-13T14:39:27.175 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_inject_migrator_session_race": "false", 2024-09-13T14:39:27.175 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2024-09-13T14:39:27.175 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_inject_skip_replaying_inotable": "false", 2024-09-13T14:39:27.175 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2024-09-13T14:39:27.175 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_join_fs": "", 2024-09-13T14:39:27.175 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_journal_format": "1", 2024-09-13T14:39:27.175 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_kill_after_journal_logs_flushed": "false", 2024-09-13T14:39:27.175 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_kill_create_at": "0", 2024-09-13T14:39:27.175 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_kill_dirfrag_at": "0", 2024-09-13T14:39:27.176 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_kill_export_at": "0", 2024-09-13T14:39:27.176 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_kill_import_at": "0", 2024-09-13T14:39:27.176 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_kill_journal_at": "0", 2024-09-13T14:39:27.176 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_kill_journal_expire_at": "0", 2024-09-13T14:39:27.176 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_kill_journal_replay_at": "0", 2024-09-13T14:39:27.176 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_kill_link_at": "0", 2024-09-13T14:39:27.176 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_kill_mdstable_at": "0", 2024-09-13T14:39:27.176 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_kill_openc_at": "0", 2024-09-13T14:39:27.176 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_kill_rename_at": "0", 2024-09-13T14:39:27.176 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_kill_shutdown_at": "0", 2024-09-13T14:39:27.176 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_log_event_large_threshold": "524288", 2024-09-13T14:39:27.176 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_log_events_per_segment": "1024", 2024-09-13T14:39:27.176 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_log_major_segment_event_ratio": "12", 2024-09-13T14:39:27.177 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_log_max_events": "-1", 2024-09-13T14:39:27.177 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_log_max_segments": "128", 2024-09-13T14:39:27.177 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_log_pause": "false", 2024-09-13T14:39:27.177 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_log_segment_size": "0", 2024-09-13T14:39:27.177 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_log_skip_corrupt_events": "false", 2024-09-13T14:39:27.177 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_log_skip_unbounded_events": "false", 2024-09-13T14:39:27.177 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_log_trim_decay_rate": "1.000000", 2024-09-13T14:39:27.177 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_log_trim_threshold": "128", 2024-09-13T14:39:27.177 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_log_trim_upkeep_interval": "1000", 2024-09-13T14:39:27.177 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_log_warn_factor": "2.000000", 2024-09-13T14:39:27.177 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_max_caps_per_client": "1048576", 2024-09-13T14:39:27.177 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_max_completed_flushes": "100000", 2024-09-13T14:39:27.177 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_max_completed_requests": "100000", 2024-09-13T14:39:27.177 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_max_export_size": "20971520", 2024-09-13T14:39:27.178 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_max_file_recover": "32", 2024-09-13T14:39:27.178 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_max_purge_files": "64", 2024-09-13T14:39:27.178 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_max_purge_ops": "8192", 2024-09-13T14:39:27.178 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2024-09-13T14:39:27.178 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_max_scrub_ops_in_progress": "5", 2024-09-13T14:39:27.178 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_max_snaps_per_dir": "100", 2024-09-13T14:39:27.178 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_metrics_update_interval": "2", 2024-09-13T14:39:27.178 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_min_caps_per_client": "100", 2024-09-13T14:39:27.178 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_min_caps_working_set": "10000", 2024-09-13T14:39:27.178 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_mon_shutdown_timeout": "5.000000", 2024-09-13T14:39:27.178 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_numa_node": "-1", 2024-09-13T14:39:27.178 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_oft_prefetch_dirfrags": "false", 2024-09-13T14:39:27.178 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_op_complaint_time": "30.000000", 2024-09-13T14:39:27.178 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_op_history_duration": "600", 2024-09-13T14:39:27.179 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_op_history_size": "20", 2024-09-13T14:39:27.179 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_op_history_slow_op_size": "20", 2024-09-13T14:39:27.179 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_op_history_slow_op_threshold": "10", 2024-09-13T14:39:27.179 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_op_log_threshold": "5", 2024-09-13T14:39:27.179 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_ping_grace": "15", 2024-09-13T14:39:27.179 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_ping_interval": "5", 2024-09-13T14:39:27.179 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2024-09-13T14:39:27.179 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_recall_global_max_decay_threshold": "131072", 2024-09-13T14:39:27.179 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_recall_max_caps": "30000", 2024-09-13T14:39:27.179 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_recall_max_decay_rate": "1.500000", 2024-09-13T14:39:27.179 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_recall_max_decay_threshold": "131072", 2024-09-13T14:39:27.179 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_recall_warning_decay_rate": "60.000000", 2024-09-13T14:39:27.179 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_recall_warning_threshold": "262144", 2024-09-13T14:39:27.179 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_reconnect_timeout": "45.000000", 2024-09-13T14:39:27.180 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_replay_interval": "1.000000", 2024-09-13T14:39:27.180 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_replay_unsafe_with_closed_session": "false", 2024-09-13T14:39:27.180 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_request_load_average_decay_rate": "60.000000", 2024-09-13T14:39:27.180 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_root_ino_gid": "0", 2024-09-13T14:39:27.180 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_root_ino_uid": "0", 2024-09-13T14:39:27.180 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_scatter_nudge_interval": "5.000000", 2024-09-13T14:39:27.180 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_server_dispatch_client_request_delay": "0", 2024-09-13T14:39:27.180 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_server_dispatch_killpoint_random": "0.000000", 2024-09-13T14:39:27.180 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_session_blocklist_on_evict": "true", 2024-09-13T14:39:27.180 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_session_blocklist_on_timeout": "true", 2024-09-13T14:39:27.180 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2024-09-13T14:39:27.180 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_session_cache_liveness_magnitude": "10", 2024-09-13T14:39:27.180 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2024-09-13T14:39:27.180 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_session_cap_acquisition_throttle": "100000", 2024-09-13T14:39:27.181 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2024-09-13T14:39:27.181 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_session_metadata_threshold": "16777216", 2024-09-13T14:39:27.181 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_sessionmap_keys_per_op": "1024", 2024-09-13T14:39:27.181 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_shutdown_check": "0", 2024-09-13T14:39:27.181 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_skip_ino": "0", 2024-09-13T14:39:27.181 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_sleep_rank_change": "0.000000", 2024-09-13T14:39:27.181 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_snap_max_uid": "4294967294", 2024-09-13T14:39:27.181 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_snap_min_uid": "0", 2024-09-13T14:39:27.181 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_snap_rstat": "false", 2024-09-13T14:39:27.181 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_standby_replay_damaged": "false", 2024-09-13T14:39:27.181 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_symlink_recovery": "true", 2024-09-13T14:39:27.181 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_task_status_update_interval": "2.000000", 2024-09-13T14:39:27.181 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_thrash_exports": "0", 2024-09-13T14:39:27.182 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_thrash_fragments": "0", 2024-09-13T14:39:27.182 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_tick_interval": "5.000000", 2024-09-13T14:39:27.182 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_valgrind_exit": "false", 2024-09-13T14:39:27.182 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_verify_backtrace": "1", 2024-09-13T14:39:27.182 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_verify_scatter": "false", 2024-09-13T14:39:27.182 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_wipe_ino_prealloc": "false", 2024-09-13T14:39:27.182 INFO:teuthology.orchestra.run.smithi096.stdout: "mds_wipe_sessions": "false", 2024-09-13T14:39:27.182 INFO:teuthology.orchestra.run.smithi096.stdout: "mempool_debug": "false", 2024-09-13T14:39:27.182 INFO:teuthology.orchestra.run.smithi096.stdout: "memstore_debug_omit_block_device_write": "false", 2024-09-13T14:39:27.182 INFO:teuthology.orchestra.run.smithi096.stdout: "memstore_device_bytes": "1073741824", 2024-09-13T14:39:27.182 INFO:teuthology.orchestra.run.smithi096.stdout: "memstore_page_set": "false", 2024-09-13T14:39:27.182 INFO:teuthology.orchestra.run.smithi096.stdout: "memstore_page_size": "65536", 2024-09-13T14:39:27.182 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_client_bytes": "134217728", 2024-09-13T14:39:27.183 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_client_messages": "512", 2024-09-13T14:39:27.183 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2024-09-13T14:39:27.183 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_connect_retry_interval": "1.000000", 2024-09-13T14:39:27.183 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2024-09-13T14:39:27.183 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2024-09-13T14:39:27.183 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_disabled_modules": "", 2024-09-13T14:39:27.183 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_enable_op_tracker": "true", 2024-09-13T14:39:27.183 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_initial_modules": "restful iostat nfs", 2024-09-13T14:39:27.183 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_max_pg_creating": "1024", 2024-09-13T14:39:27.183 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_max_pg_num_change": "128", 2024-09-13T14:39:27.183 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_mds_bytes": "134217728", 2024-09-13T14:39:27.183 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_mds_messages": "128", 2024-09-13T14:39:27.183 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2024-09-13T14:39:27.183 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_mon_bytes": "134217728", 2024-09-13T14:39:27.184 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_mon_messages": "128", 2024-09-13T14:39:27.184 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_num_op_tracker_shard": "32", 2024-09-13T14:39:27.184 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_op_complaint_time": "30.000000", 2024-09-13T14:39:27.184 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_op_history_duration": "600", 2024-09-13T14:39:27.184 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_op_history_size": "20", 2024-09-13T14:39:27.184 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_op_history_slow_op_size": "20", 2024-09-13T14:39:27.184 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_op_history_slow_op_threshold": "10.000000", 2024-09-13T14:39:27.184 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_op_log_threshold": "5", 2024-09-13T14:39:27.184 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_osd_bytes": "536870912", 2024-09-13T14:39:27.184 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_osd_messages": "8192", 2024-09-13T14:39:27.184 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_pool": "true", 2024-09-13T14:39:27.184 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_service_beacon_grace": "60.000000", 2024-09-13T14:39:27.184 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_standby_modules": "true", 2024-09-13T14:39:27.184 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_stats_period": "5", 2024-09-13T14:39:27.185 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_stats_threshold": "5", 2024-09-13T14:39:27.185 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_tick_period": "2", 2024-09-13T14:39:27.185 INFO:teuthology.orchestra.run.smithi096.stdout: "mgr_ttl_cache_expire_seconds": "0", 2024-09-13T14:39:27.185 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_accept_timeout_factor": "2.000000", 2024-09-13T14:39:27.185 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_allow_pool_delete": "true", 2024-09-13T14:39:27.185 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_allow_pool_size_one": "false", 2024-09-13T14:39:27.185 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_auth_validate_all_caps": "true", 2024-09-13T14:39:27.185 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2024-09-13T14:39:27.185 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2024-09-13T14:39:27.185 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_client_bytes": "104857600", 2024-09-13T14:39:27.185 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_client_directed_command_retry": "2", 2024-09-13T14:39:27.185 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_client_hunt_interval": "3.000000", 2024-09-13T14:39:27.185 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2024-09-13T14:39:27.186 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2024-09-13T14:39:27.186 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2024-09-13T14:39:27.186 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_client_hunt_parallel": "3", 2024-09-13T14:39:27.186 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_client_log_interval": "1.000000", 2024-09-13T14:39:27.186 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_client_max_log_entries_per_message": "1000", 2024-09-13T14:39:27.186 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_client_ping_interval": "10.000000", 2024-09-13T14:39:27.186 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_client_ping_timeout": "30.000000", 2024-09-13T14:39:27.186 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_client_target_rank": "-1", 2024-09-13T14:39:27.186 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_clock_drift_allowed": "1.000000", 2024-09-13T14:39:27.186 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2024-09-13T14:39:27.186 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2024-09-13T14:39:27.186 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_cluster_log_level": "debug", 2024-09-13T14:39:27.186 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_cluster_log_to_file": "true", 2024-09-13T14:39:27.186 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_cluster_log_to_graylog": "false", 2024-09-13T14:39:27.187 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2024-09-13T14:39:27.187 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_cluster_log_to_graylog_port": "12201", 2024-09-13T14:39:27.187 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_cluster_log_to_journald": "true", 2024-09-13T14:39:27.187 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_cluster_log_to_stderr": "false", 2024-09-13T14:39:27.187 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_cluster_log_to_syslog": "default=false", 2024-09-13T14:39:27.187 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2024-09-13T14:39:27.187 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_compact_on_bootstrap": "false", 2024-09-13T14:39:27.187 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_compact_on_start": "false", 2024-09-13T14:39:27.187 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_compact_on_trim": "true", 2024-09-13T14:39:27.187 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_con_tracker_persist_interval": "10", 2024-09-13T14:39:27.187 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_con_tracker_score_halflife": "43200", 2024-09-13T14:39:27.187 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_config_key_max_entry_size": "65536", 2024-09-13T14:39:27.187 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_cpu_threads": "4", 2024-09-13T14:39:27.187 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_crush_min_required_version": "hammer", 2024-09-13T14:39:27.188 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_daemon_bytes": "419430400", 2024-09-13T14:39:27.188 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2024-09-13T14:39:27.188 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_data_avail_crit": "5", 2024-09-13T14:39:27.188 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_data_avail_warn": "5", 2024-09-13T14:39:27.188 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_data_size_warn": "16106127360", 2024-09-13T14:39:27.188 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_debug_block_osdmap_trim": "false", 2024-09-13T14:39:27.188 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_debug_deprecated_as_obsolete": "false", 2024-09-13T14:39:27.188 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_debug_dump_json": "false", 2024-09-13T14:39:27.188 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2024-09-13T14:39:27.188 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_debug_dump_transactions": "false", 2024-09-13T14:39:27.188 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_debug_extra_checks": "false", 2024-09-13T14:39:27.188 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_debug_no_initial_persistent_features": "false", 2024-09-13T14:39:27.188 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2024-09-13T14:39:27.189 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_debug_no_require_reef": "false", 2024-09-13T14:39:27.189 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_debug_no_require_squid": "false", 2024-09-13T14:39:27.189 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2024-09-13T14:39:27.189 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_delta_reset_interval": "10.000000", 2024-09-13T14:39:27.189 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_dns_srv_name": "ceph-mon", 2024-09-13T14:39:27.189 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_down_mkfs_grace": "60", 2024-09-13T14:39:27.189 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_down_uptime_grace": "60", 2024-09-13T14:39:27.189 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_election_default_strategy": "1", 2024-09-13T14:39:27.189 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_election_timeout": "5.000000", 2024-09-13T14:39:27.189 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2024-09-13T14:39:27.189 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_elector_ping_divisor": "2", 2024-09-13T14:39:27.189 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_elector_ping_timeout": "2.000000", 2024-09-13T14:39:27.189 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_enable_op_tracker": "true", 2024-09-13T14:39:27.189 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_fake_pool_delete": "false", 2024-09-13T14:39:27.190 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_force_quorum_join": "false", 2024-09-13T14:39:27.190 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_fsmap_prune_threshold": "300", 2024-09-13T14:39:27.190 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_globalid_prealloc": "10000", 2024-09-13T14:39:27.190 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_health_detail_to_clog": "true", 2024-09-13T14:39:27.190 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_health_log_update_period": "5", 2024-09-13T14:39:27.190 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_health_max_detail": "50", 2024-09-13T14:39:27.190 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_health_to_clog": "true", 2024-09-13T14:39:27.190 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_health_to_clog_interval": "600", 2024-09-13T14:39:27.190 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2024-09-13T14:39:27.190 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_host": "[v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] [v2:172.21.15.96:3301/0,v1:172.21.15.96:6790/0]", 2024-09-13T14:39:27.190 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_host_override": "", 2024-09-13T14:39:27.190 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_initial_members": "", 2024-09-13T14:39:27.190 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2024-09-13T14:39:27.190 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2024-09-13T14:39:27.191 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_inject_transaction_delay_max": "10.000000", 2024-09-13T14:39:27.191 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2024-09-13T14:39:27.191 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_keyvaluedb": "rocksdb", 2024-09-13T14:39:27.191 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_lease": "5.000000", 2024-09-13T14:39:27.191 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2024-09-13T14:39:27.191 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_lease_renew_interval_factor": "0.600000", 2024-09-13T14:39:27.191 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_log_full_interval": "50", 2024-09-13T14:39:27.191 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_log_max": "10000", 2024-09-13T14:39:27.191 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_log_max_summary": "50", 2024-09-13T14:39:27.191 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_max_log_entries_per_event": "4096", 2024-09-13T14:39:27.191 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_max_log_epochs": "500", 2024-09-13T14:39:27.191 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_max_mdsmap_epochs": "500", 2024-09-13T14:39:27.191 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_max_mgrmap_epochs": "500", 2024-09-13T14:39:27.191 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_max_nvmeof_epochs": "500", 2024-09-13T14:39:27.192 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_max_osd": "10000", 2024-09-13T14:39:27.192 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_max_pg_per_osd": "10000", 2024-09-13T14:39:27.192 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_max_pool_pg_num": "65536", 2024-09-13T14:39:27.192 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_max_snap_prune_per_epoch": "100", 2024-09-13T14:39:27.192 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_mds_blocklist_interval": "86400.000000", 2024-09-13T14:39:27.192 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_mds_force_trim_to": "0", 2024-09-13T14:39:27.192 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_mds_skip_sanity": "true", 2024-09-13T14:39:27.192 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_memory_autotune": "true", 2024-09-13T14:39:27.192 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_memory_target": "2147483648", 2024-09-13T14:39:27.192 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_mgr_beacon_grace": "30", 2024-09-13T14:39:27.193 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2024-09-13T14:39:27.193 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_mgr_digest_period": "5", 2024-09-13T14:39:27.193 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_mgr_inactive_grace": "60", 2024-09-13T14:39:27.193 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_mgr_mkfs_grace": "240", 2024-09-13T14:39:27.193 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2024-09-13T14:39:27.193 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_min_osdmap_epochs": "500", 2024-09-13T14:39:27.193 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_nvmeofgw_beacon_grace": "10", 2024-09-13T14:39:27.193 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_nvmeofgw_set_group_id_retry": "1000", 2024-09-13T14:39:27.193 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_op_complaint_time": "30", 2024-09-13T14:39:27.193 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_op_history_duration": "600", 2024-09-13T14:39:27.194 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_op_history_size": "20", 2024-09-13T14:39:27.194 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_op_history_slow_op_size": "20", 2024-09-13T14:39:27.194 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_op_history_slow_op_threshold": "10", 2024-09-13T14:39:27.194 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_op_log_threshold": "5", 2024-09-13T14:39:27.194 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_adjust_down_out_interval": "true", 2024-09-13T14:39:27.194 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2024-09-13T14:39:27.194 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2024-09-13T14:39:27.194 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_auto_mark_in": "false", 2024-09-13T14:39:27.194 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_auto_mark_new_in": "true", 2024-09-13T14:39:27.194 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2024-09-13T14:39:27.194 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2024-09-13T14:39:27.194 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_cache_size": "500", 2024-09-13T14:39:27.194 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_cache_size_min": "134217728", 2024-09-13T14:39:27.194 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_crush_smoke_test": "true", 2024-09-13T14:39:27.195 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_destroyed_out_interval": "600", 2024-09-13T14:39:27.195 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_down_out_interval": "600", 2024-09-13T14:39:27.195 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_down_out_subtree_limit": "rack", 2024-09-13T14:39:27.195 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2024-09-13T14:39:27.195 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_force_trim_to": "0", 2024-09-13T14:39:27.195 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_full_ratio": "0.950000", 2024-09-13T14:39:27.195 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2024-09-13T14:39:27.195 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_laggy_halflife": "3600", 2024-09-13T14:39:27.195 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_laggy_max_interval": "300", 2024-09-13T14:39:27.195 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_laggy_weight": "0.300000", 2024-09-13T14:39:27.195 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2024-09-13T14:39:27.195 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_max_initial_pgs": "1024", 2024-09-13T14:39:27.195 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_min_down_reporters": "2", 2024-09-13T14:39:27.195 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_min_in_ratio": "0.750000", 2024-09-13T14:39:27.196 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_min_up_ratio": "0.300000", 2024-09-13T14:39:27.196 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_nearfull_ratio": "0.850000", 2024-09-13T14:39:27.196 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_prime_pg_temp": "true", 2024-09-13T14:39:27.196 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2024-09-13T14:39:27.196 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2024-09-13T14:39:27.196 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_report_timeout": "900", 2024-09-13T14:39:27.196 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_reporter_subtree_level": "osd", 2024-09-13T14:39:27.197 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2024-09-13T14:39:27.197 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_warn_num_repaired": "10", 2024-09-13T14:39:27.197 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osd_warn_op_age": "32.000000", 2024-09-13T14:39:27.197 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osdmap_full_prune_enabled": "true", 2024-09-13T14:39:27.197 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osdmap_full_prune_interval": "10", 2024-09-13T14:39:27.197 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osdmap_full_prune_min": "10000", 2024-09-13T14:39:27.197 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_osdmap_full_prune_txsize": "100", 2024-09-13T14:39:27.198 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2024-09-13T14:39:27.198 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_pg_stuck_threshold": "60", 2024-09-13T14:39:27.198 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2024-09-13T14:39:27.198 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_pg_warn_min_objects": "10000", 2024-09-13T14:39:27.198 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_pg_warn_min_per_osd": "0", 2024-09-13T14:39:27.198 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_pg_warn_min_pool_objects": "1000", 2024-09-13T14:39:27.198 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_pool_quota_crit_threshold": "0", 2024-09-13T14:39:27.198 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_pool_quota_warn_threshold": "0", 2024-09-13T14:39:27.198 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_probe_timeout": "2.000000", 2024-09-13T14:39:27.198 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_reweight_max_change": "0.050000", 2024-09-13T14:39:27.198 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_reweight_max_osds": "4", 2024-09-13T14:39:27.198 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_reweight_min_bytes_per_osd": "10", 2024-09-13T14:39:27.198 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_reweight_min_pgs_per_osd": "4", 2024-09-13T14:39:27.198 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2024-09-13T14:39:27.199 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2024-09-13T14:39:27.199 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2024-09-13T14:39:27.199 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_scrub_interval": "86400", 2024-09-13T14:39:27.199 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_scrub_max_keys": "100", 2024-09-13T14:39:27.199 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_scrub_timeout": "300", 2024-09-13T14:39:27.199 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_session_timeout": "300", 2024-09-13T14:39:27.199 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_smart_report_timeout": "5", 2024-09-13T14:39:27.199 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_stat_smooth_intervals": "6", 2024-09-13T14:39:27.199 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2024-09-13T14:39:27.199 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_stretch_pool_min_size": "2", 2024-09-13T14:39:27.199 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_stretch_pool_size": "4", 2024-09-13T14:39:27.199 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2024-09-13T14:39:27.199 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_subscribe_interval": "86400.000000", 2024-09-13T14:39:27.200 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_sync_debug": "false", 2024-09-13T14:39:27.200 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_sync_max_payload_keys": "2000", 2024-09-13T14:39:27.200 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_sync_max_payload_size": "1048576", 2024-09-13T14:39:27.200 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_sync_provider_kill_at": "0", 2024-09-13T14:39:27.200 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_sync_requester_kill_at": "0", 2024-09-13T14:39:27.200 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_sync_timeout": "60.000000", 2024-09-13T14:39:27.200 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_target_pg_per_osd": "100", 2024-09-13T14:39:27.200 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_tick_interval": "5", 2024-09-13T14:39:27.200 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_timecheck_interval": "300.000000", 2024-09-13T14:39:27.200 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_timecheck_skew_interval": "30.000000", 2024-09-13T14:39:27.200 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_use_min_delay_socket": "false", 2024-09-13T14:39:27.200 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_older_version_delay": "604800", 2024-09-13T14:39:27.200 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2024-09-13T14:39:27.200 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2024-09-13T14:39:27.201 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2024-09-13T14:39:27.201 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_filestore_osds": "true", 2024-09-13T14:39:27.201 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2024-09-13T14:39:27.201 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2024-09-13T14:39:27.201 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2024-09-13T14:39:27.201 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_misplaced": "false", 2024-09-13T14:39:27.201 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2024-09-13T14:39:27.201 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_older_version": "true", 2024-09-13T14:39:27.201 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2024-09-13T14:39:27.201 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_pool_no_app": "true", 2024-09-13T14:39:27.201 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_pool_no_redundancy": "true", 2024-09-13T14:39:27.201 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2024-09-13T14:39:27.201 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2024-09-13T14:39:27.201 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2024-09-13T14:39:27.202 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_on_too_few_osds": "false", 2024-09-13T14:39:27.202 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2024-09-13T14:39:27.202 INFO:teuthology.orchestra.run.smithi096.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2024-09-13T14:39:27.202 INFO:teuthology.orchestra.run.smithi096.stdout: "monmap": "", 2024-09-13T14:39:27.202 INFO:teuthology.orchestra.run.smithi096.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-13T14:39:27.202 INFO:teuthology.orchestra.run.smithi096.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2024-09-13T14:39:27.202 INFO:teuthology.orchestra.run.smithi096.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2024-09-13T14:39:27.202 INFO:teuthology.orchestra.run.smithi096.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-13T14:39:27.202 INFO:teuthology.orchestra.run.smithi096.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2024-09-13T14:39:27.202 INFO:teuthology.orchestra.run.smithi096.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2024-09-13T14:39:27.202 INFO:teuthology.orchestra.run.smithi096.stdout: "motr_tracing_enabled": "false", 2024-09-13T14:39:27.202 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_op_threads": "3", 2024-09-13T14:39:27.202 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_buffer_size": "131072", 2024-09-13T14:39:27.203 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_cm": "false", 2024-09-13T14:39:27.203 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_device_name": "", 2024-09-13T14:39:27.203 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_dscp": "96", 2024-09-13T14:39:27.203 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_enable_hugepage": "false", 2024-09-13T14:39:27.203 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_gid_idx": "0", 2024-09-13T14:39:27.203 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_local_gid": "", 2024-09-13T14:39:27.203 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_polling_us": "1000", 2024-09-13T14:39:27.203 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_port_num": "1", 2024-09-13T14:39:27.203 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_receive_buffers": "32768", 2024-09-13T14:39:27.203 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_receive_queue_len": "4096", 2024-09-13T14:39:27.203 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_roce_ver": "1", 2024-09-13T14:39:27.203 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_send_buffers": "1024", 2024-09-13T14:39:27.203 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_sl": "3", 2024-09-13T14:39:27.203 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_support_srq": "true", 2024-09-13T14:39:27.204 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_rdma_type": "ib", 2024-09-13T14:39:27.204 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_async_reap_threshold": "5", 2024-09-13T14:39:27.204 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_bind_before_connect": "false", 2024-09-13T14:39:27.204 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_bind_ipv4": "true", 2024-09-13T14:39:27.204 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_bind_ipv6": "false", 2024-09-13T14:39:27.204 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_bind_msgr1": "true", 2024-09-13T14:39:27.204 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_bind_msgr2": "true", 2024-09-13T14:39:27.204 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_bind_port_max": "7568", 2024-09-13T14:39:27.204 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_bind_port_min": "6800", 2024-09-13T14:39:27.204 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_bind_prefer_ipv4": "false", 2024-09-13T14:39:27.204 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_bind_retry_count": "3", 2024-09-13T14:39:27.204 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_bind_retry_delay": "5", 2024-09-13T14:39:27.204 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_blackhole_client": "false", 2024-09-13T14:39:27.204 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_blackhole_mds": "false", 2024-09-13T14:39:27.205 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_blackhole_mgr": "false", 2024-09-13T14:39:27.205 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_blackhole_mon": "false", 2024-09-13T14:39:27.205 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_blackhole_osd": "false", 2024-09-13T14:39:27.205 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_client_mode": "crc secure", 2024-09-13T14:39:27.205 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_client_throttle_retry_time_interval": "5000", 2024-09-13T14:39:27.205 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_cluster_mode": "crc secure", 2024-09-13T14:39:27.205 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_cluster_type": "", 2024-09-13T14:39:27.205 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_compress_secure": "false", 2024-09-13T14:39:27.206 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_connection_idle_timeout": "900", 2024-09-13T14:39:27.206 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_connection_ready_timeout": "10", 2024-09-13T14:39:27.206 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_crc_data": "true", 2024-09-13T14:39:27.206 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_crc_header": "true", 2024-09-13T14:39:27.206 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_die_on_bad_msg": "false", 2024-09-13T14:39:27.206 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_die_on_bug": "true", 2024-09-13T14:39:27.206 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_die_on_old_message": "true", 2024-09-13T14:39:27.207 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_die_on_skipped_message": "false", 2024-09-13T14:39:27.207 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_die_on_unhandled_msg": "false", 2024-09-13T14:39:27.207 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dispatch_throttle_bytes": "104857600", 2024-09-13T14:39:27.207 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dpdk_coremask": "0xF", 2024-09-13T14:39:27.207 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dpdk_debug_allow_loopback": "false", 2024-09-13T14:39:27.207 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dpdk_devs_allowlist": "", 2024-09-13T14:39:27.207 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dpdk_enable_tso": "true", 2024-09-13T14:39:27.207 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2024-09-13T14:39:27.207 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dpdk_host_ipv4_addr": "", 2024-09-13T14:39:27.207 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dpdk_hugepages": "", 2024-09-13T14:39:27.207 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dpdk_hw_flow_control": "true", 2024-09-13T14:39:27.207 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2024-09-13T14:39:27.207 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dpdk_lro": "true", 2024-09-13T14:39:27.207 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dpdk_memory_channel": "4", 2024-09-13T14:39:27.208 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2024-09-13T14:39:27.208 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dpdk_pmd": "", 2024-09-13T14:39:27.208 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dpdk_port_id": "0", 2024-09-13T14:39:27.208 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2024-09-13T14:39:27.208 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dump_corrupt_message_level": "1", 2024-09-13T14:39:27.208 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_dump_on_send": "false", 2024-09-13T14:39:27.208 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_initial_backoff": "0.200000", 2024-09-13T14:39:27.208 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_inject_delay_max": "1.000000", 2024-09-13T14:39:27.208 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_inject_delay_probability": "0.000000", 2024-09-13T14:39:27.208 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_inject_delay_type": "", 2024-09-13T14:39:27.208 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_inject_internal_delays": "0.000000", 2024-09-13T14:39:27.208 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_inject_network_congestion": "0", 2024-09-13T14:39:27.208 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_inject_socket_failures": "0", 2024-09-13T14:39:27.208 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_learn_addr_from_peer": "true", 2024-09-13T14:39:27.209 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_max_accept_failures": "4", 2024-09-13T14:39:27.209 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_max_backoff": "15.000000", 2024-09-13T14:39:27.209 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_mon_client_mode": "secure crc", 2024-09-13T14:39:27.209 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_mon_cluster_mode": "secure crc", 2024-09-13T14:39:27.209 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_mon_service_mode": "secure crc", 2024-09-13T14:39:27.209 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_osd_compress_min_size": "1024", 2024-09-13T14:39:27.209 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_osd_compress_mode": "none", 2024-09-13T14:39:27.209 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_osd_compression_algorithm": "snappy", 2024-09-13T14:39:27.209 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2024-09-13T14:39:27.209 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_pq_min_cost": "65536", 2024-09-13T14:39:27.209 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_public_type": "", 2024-09-13T14:39:27.209 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_service_mode": "crc secure", 2024-09-13T14:39:27.209 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_tcp_listen_backlog": "512", 2024-09-13T14:39:27.209 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_tcp_nodelay": "true", 2024-09-13T14:39:27.210 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_tcp_prefetch_max_size": "4096", 2024-09-13T14:39:27.210 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_tcp_rcvbuf": "0", 2024-09-13T14:39:27.210 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_time_events_min_wait_interval": "1000", 2024-09-13T14:39:27.210 INFO:teuthology.orchestra.run.smithi096.stdout: "ms_type": "async+posix", 2024-09-13T14:39:27.210 INFO:teuthology.orchestra.run.smithi096.stdout: "no_config_file": "false", 2024-09-13T14:39:27.210 INFO:teuthology.orchestra.run.smithi096.stdout: "nvmeof_mon_client_disconnect_panic": "100", 2024-09-13T14:39:27.210 INFO:teuthology.orchestra.run.smithi096.stdout: "nvmeof_mon_client_tick_period": "2", 2024-09-13T14:39:27.210 INFO:teuthology.orchestra.run.smithi096.stdout: "objecter_completion_locks_per_session": "32", 2024-09-13T14:39:27.210 INFO:teuthology.orchestra.run.smithi096.stdout: "objecter_debug_inject_relock_delay": "false", 2024-09-13T14:39:27.210 INFO:teuthology.orchestra.run.smithi096.stdout: "objecter_inflight_op_bytes": "104857600", 2024-09-13T14:39:27.210 INFO:teuthology.orchestra.run.smithi096.stdout: "objecter_inflight_ops": "1024", 2024-09-13T14:39:27.210 INFO:teuthology.orchestra.run.smithi096.stdout: "objecter_inject_no_watch_ping": "false", 2024-09-13T14:39:27.210 INFO:teuthology.orchestra.run.smithi096.stdout: "objecter_retry_writes_after_first_reply": "false", 2024-09-13T14:39:27.210 INFO:teuthology.orchestra.run.smithi096.stdout: "objecter_tick_interval": "5.000000", 2024-09-13T14:39:27.211 INFO:teuthology.orchestra.run.smithi096.stdout: "objecter_timeout": "10.000000", 2024-09-13T14:39:27.211 INFO:teuthology.orchestra.run.smithi096.stdout: "objectstore_blackhole": "false", 2024-09-13T14:39:27.211 INFO:teuthology.orchestra.run.smithi096.stdout: "objectstore_debug_throw_on_failed_txc": "false", 2024-09-13T14:39:27.211 INFO:teuthology.orchestra.run.smithi096.stdout: "openssl_engine_opts": "", 2024-09-13T14:39:27.211 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_agent_delay_time": "5.000000", 2024-09-13T14:39:27.211 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_agent_hist_halflife": "1000", 2024-09-13T14:39:27.211 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_agent_max_low_ops": "2", 2024-09-13T14:39:27.211 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_agent_max_ops": "4", 2024-09-13T14:39:27.211 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_agent_min_evict_effort": "0.100000", 2024-09-13T14:39:27.212 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_agent_quantize_effort": "0.100000", 2024-09-13T14:39:27.212 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_agent_slop": "0.020000", 2024-09-13T14:39:27.212 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_aggregated_slow_ops_logging": "true", 2024-09-13T14:39:27.212 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_allow_recovery_below_min_size": "true", 2024-09-13T14:39:27.212 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_asio_thread_count": "2", 2024-09-13T14:39:27.212 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_async_recovery_min_cost": "100", 2024-09-13T14:39:27.212 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_auto_mark_unfound_lost": "false", 2024-09-13T14:39:27.212 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_backfill_retry_interval": "30.000000", 2024-09-13T14:39:27.212 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_backfill_scan_max": "512", 2024-09-13T14:39:27.212 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_backfill_scan_min": "64", 2024-09-13T14:39:27.212 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_backoff_on_degraded": "false", 2024-09-13T14:39:27.212 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_backoff_on_peering": "false", 2024-09-13T14:39:27.212 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_backoff_on_unfound": "true", 2024-09-13T14:39:27.212 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_beacon_report_interval": "300", 2024-09-13T14:39:27.213 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_bench_duration": "30", 2024-09-13T14:39:27.213 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_bench_large_size_max_throughput": "104857600", 2024-09-13T14:39:27.213 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_bench_max_block_size": "67108864", 2024-09-13T14:39:27.213 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_bench_small_size_max_iops": "100", 2024-09-13T14:39:27.213 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_blkin_trace_all": "false", 2024-09-13T14:39:27.213 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_blocked_scrub_grace_period": "120", 2024-09-13T14:39:27.213 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2024-09-13T14:39:27.213 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2024-09-13T14:39:27.213 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2024-09-13T14:39:27.213 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_check_for_log_corruption": "false", 2024-09-13T14:39:27.213 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_check_max_object_name_len_on_startup": "true", 2024-09-13T14:39:27.213 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-13T14:39:27.214 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2024-09-13T14:39:27.214 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-13T14:39:27.214 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_class_update_on_start": "true", 2024-09-13T14:39:27.214 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_client_message_cap": "256", 2024-09-13T14:39:27.214 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_client_message_size_cap": "524288000", 2024-09-13T14:39:27.214 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_client_op_priority": "63", 2024-09-13T14:39:27.214 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_client_watch_timeout": "30", 2024-09-13T14:39:27.214 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_command_max_records": "256", 2024-09-13T14:39:27.214 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_compact_on_start": "false", 2024-09-13T14:39:27.214 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_copyfrom_max_chunk": "8388608", 2024-09-13T14:39:27.214 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_crush_chooseleaf_type": "1", 2024-09-13T14:39:27.214 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_crush_initial_weight": "-1.000000", 2024-09-13T14:39:27.214 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_crush_update_on_start": "true", 2024-09-13T14:39:27.215 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_crush_update_weight_set": "true", 2024-09-13T14:39:27.215 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2024-09-13T14:39:27.215 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2024-09-13T14:39:27.215 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2024-09-13T14:39:27.215 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_drop_ping_duration": "0", 2024-09-13T14:39:27.215 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_drop_ping_probability": "0.000000", 2024-09-13T14:39:27.215 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_feed_pullee": "-1", 2024-09-13T14:39:27.215 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_inject_copyfrom_error": "false", 2024-09-13T14:39:27.215 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2024-09-13T14:39:27.215 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2024-09-13T14:39:27.215 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_misdirected_ops": "false", 2024-09-13T14:39:27.215 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_no_acting_change": "false", 2024-09-13T14:39:27.215 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_no_purge_strays": "false", 2024-09-13T14:39:27.215 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_op_order": "false", 2024-09-13T14:39:27.216 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_pg_log_writeout": "false", 2024-09-13T14:39:27.216 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_pretend_recovery_active": "false", 2024-09-13T14:39:27.216 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_random_push_read_error": "0.000000", 2024-09-13T14:39:27.216 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2024-09-13T14:39:27.216 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_shutdown": "false", 2024-09-13T14:39:27.216 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2024-09-13T14:39:27.216 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2024-09-13T14:39:27.216 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_trim_objects": "false", 2024-09-13T14:39:27.216 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_verify_cached_snaps": "false", 2024-09-13T14:39:27.216 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_verify_missing_on_start": "false", 2024-09-13T14:39:27.216 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_verify_snaps": "false", 2024-09-13T14:39:27.216 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_debug_verify_stray_on_activate": "false", 2024-09-13T14:39:27.216 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_deep_scrub_interval": "604800.000000", 2024-09-13T14:39:27.217 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_deep_scrub_keys": "1024", 2024-09-13T14:39:27.217 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2024-09-13T14:39:27.217 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2024-09-13T14:39:27.217 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2024-09-13T14:39:27.217 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_deep_scrub_stride": "524288", 2024-09-13T14:39:27.217 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2024-09-13T14:39:27.217 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_default_data_pool_replay_window": "45", 2024-09-13T14:39:27.217 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_default_notify_timeout": "30", 2024-09-13T14:39:27.217 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_delete_sleep": "0.000000", 2024-09-13T14:39:27.217 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_delete_sleep_hdd": "5.000000", 2024-09-13T14:39:27.217 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_delete_sleep_hybrid": "1.000000", 2024-09-13T14:39:27.217 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_delete_sleep_ssd": "1.000000", 2024-09-13T14:39:27.217 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_discard_disconnected_ops": "true", 2024-09-13T14:39:27.217 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_ec_partial_reads": "true", 2024-09-13T14:39:27.218 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_enable_op_tracker": "true", 2024-09-13T14:39:27.218 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2024-09-13T14:39:27.218 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_extblkdev_plugins": "vdo", 2024-09-13T14:39:27.218 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_failsafe_full_ratio": "0.970000", 2024-09-13T14:39:27.218 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_fast_fail_on_connection_refused": "true", 2024-09-13T14:39:27.218 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_fast_info": "true", 2024-09-13T14:39:27.218 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_fast_shutdown": "true", 2024-09-13T14:39:27.218 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_fast_shutdown_notify_mon": "true", 2024-09-13T14:39:27.218 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_fast_shutdown_timeout": "15", 2024-09-13T14:39:27.218 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_find_best_info_ignore_history_les": "false", 2024-09-13T14:39:27.218 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_force_auth_primary_missing_objects": "100", 2024-09-13T14:39:27.218 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2024-09-13T14:39:27.218 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_function_tracing": "false", 2024-09-13T14:39:27.218 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_heartbeat_grace": "20", 2024-09-13T14:39:27.219 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_heartbeat_interval": "6", 2024-09-13T14:39:27.219 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2024-09-13T14:39:27.219 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_heartbeat_min_peers": "10", 2024-09-13T14:39:27.219 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_heartbeat_min_size": "2000", 2024-09-13T14:39:27.219 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_heartbeat_stale": "600", 2024-09-13T14:39:27.219 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2024-09-13T14:39:27.219 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_hit_set_max_size": "100000", 2024-09-13T14:39:27.220 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_hit_set_min_size": "1000", 2024-09-13T14:39:27.220 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_hit_set_namespace": ".ceph-internal", 2024-09-13T14:39:27.220 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_ignore_stale_divergent_priors": "false", 2024-09-13T14:39:27.221 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2024-09-13T14:39:27.221 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_inject_failure_on_pg_removal": "false", 2024-09-13T14:39:27.221 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2024-09-13T14:39:27.221 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_journal_flush_on_shutdown": "true", 2024-09-13T14:39:27.221 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_journal_size": "5120", 2024-09-13T14:39:27.221 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_kill_backfill_at": "0", 2024-09-13T14:39:27.221 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_loop_before_reset_tphandle": "64", 2024-09-13T14:39:27.221 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_map_cache_size": "50", 2024-09-13T14:39:27.221 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_map_dedup": "true", 2024-09-13T14:39:27.221 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_map_message_max": "40", 2024-09-13T14:39:27.221 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_map_message_max_bytes": "10485760", 2024-09-13T14:39:27.221 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_map_share_max_epochs": "40", 2024-09-13T14:39:27.222 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_attr_name_len": "100", 2024-09-13T14:39:27.222 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_attr_size": "0", 2024-09-13T14:39:27.222 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_backfills": "1", 2024-09-13T14:39:27.222 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_markdown_count": "5", 2024-09-13T14:39:27.223 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_markdown_period": "600", 2024-09-13T14:39:27.223 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_object_name_len": "2048", 2024-09-13T14:39:27.223 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_object_namespace_len": "256", 2024-09-13T14:39:27.223 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_object_size": "134217728", 2024-09-13T14:39:27.223 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2024-09-13T14:39:27.223 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_omap_entries_per_request": "1024", 2024-09-13T14:39:27.223 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_pg_blocked_by": "16", 2024-09-13T14:39:27.223 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_pg_log_entries": "10000", 2024-09-13T14:39:27.223 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2024-09-13T14:39:27.223 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_pgls": "1024", 2024-09-13T14:39:27.223 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_push_cost": "8388608", 2024-09-13T14:39:27.223 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_push_objects": "10", 2024-09-13T14:39:27.223 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_scrubs": "3", 2024-09-13T14:39:27.223 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2024-09-13T14:39:27.224 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_trimming_pgs": "2", 2024-09-13T14:39:27.224 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_write_op_reply_len": "64", 2024-09-13T14:39:27.224 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_max_write_size": "90", 2024-09-13T14:39:27.224 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2024-09-13T14:39:27.224 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2024-09-13T14:39:27.224 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2024-09-13T14:39:27.224 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2024-09-13T14:39:27.224 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2024-09-13T14:39:27.224 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2024-09-13T14:39:27.224 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2024-09-13T14:39:27.224 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_override_recovery_settings": "false", 2024-09-13T14:39:27.224 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_profile": "balanced", 2024-09-13T14:39:27.224 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2024-09-13T14:39:27.225 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2024-09-13T14:39:27.225 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2024-09-13T14:39:27.225 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2024-09-13T14:39:27.225 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2024-09-13T14:39:27.225 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2024-09-13T14:39:27.225 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2024-09-13T14:39:27.225 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2024-09-13T14:39:27.225 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2024-09-13T14:39:27.225 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_scheduler_client_wgt": "1", 2024-09-13T14:39:27.225 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mclock_skip_benchmark": "false", 2024-09-13T14:39:27.225 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_memory_base": "805306368", 2024-09-13T14:39:27.225 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_memory_cache_min": "134217728", 2024-09-13T14:39:27.225 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_memory_cache_resize_interval": "1.000000", 2024-09-13T14:39:27.225 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_memory_expected_fragmentation": "0.150000", 2024-09-13T14:39:27.226 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_memory_target": "4294967296", 2024-09-13T14:39:27.226 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_memory_target_autotune": "false", 2024-09-13T14:39:27.226 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2024-09-13T14:39:27.226 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_min_pg_log_entries": "250", 2024-09-13T14:39:27.226 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_min_recovery_priority": "0", 2024-09-13T14:39:27.226 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mon_heartbeat_interval": "30", 2024-09-13T14:39:27.226 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2024-09-13T14:39:27.226 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mon_report_interval": "5", 2024-09-13T14:39:27.226 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mon_report_max_in_flight": "2", 2024-09-13T14:39:27.226 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_mon_shutdown_timeout": "5.000000", 2024-09-13T14:39:27.226 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_num_cache_shards": "32", 2024-09-13T14:39:27.226 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_num_op_tracker_shard": "32", 2024-09-13T14:39:27.226 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_numa_auto_affinity": "true", 2024-09-13T14:39:27.226 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_numa_node": "-1", 2024-09-13T14:39:27.227 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_numa_prefer_iface": "true", 2024-09-13T14:39:27.227 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_object_clean_region_max_num_intervals": "10", 2024-09-13T14:39:27.227 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_objecter_finishers": "1", 2024-09-13T14:39:27.227 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_objectstore": "bluestore", 2024-09-13T14:39:27.227 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_objectstore_fuse": "false", 2024-09-13T14:39:27.227 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_objectstore_ideal_list_max": "64", 2024-09-13T14:39:27.227 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_objectstore_tracing": "false", 2024-09-13T14:39:27.227 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_complaint_time": "30.000000", 2024-09-13T14:39:27.227 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_history_duration": "600", 2024-09-13T14:39:27.227 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_history_size": "20", 2024-09-13T14:39:27.227 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_history_slow_op_size": "20", 2024-09-13T14:39:27.227 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2024-09-13T14:39:27.227 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_log_threshold": "5", 2024-09-13T14:39:27.227 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_num_shards": "0", 2024-09-13T14:39:27.228 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_num_shards_hdd": "5", 2024-09-13T14:39:27.228 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_num_shards_ssd": "8", 2024-09-13T14:39:27.228 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_num_threads_per_shard": "0", 2024-09-13T14:39:27.228 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2024-09-13T14:39:27.228 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2024-09-13T14:39:27.228 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2024-09-13T14:39:27.228 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_pq_min_cost": "65536", 2024-09-13T14:39:27.228 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_queue": "mclock_scheduler", 2024-09-13T14:39:27.228 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_queue_cut_off": "high", 2024-09-13T14:39:27.228 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_thread_suicide_timeout": "150", 2024-09-13T14:39:27.228 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_op_thread_timeout": "15", 2024-09-13T14:39:27.228 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_open_classes_on_start": "true", 2024-09-13T14:39:27.228 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_os_flags": "0", 2024-09-13T14:39:27.228 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_peering_op_priority": "255", 2024-09-13T14:39:27.229 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pg_delete_cost": "1048576", 2024-09-13T14:39:27.229 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pg_delete_priority": "5", 2024-09-13T14:39:27.229 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2024-09-13T14:39:27.229 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2024-09-13T14:39:27.229 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pg_log_dups_tracked": "3000", 2024-09-13T14:39:27.229 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pg_log_trim_max": "10000", 2024-09-13T14:39:27.229 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pg_log_trim_min": "100", 2024-09-13T14:39:27.229 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2024-09-13T14:39:27.229 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pg_object_context_cache_count": "64", 2024-09-13T14:39:27.229 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pg_stat_report_interval_max_epochs": "500", 2024-09-13T14:39:27.229 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pg_stat_report_interval_max_seconds": "5", 2024-09-13T14:39:27.229 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2024-09-13T14:39:27.229 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_cache_min_evict_age": "0", 2024-09-13T14:39:27.230 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_cache_min_flush_age": "0", 2024-09-13T14:39:27.230 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2024-09-13T14:39:27.230 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2024-09-13T14:39:27.230 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2024-09-13T14:39:27.230 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_crimson": "false", 2024-09-13T14:39:27.230 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_crush_rule": "-1", 2024-09-13T14:39:27.230 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_ec_fast_read": "false", 2024-09-13T14:39:27.230 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2024-09-13T14:39:27.231 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_flag_bulk": "false", 2024-09-13T14:39:27.231 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_flag_hashpspool": "true", 2024-09-13T14:39:27.231 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_flag_nodelete": "false", 2024-09-13T14:39:27.231 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_flag_nopgchange": "false", 2024-09-13T14:39:27.231 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_flag_nosizechange": "false", 2024-09-13T14:39:27.231 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_flags": "0", 2024-09-13T14:39:27.232 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2024-09-13T14:39:27.232 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_min_size": "0", 2024-09-13T14:39:27.232 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2024-09-13T14:39:27.232 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_pg_num": "32", 2024-09-13T14:39:27.232 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_pgp_num": "0", 2024-09-13T14:39:27.232 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2024-09-13T14:39:27.232 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_read_ratio": "70", 2024-09-13T14:39:27.232 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_size": "3", 2024-09-13T14:39:27.233 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_default_type": "replicated", 2024-09-13T14:39:27.233 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2024-09-13T14:39:27.233 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_pool_use_gmt_hitset": "true", 2024-09-13T14:39:27.233 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_push_per_object_cost": "1000", 2024-09-13T14:39:27.233 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_read_ec_check_for_errors": "false", 2024-09-13T14:39:27.233 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recover_clone_overlap": "true", 2024-09-13T14:39:27.233 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recover_clone_overlap_limit": "10", 2024-09-13T14:39:27.233 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_cost": "20971520", 2024-09-13T14:39:27.233 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_delay_start": "0.000000", 2024-09-13T14:39:27.233 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_max_active": "0", 2024-09-13T14:39:27.233 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_max_active_hdd": "3", 2024-09-13T14:39:27.233 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_max_active_ssd": "10", 2024-09-13T14:39:27.233 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_max_chunk": "8388608", 2024-09-13T14:39:27.234 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2024-09-13T14:39:27.234 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_max_single_start": "1", 2024-09-13T14:39:27.234 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_op_priority": "3", 2024-09-13T14:39:27.234 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_op_warn_multiple": "16", 2024-09-13T14:39:27.234 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_priority": "5", 2024-09-13T14:39:27.234 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_retry_interval": "30.000000", 2024-09-13T14:39:27.234 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_sleep": "0.000000", 2024-09-13T14:39:27.234 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_sleep_hdd": "0.100000", 2024-09-13T14:39:27.234 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2024-09-13T14:39:27.234 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_recovery_sleep_ssd": "0.000000", 2024-09-13T14:39:27.234 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_repair_during_recovery": "false", 2024-09-13T14:39:27.234 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_requested_scrub_priority": "120", 2024-09-13T14:39:27.234 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2024-09-13T14:39:27.234 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_rollback_to_cluster_snap": "", 2024-09-13T14:39:27.235 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_auto_repair": "false", 2024-09-13T14:39:27.235 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_auto_repair_num_errors": "5", 2024-09-13T14:39:27.235 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_backoff_ratio": "0.660000", 2024-09-13T14:39:27.235 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_begin_hour": "0", 2024-09-13T14:39:27.235 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_begin_week_day": "0", 2024-09-13T14:39:27.235 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_chunk_max": "25", 2024-09-13T14:39:27.235 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_chunk_min": "5", 2024-09-13T14:39:27.235 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_cost": "52428800", 2024-09-13T14:39:27.235 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_disable_reservation_queuing": "false", 2024-09-13T14:39:27.235 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_during_recovery": "false", 2024-09-13T14:39:27.235 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_end_hour": "0", 2024-09-13T14:39:27.235 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_end_week_day": "0", 2024-09-13T14:39:27.235 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_event_cost": "4096", 2024-09-13T14:39:27.235 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_extended_sleep": "0.000000", 2024-09-13T14:39:27.236 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2024-09-13T14:39:27.236 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_invalid_stats": "true", 2024-09-13T14:39:27.236 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_load_threshold": "0.500000", 2024-09-13T14:39:27.236 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_max_interval": "604800.000000", 2024-09-13T14:39:27.236 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_max_preemptions": "5", 2024-09-13T14:39:27.236 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_min_interval": "86400.000000", 2024-09-13T14:39:27.236 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_priority": "5", 2024-09-13T14:39:27.236 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_scrub_sleep": "0.000000", 2024-09-13T14:39:27.236 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_shallow_scrub_chunk_max": "100", 2024-09-13T14:39:27.236 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_shallow_scrub_chunk_min": "50", 2024-09-13T14:39:27.236 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_shutdown_pgref_assert": "false", 2024-09-13T14:39:27.236 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_skip_check_past_interval_bounds": "false", 2024-09-13T14:39:27.236 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_skip_data_digest": "false", 2024-09-13T14:39:27.237 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_smart_report_timeout": "5", 2024-09-13T14:39:27.237 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_snap_trim_cost": "1048576", 2024-09-13T14:39:27.237 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_snap_trim_priority": "5", 2024-09-13T14:39:27.237 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_snap_trim_sleep": "0.000000", 2024-09-13T14:39:27.237 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2024-09-13T14:39:27.237 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2024-09-13T14:39:27.237 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2024-09-13T14:39:27.237 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_stats_update_period_not_scrubbing": "120", 2024-09-13T14:39:27.237 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_stats_update_period_scrubbing": "15", 2024-09-13T14:39:27.237 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2024-09-13T14:39:27.237 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_target_transaction_size": "30", 2024-09-13T14:39:27.237 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_tier_default_cache_hit_set_count": "4", 2024-09-13T14:39:27.237 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2024-09-13T14:39:27.237 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2024-09-13T14:39:27.238 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2024-09-13T14:39:27.238 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2024-09-13T14:39:27.238 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2024-09-13T14:39:27.238 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2024-09-13T14:39:27.238 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_tier_default_cache_mode": "writeback", 2024-09-13T14:39:27.238 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2024-09-13T14:39:27.238 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_tier_promote_max_objects_sec": "25", 2024-09-13T14:39:27.238 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_tracing": "false", 2024-09-13T14:39:27.238 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_use_stale_snap": "false", 2024-09-13T14:39:27.238 INFO:teuthology.orchestra.run.smithi096.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2024-09-13T14:39:27.238 INFO:teuthology.orchestra.run.smithi096.stdout: "osdc_blkin_trace_all": "false", 2024-09-13T14:39:27.238 INFO:teuthology.orchestra.run.smithi096.stdout: "paxos_kill_at": "0", 2024-09-13T14:39:27.238 INFO:teuthology.orchestra.run.smithi096.stdout: "paxos_max_join_drift": "10", 2024-09-13T14:39:27.238 INFO:teuthology.orchestra.run.smithi096.stdout: "paxos_min": "500", 2024-09-13T14:39:27.239 INFO:teuthology.orchestra.run.smithi096.stdout: "paxos_min_wait": "0.050000", 2024-09-13T14:39:27.239 INFO:teuthology.orchestra.run.smithi096.stdout: "paxos_propose_interval": "1.000000", 2024-09-13T14:39:27.239 INFO:teuthology.orchestra.run.smithi096.stdout: "paxos_service_trim_max": "500", 2024-09-13T14:39:27.239 INFO:teuthology.orchestra.run.smithi096.stdout: "paxos_service_trim_max_multiplier": "20", 2024-09-13T14:39:27.239 INFO:teuthology.orchestra.run.smithi096.stdout: "paxos_service_trim_min": "250", 2024-09-13T14:39:27.239 INFO:teuthology.orchestra.run.smithi096.stdout: "paxos_stash_full_interval": "25", 2024-09-13T14:39:27.239 INFO:teuthology.orchestra.run.smithi096.stdout: "paxos_trim_max": "500", 2024-09-13T14:39:27.239 INFO:teuthology.orchestra.run.smithi096.stdout: "paxos_trim_min": "250", 2024-09-13T14:39:27.239 INFO:teuthology.orchestra.run.smithi096.stdout: "perf": "true", 2024-09-13T14:39:27.239 INFO:teuthology.orchestra.run.smithi096.stdout: "pid_file": "", 2024-09-13T14:39:27.239 INFO:teuthology.orchestra.run.smithi096.stdout: "plugin_crypto_accelerator": "crypto_isal", 2024-09-13T14:39:27.239 INFO:teuthology.orchestra.run.smithi096.stdout: "plugin_dir": "/usr/lib64/ceph", 2024-09-13T14:39:27.239 INFO:teuthology.orchestra.run.smithi096.stdout: "public_addr": "-", 2024-09-13T14:39:27.239 INFO:teuthology.orchestra.run.smithi096.stdout: "public_addrv": "", 2024-09-13T14:39:27.240 INFO:teuthology.orchestra.run.smithi096.stdout: "public_bind_addr": "-", 2024-09-13T14:39:27.240 INFO:teuthology.orchestra.run.smithi096.stdout: "public_network": "172.21.0.0/20", 2024-09-13T14:39:27.240 INFO:teuthology.orchestra.run.smithi096.stdout: "public_network_interface": "", 2024-09-13T14:39:27.240 INFO:teuthology.orchestra.run.smithi096.stdout: "qat_compressor_busy_polling": "false", 2024-09-13T14:39:27.240 INFO:teuthology.orchestra.run.smithi096.stdout: "qat_compressor_enabled": "false", 2024-09-13T14:39:27.240 INFO:teuthology.orchestra.run.smithi096.stdout: "qat_compressor_session_max_number": "256", 2024-09-13T14:39:27.240 INFO:teuthology.orchestra.run.smithi096.stdout: "rados_mon_op_timeout": "0", 2024-09-13T14:39:27.240 INFO:teuthology.orchestra.run.smithi096.stdout: "rados_osd_op_timeout": "0", 2024-09-13T14:39:27.240 INFO:teuthology.orchestra.run.smithi096.stdout: "rados_tracing": "false", 2024-09-13T14:39:27.240 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_atime_update_interval": "60", 2024-09-13T14:39:27.240 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2024-09-13T14:39:27.240 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_balance_parent_reads": "false", 2024-09-13T14:39:27.240 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_balance_snap_reads": "false", 2024-09-13T14:39:27.240 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_blkin_trace_all": "false", 2024-09-13T14:39:27.241 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_blocklist_expire_seconds": "0", 2024-09-13T14:39:27.241 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_blocklist_on_break_lock": "true", 2024-09-13T14:39:27.241 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_cache": "true", 2024-09-13T14:39:27.241 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_cache_block_writes_upfront": "false", 2024-09-13T14:39:27.241 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_cache_max_dirty": "25165824", 2024-09-13T14:39:27.241 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_cache_max_dirty_age": "1.000000", 2024-09-13T14:39:27.241 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_cache_max_dirty_object": "0", 2024-09-13T14:39:27.241 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_cache_policy": "writearound", 2024-09-13T14:39:27.241 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_cache_size": "33554432", 2024-09-13T14:39:27.241 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_cache_target_dirty": "16777216", 2024-09-13T14:39:27.241 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_cache_writethrough_until_flush": "true", 2024-09-13T14:39:27.241 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_clone_copy_on_read": "false", 2024-09-13T14:39:27.241 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_compression_hint": "none", 2024-09-13T14:39:27.241 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_concurrent_management_ops": "10", 2024-09-13T14:39:27.242 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_config_pool_override_update_timestamp": "0", 2024-09-13T14:39:27.242 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_default_clone_format": "auto", 2024-09-13T14:39:27.242 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_default_data_pool": "", 2024-09-13T14:39:27.242 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_default_features": "61", 2024-09-13T14:39:27.242 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_default_format": "2", 2024-09-13T14:39:27.244 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_default_map_options": "", 2024-09-13T14:39:27.244 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_default_order": "22", 2024-09-13T14:39:27.244 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_default_pool": "rbd", 2024-09-13T14:39:27.244 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2024-09-13T14:39:27.244 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_default_stripe_count": "0", 2024-09-13T14:39:27.244 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_default_stripe_unit": "0", 2024-09-13T14:39:27.244 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_disable_zero_copy_writes": "true", 2024-09-13T14:39:27.244 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_discard_granularity_bytes": "65536", 2024-09-13T14:39:27.244 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_discard_on_zeroed_write_same": "true", 2024-09-13T14:39:27.244 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_enable_alloc_hint": "true", 2024-09-13T14:39:27.244 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2024-09-13T14:39:27.244 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_io_scheduler": "simple", 2024-09-13T14:39:27.245 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2024-09-13T14:39:27.245 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_journal_commit_age": "5.000000", 2024-09-13T14:39:27.245 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2024-09-13T14:39:27.245 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_journal_max_payload_bytes": "16384", 2024-09-13T14:39:27.245 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_journal_object_flush_age": "0.000000", 2024-09-13T14:39:27.245 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_journal_object_flush_bytes": "1048576", 2024-09-13T14:39:27.245 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_journal_object_flush_interval": "0", 2024-09-13T14:39:27.245 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2024-09-13T14:39:27.245 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2024-09-13T14:39:27.245 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_journal_order": "24", 2024-09-13T14:39:27.245 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_journal_pool": "", 2024-09-13T14:39:27.245 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_journal_splay_width": "4", 2024-09-13T14:39:27.245 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_localize_parent_reads": "false", 2024-09-13T14:39:27.245 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_localize_snap_reads": "false", 2024-09-13T14:39:27.246 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2024-09-13T14:39:27.246 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2024-09-13T14:39:27.246 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2024-09-13T14:39:27.246 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:27 smithi096 ceph-mon[98901]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:27.246 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:27 smithi096 ceph-mon[102928]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:27.247 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2024-09-13T14:39:27.247 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2024-09-13T14:39:27.247 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2024-09-13T14:39:27.247 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_image_policy_type": "simple", 2024-09-13T14:39:27.247 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2024-09-13T14:39:27.247 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_image_state_check_interval": "30", 2024-09-13T14:39:27.247 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2024-09-13T14:39:27.247 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2024-09-13T14:39:27.248 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2024-09-13T14:39:27.248 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2024-09-13T14:39:27.248 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2024-09-13T14:39:27.248 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_memory_autotune": "true", 2024-09-13T14:39:27.248 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_memory_base": "805306368", 2024-09-13T14:39:27.248 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2024-09-13T14:39:27.248 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_memory_cache_min": "134217728", 2024-09-13T14:39:27.248 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2024-09-13T14:39:27.248 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2024-09-13T14:39:27.248 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_memory_target": "4294967296", 2024-09-13T14:39:27.248 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_perf_stats_prio": "5", 2024-09-13T14:39:27.248 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2024-09-13T14:39:27.248 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2024-09-13T14:39:27.249 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirroring_delete_delay": "0", 2024-09-13T14:39:27.249 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2024-09-13T14:39:27.249 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirroring_replay_delay": "0", 2024-09-13T14:39:27.249 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2024-09-13T14:39:27.249 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2024-09-13T14:39:27.249 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_move_to_trash_on_remove": "false", 2024-09-13T14:39:27.249 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2024-09-13T14:39:27.249 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_mtime_update_interval": "60", 2024-09-13T14:39:27.249 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_non_blocking_aio": "true", 2024-09-13T14:39:27.249 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_op_thread_timeout": "60", 2024-09-13T14:39:27.249 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_op_threads": "1", 2024-09-13T14:39:27.249 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_parent_cache_enabled": "false", 2024-09-13T14:39:27.249 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_persistent_cache_mode": "disabled", 2024-09-13T14:39:27.249 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_persistent_cache_path": "/tmp", 2024-09-13T14:39:27.250 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_persistent_cache_size": "1073741824", 2024-09-13T14:39:27.250 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_plugins": "", 2024-09-13T14:39:27.250 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_bps_burst": "0", 2024-09-13T14:39:27.250 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_bps_burst_seconds": "1", 2024-09-13T14:39:27.250 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_bps_limit": "0", 2024-09-13T14:39:27.250 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_exclude_ops": "0", 2024-09-13T14:39:27.250 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_iops_burst": "0", 2024-09-13T14:39:27.250 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_iops_burst_seconds": "1", 2024-09-13T14:39:27.250 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_iops_limit": "0", 2024-09-13T14:39:27.250 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_read_bps_burst": "0", 2024-09-13T14:39:27.250 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2024-09-13T14:39:27.250 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_read_bps_limit": "0", 2024-09-13T14:39:27.250 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_read_iops_burst": "0", 2024-09-13T14:39:27.251 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2024-09-13T14:39:27.251 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_read_iops_limit": "0", 2024-09-13T14:39:27.251 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_schedule_tick_min": "50", 2024-09-13T14:39:27.251 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_write_bps_burst": "0", 2024-09-13T14:39:27.251 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2024-09-13T14:39:27.251 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_write_bps_limit": "0", 2024-09-13T14:39:27.251 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_write_iops_burst": "0", 2024-09-13T14:39:27.251 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2024-09-13T14:39:27.251 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_qos_write_iops_limit": "0", 2024-09-13T14:39:27.251 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_quiesce_notification_attempts": "10", 2024-09-13T14:39:27.251 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_read_from_replica_policy": "default", 2024-09-13T14:39:27.251 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2024-09-13T14:39:27.251 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_readahead_max_bytes": "524288", 2024-09-13T14:39:27.251 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_readahead_trigger_requests": "10", 2024-09-13T14:39:27.252 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_request_timed_out_seconds": "30", 2024-09-13T14:39:27.252 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_skip_partial_discard": "true", 2024-09-13T14:39:27.252 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2024-09-13T14:39:27.252 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_tracing": "false", 2024-09-13T14:39:27.252 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_validate_names": "true", 2024-09-13T14:39:27.252 INFO:teuthology.orchestra.run.smithi096.stdout: "rbd_validate_pool": "true", 2024-09-13T14:39:27.252 INFO:teuthology.orchestra.run.smithi096.stdout: "restapi_base_url": "", 2024-09-13T14:39:27.252 INFO:teuthology.orchestra.run.smithi096.stdout: "restapi_log_level": "", 2024-09-13T14:39:27.252 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_account_default_quota_max_objects": "-1", 2024-09-13T14:39:27.252 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_account_default_quota_max_size": "-1", 2024-09-13T14:39:27.252 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_acl_grants_max_num": "100", 2024-09-13T14:39:27.252 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_admin_entry": "admin", 2024-09-13T14:39:27.252 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2024-09-13T14:39:27.253 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_asio_assert_yielding": "false", 2024-09-13T14:39:27.253 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_backend_store": "rados", 2024-09-13T14:39:27.253 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_barbican_url": "", 2024-09-13T14:39:27.253 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_beast_enable_async": "true", 2024-09-13T14:39:27.253 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_bucket_counters_cache": "false", 2024-09-13T14:39:27.253 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_bucket_counters_cache_size": "10000", 2024-09-13T14:39:27.253 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2024-09-13T14:39:27.253 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_bucket_default_quota_max_size": "-1", 2024-09-13T14:39:27.253 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_bucket_index_max_aio": "128", 2024-09-13T14:39:27.253 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2024-09-13T14:39:27.253 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_bucket_quota_cache_size": "10000", 2024-09-13T14:39:27.253 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_bucket_quota_ttl": "600", 2024-09-13T14:39:27.253 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_bucket_sync_spawn_window": "20", 2024-09-13T14:39:27.253 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_cache_enabled": "true", 2024-09-13T14:39:27.254 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_cache_expiry_interval": "900", 2024-09-13T14:39:27.254 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_cache_lru_size": "25000", 2024-09-13T14:39:27.254 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_config_store": "rados", 2024-09-13T14:39:27.254 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_content_length_compat": "false", 2024-09-13T14:39:27.254 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_copy_obj_progress": "true", 2024-09-13T14:39:27.254 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2024-09-13T14:39:27.254 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_cors_rules_max_num": "100", 2024-09-13T14:39:27.254 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_cross_domain_policy": "", 2024-09-13T14:39:27.254 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_default_encryption_key": "", 2024-09-13T14:39:27.254 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_kmip_addr": "", 2024-09-13T14:39:27.254 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_kmip_ca_path": "", 2024-09-13T14:39:27.254 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_kmip_client_cert": "", 2024-09-13T14:39:27.254 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_kmip_client_key": "", 2024-09-13T14:39:27.254 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_kmip_kms_key_template": "", 2024-09-13T14:39:27.255 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_kmip_password": "", 2024-09-13T14:39:27.255 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2024-09-13T14:39:27.255 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_kmip_username": "", 2024-09-13T14:39:27.255 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_require_ssl": "true", 2024-09-13T14:39:27.255 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2024-09-13T14:39:27.255 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2024-09-13T14:39:27.255 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_sse_s3_backend": "vault", 2024-09-13T14:39:27.255 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2024-09-13T14:39:27.255 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2024-09-13T14:39:27.255 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2024-09-13T14:39:27.255 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2024-09-13T14:39:27.255 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2024-09-13T14:39:27.255 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2024-09-13T14:39:27.255 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2024-09-13T14:39:27.256 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2024-09-13T14:39:27.256 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2024-09-13T14:39:27.256 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2024-09-13T14:39:27.256 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2024-09-13T14:39:27.256 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_suppress_logs": "true", 2024-09-13T14:39:27.256 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_vault_addr": "", 2024-09-13T14:39:27.256 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_vault_auth": "token", 2024-09-13T14:39:27.256 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_vault_namespace": "", 2024-09-13T14:39:27.256 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_vault_prefix": "", 2024-09-13T14:39:27.256 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_vault_secret_engine": "transit", 2024-09-13T14:39:27.256 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_vault_ssl_cacert": "", 2024-09-13T14:39:27.256 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2024-09-13T14:39:27.256 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2024-09-13T14:39:27.256 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_vault_token_file": "", 2024-09-13T14:39:27.257 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_crypt_vault_verify_ssl": "true", 2024-09-13T14:39:27.257 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_curl_buffersize": "524288", 2024-09-13T14:39:27.257 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_curl_low_speed_limit": "1024", 2024-09-13T14:39:27.257 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_curl_low_speed_time": "300", 2024-09-13T14:39:27.257 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_curl_tcp_keepalive": "0", 2024-09-13T14:39:27.257 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_curl_wait_timeout_ms": "1000", 2024-09-13T14:39:27.257 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2024-09-13T14:39:27.257 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2024-09-13T14:39:27.257 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2024-09-13T14:39:27.257 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2024-09-13T14:39:27.257 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d3n_l1_fadvise": "4", 2024-09-13T14:39:27.257 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2024-09-13T14:39:27.257 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d3n_libaio_aio_num": "64", 2024-09-13T14:39:27.258 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d3n_libaio_aio_threads": "20", 2024-09-13T14:39:27.258 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d4n_address": "127.0.0.1:6379", 2024-09-13T14:39:27.258 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d4n_l1_datacache_address": "127.0.0.1:6379", 2024-09-13T14:39:27.258 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d4n_l1_datacache_persistent_path": "/tmp/rgw_d4n_datacache/", 2024-09-13T14:39:27.258 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d4n_l1_datacache_size": "1073741824", 2024-09-13T14:39:27.258 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d4n_l1_evict_cache_on_start": "true", 2024-09-13T14:39:27.258 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d4n_l1_fadvise": "4", 2024-09-13T14:39:27.258 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d4n_libaio_aio_num": "64", 2024-09-13T14:39:27.258 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_d4n_libaio_aio_threads": "20", 2024-09-13T14:39:27.258 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2024-09-13T14:39:27.258 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_data_log_changes_size": "1000", 2024-09-13T14:39:27.258 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_data_log_num_shards": "128", 2024-09-13T14:39:27.258 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_data_log_obj_prefix": "data_log", 2024-09-13T14:39:27.259 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_data_log_window": "30", 2024-09-13T14:39:27.259 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_data_notify_interval_msec": "0", 2024-09-13T14:39:27.259 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_data_sync_poll_interval": "20", 2024-09-13T14:39:27.259 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_data_sync_spawn_window": "20", 2024-09-13T14:39:27.259 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_debug_inject_latency_bi_unlink": "0", 2024-09-13T14:39:27.259 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2024-09-13T14:39:27.259 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_debug_inject_set_olh_err": "0", 2024-09-13T14:39:27.259 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_default_data_log_backing": "fifo", 2024-09-13T14:39:27.259 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_default_realm_info_oid": "default.realm", 2024-09-13T14:39:27.259 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_default_region_info_oid": "default.region", 2024-09-13T14:39:27.259 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_default_zone_info_oid": "default.zone", 2024-09-13T14:39:27.259 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2024-09-13T14:39:27.259 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_defer_to_bucket_acls": "", 2024-09-13T14:39:27.260 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_delete_multi_obj_max_num": "1000", 2024-09-13T14:39:27.260 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_disable_s3select": "false", 2024-09-13T14:39:27.260 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dmclock_admin_lim": "0.000000", 2024-09-13T14:39:27.260 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dmclock_admin_res": "100.000000", 2024-09-13T14:39:27.260 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2024-09-13T14:39:27.260 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dmclock_auth_lim": "0.000000", 2024-09-13T14:39:27.260 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dmclock_auth_res": "200.000000", 2024-09-13T14:39:27.260 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2024-09-13T14:39:27.260 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dmclock_data_lim": "0.000000", 2024-09-13T14:39:27.260 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dmclock_data_res": "500.000000", 2024-09-13T14:39:27.260 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dmclock_data_wgt": "500.000000", 2024-09-13T14:39:27.260 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2024-09-13T14:39:27.260 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dmclock_metadata_res": "500.000000", 2024-09-13T14:39:27.261 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2024-09-13T14:39:27.261 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dns_name": "", 2024-09-13T14:39:27.261 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dns_s3website_name": "", 2024-09-13T14:39:27.261 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dynamic_resharding": "true", 2024-09-13T14:39:27.261 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dynamic_resharding_may_reduce": "true", 2024-09-13T14:39:27.261 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_dynamic_resharding_reduction_wait": "120", 2024-09-13T14:39:27.261 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2024-09-13T14:39:27.261 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_enable_gc_threads": "true", 2024-09-13T14:39:27.261 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_enable_lc_threads": "true", 2024-09-13T14:39:27.261 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_enable_mdsearch": "true", 2024-09-13T14:39:27.261 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_enable_ops_log": "false", 2024-09-13T14:39:27.261 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_enable_quota_threads": "true", 2024-09-13T14:39:27.261 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_enable_static_website": "false", 2024-09-13T14:39:27.262 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_enable_usage_log": "false", 2024-09-13T14:39:27.262 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_enforce_swift_acls": "true", 2024-09-13T14:39:27.262 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_exit_timeout_secs": "120", 2024-09-13T14:39:27.262 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_expose_bucket": "false", 2024-09-13T14:39:27.262 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_extended_http_attrs": "", 2024-09-13T14:39:27.262 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_filter": "none", 2024-09-13T14:39:27.262 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2024-09-13T14:39:27.262 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_frontends": "beast port=7480", 2024-09-13T14:39:27.262 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_gc_max_concurrent_io": "10", 2024-09-13T14:39:27.262 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_gc_max_deferred": "50", 2024-09-13T14:39:27.262 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2024-09-13T14:39:27.262 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_gc_max_objs": "32", 2024-09-13T14:39:27.262 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_gc_max_queue_size": "134213632", 2024-09-13T14:39:27.262 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_gc_max_trim_chunk": "16", 2024-09-13T14:39:27.263 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_gc_obj_min_wait": "7200", 2024-09-13T14:39:27.263 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_gc_processor_max_time": "3600", 2024-09-13T14:39:27.263 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_gc_processor_period": "3600", 2024-09-13T14:39:27.263 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_get_obj_max_req_size": "4194304", 2024-09-13T14:39:27.263 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_get_obj_window_size": "16777216", 2024-09-13T14:39:27.263 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_healthcheck_disabling_path": "", 2024-09-13T14:39:27.263 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_ignore_get_invalid_range": "false", 2024-09-13T14:39:27.263 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_init_timeout": "300", 2024-09-13T14:39:27.263 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2024-09-13T14:39:27.263 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2024-09-13T14:39:27.263 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_kafka_connection_idle": "300", 2024-09-13T14:39:27.263 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_kafka_message_timeout": "5000", 2024-09-13T14:39:27.263 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_kafka_sleep_timeout": "10", 2024-09-13T14:39:27.263 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_accepted_admin_roles": "", 2024-09-13T14:39:27.264 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_accepted_reader_roles": "", 2024-09-13T14:39:27.264 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2024-09-13T14:39:27.264 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_admin_domain": "", 2024-09-13T14:39:27.264 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_admin_password": "", 2024-09-13T14:39:27.264 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_admin_password_path": "", 2024-09-13T14:39:27.264 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_admin_project": "", 2024-09-13T14:39:27.264 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_admin_tenant": "", 2024-09-13T14:39:27.264 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_admin_token": "", 2024-09-13T14:39:27.264 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_admin_token_path": "", 2024-09-13T14:39:27.264 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_admin_user": "", 2024-09-13T14:39:27.264 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_barbican_domain": "", 2024-09-13T14:39:27.264 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_barbican_password": "", 2024-09-13T14:39:27.265 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_barbican_project": "", 2024-09-13T14:39:27.265 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_barbican_tenant": "", 2024-09-13T14:39:27.265 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_barbican_user": "", 2024-09-13T14:39:27.265 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2024-09-13T14:39:27.265 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_implicit_tenants": "false", 2024-09-13T14:39:27.265 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2024-09-13T14:39:27.265 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_service_token_enabled": "false", 2024-09-13T14:39:27.265 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_token_cache_size": "10000", 2024-09-13T14:39:27.265 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_url": "", 2024-09-13T14:39:27.265 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_keystone_verify_ssl": "true", 2024-09-13T14:39:27.265 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_lc_debug_interval": "-1", 2024-09-13T14:39:27.265 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_lc_lock_max_time": "90", 2024-09-13T14:39:27.265 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_lc_max_objs": "32", 2024-09-13T14:39:27.265 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_lc_max_rules": "1000", 2024-09-13T14:39:27.266 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_lc_max_worker": "3", 2024-09-13T14:39:27.266 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_lc_max_wp_worker": "3", 2024-09-13T14:39:27.266 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_lc_thread_delay": "0", 2024-09-13T14:39:27.266 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2024-09-13T14:39:27.266 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_ldap_dnattr": "uid", 2024-09-13T14:39:27.266 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2024-09-13T14:39:27.266 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_ldap_searchfilter": "", 2024-09-13T14:39:27.266 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2024-09-13T14:39:27.266 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_ldap_uri": "", 2024-09-13T14:39:27.266 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_lfuda_sync_frequency": "60", 2024-09-13T14:39:27.266 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2024-09-13T14:39:27.266 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_list_bucket_min_readahead": "1000", 2024-09-13T14:39:27.266 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_list_buckets_max_chunk": "1000", 2024-09-13T14:39:27.267 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_log_http_headers": "", 2024-09-13T14:39:27.267 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_log_nonexistent_bucket": "false", 2024-09-13T14:39:27.267 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2024-09-13T14:39:27.267 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_log_object_name_utc": "false", 2024-09-13T14:39:27.267 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_lua_max_memory_per_state": "128000", 2024-09-13T14:39:27.267 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_luarocks_location": "/tmp/rgw_luarocks/mon.a", 2024-09-13T14:39:27.267 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_max_attr_name_len": "0", 2024-09-13T14:39:27.267 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_max_attr_size": "0", 2024-09-13T14:39:27.267 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_max_attrs_num_in_req": "0", 2024-09-13T14:39:27.267 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_max_chunk_size": "4194304", 2024-09-13T14:39:27.267 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_max_concurrent_requests": "1024", 2024-09-13T14:39:27.267 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2024-09-13T14:39:27.267 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_max_dynamic_shards": "1999", 2024-09-13T14:39:27.267 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_max_listing_results": "1000", 2024-09-13T14:39:27.268 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_max_notify_retries": "10", 2024-09-13T14:39:27.268 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_max_objs_per_shard": "100000", 2024-09-13T14:39:27.268 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_max_put_param_size": "1048576", 2024-09-13T14:39:27.268 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_max_put_size": "5368709120", 2024-09-13T14:39:27.268 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_max_slo_entries": "1000", 2024-09-13T14:39:27.268 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_md_log_max_shards": "64", 2024-09-13T14:39:27.268 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_md_notify_interval_msec": "200", 2024-09-13T14:39:27.268 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_meta_sync_poll_interval": "20", 2024-09-13T14:39:27.268 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_meta_sync_spawn_window": "20", 2024-09-13T14:39:27.268 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_mime_types_file": "/etc/mime.types", 2024-09-13T14:39:27.268 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_mp_lock_max_time": "600", 2024-09-13T14:39:27.268 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_multi_obj_del_max_aio": "16", 2024-09-13T14:39:27.268 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_multipart_min_part_size": "5242880", 2024-09-13T14:39:27.269 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_multipart_part_upload_limit": "10000", 2024-09-13T14:39:27.269 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_nfs_fhcache_partitions": "3", 2024-09-13T14:39:27.269 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_nfs_fhcache_size": "2017", 2024-09-13T14:39:27.269 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_nfs_frontends": "rgw-nfs", 2024-09-13T14:39:27.269 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2024-09-13T14:39:27.269 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_nfs_lru_lanes": "5", 2024-09-13T14:39:27.269 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_nfs_max_gc": "300", 2024-09-13T14:39:27.269 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_nfs_namespace_expire_secs": "300", 2024-09-13T14:39:27.269 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_nfs_run_gc_threads": "false", 2024-09-13T14:39:27.269 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_nfs_run_lc_threads": "false", 2024-09-13T14:39:27.269 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_nfs_run_quota_threads": "false", 2024-09-13T14:39:27.269 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_nfs_run_sync_thread": "false", 2024-09-13T14:39:27.269 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_nfs_s3_fast_attrs": "false", 2024-09-13T14:39:27.269 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_nfs_write_completion_interval_s": "10", 2024-09-13T14:39:27.270 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_num_async_rados_threads": "32", 2024-09-13T14:39:27.270 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_num_control_oids": "8", 2024-09-13T14:39:27.270 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_numa_node": "-1", 2024-09-13T14:39:27.270 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_obj_stripe_size": "4194304", 2024-09-13T14:39:27.270 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_obj_tombstone_cache_size": "1000", 2024-09-13T14:39:27.270 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_objexp_chunk_size": "100", 2024-09-13T14:39:27.270 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_objexp_gc_interval": "600", 2024-09-13T14:39:27.272 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_objexp_hints_num_shards": "127", 2024-09-13T14:39:27.273 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_olh_pending_timeout_sec": "3600", 2024-09-13T14:39:27.273 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_op_thread_suicide_timeout": "0", 2024-09-13T14:39:27.273 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_op_thread_timeout": "600", 2024-09-13T14:39:27.273 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_op_tracing": "false", 2024-09-13T14:39:27.273 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_opa_token": "", 2024-09-13T14:39:27.273 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_opa_url": "", 2024-09-13T14:39:27.273 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_opa_verify_ssl": "true", 2024-09-13T14:39:27.273 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_ops_log_data_backlog": "5242880", 2024-09-13T14:39:27.274 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2024-09-13T14:39:27.274 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_ops_log_rados": "false", 2024-09-13T14:39:27.274 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_ops_log_socket_path": "", 2024-09-13T14:39:27.274 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_override_bucket_index_max_shards": "0", 2024-09-13T14:39:27.274 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2024-09-13T14:39:27.274 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2024-09-13T14:39:27.274 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_period_push_interval": "2.000000", 2024-09-13T14:39:27.274 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_period_push_interval_max": "30.000000", 2024-09-13T14:39:27.274 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_period_root_pool": ".rgw.root", 2024-09-13T14:39:27.274 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_policy_reject_invalid_principals": "true", 2024-09-13T14:39:27.274 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2024-09-13T14:39:27.274 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_posix_cache_lanes": "3", 2024-09-13T14:39:27.275 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_posix_cache_lmdb_count": "3", 2024-09-13T14:39:27.275 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_posix_cache_max_buckets": "100", 2024-09-13T14:39:27.275 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_posix_cache_partitions": "3", 2024-09-13T14:39:27.275 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2024-09-13T14:39:27.275 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_print_continue": "true", 2024-09-13T14:39:27.275 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_print_prohibited_content_length": "false", 2024-09-13T14:39:27.275 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_put_obj_max_window_size": "67108864", 2024-09-13T14:39:27.275 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_put_obj_min_window_size": "16777216", 2024-09-13T14:39:27.275 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2024-09-13T14:39:27.275 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_rados_pool_recovery_priority": "5", 2024-09-13T14:39:27.275 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_rados_tracing": "false", 2024-09-13T14:39:27.275 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_realm": "", 2024-09-13T14:39:27.275 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_realm_id": "", 2024-09-13T14:39:27.275 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_realm_root_pool": ".rgw.root", 2024-09-13T14:39:27.276 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_region": "", 2024-09-13T14:39:27.276 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_region_root_pool": ".rgw.root", 2024-09-13T14:39:27.276 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_relaxed_region_enforcement": "false", 2024-09-13T14:39:27.276 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_relaxed_s3_bucket_names": "false", 2024-09-13T14:39:27.276 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2024-09-13T14:39:27.276 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_request_uri": "", 2024-09-13T14:39:27.276 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_reshard_batch_size": "64", 2024-09-13T14:39:27.276 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_reshard_bucket_lock_duration": "360", 2024-09-13T14:39:27.276 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_reshard_debug_interval": "-1", 2024-09-13T14:39:27.276 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_reshard_max_aio": "128", 2024-09-13T14:39:27.276 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_reshard_num_logs": "16", 2024-09-13T14:39:27.276 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_reshard_thread_interval": "600", 2024-09-13T14:39:27.277 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_resolve_cname": "false", 2024-09-13T14:39:27.277 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_rest_getusage_op_compat": "false", 2024-09-13T14:39:27.277 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_run_sync_thread": "true", 2024-09-13T14:39:27.277 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_s3_auth_disable_signature_url": "false", 2024-09-13T14:39:27.277 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_s3_auth_order": "sts, external, local", 2024-09-13T14:39:27.277 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_s3_auth_use_keystone": "false", 2024-09-13T14:39:27.277 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_s3_auth_use_ldap": "false", 2024-09-13T14:39:27.277 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_s3_auth_use_rados": "true", 2024-09-13T14:39:27.278 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_s3_auth_use_sts": "false", 2024-09-13T14:39:27.278 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_s3_client_max_sig_ver": "-1", 2024-09-13T14:39:27.278 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_s3_success_create_obj_status": "0", 2024-09-13T14:39:27.278 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_safe_max_objects_per_shard": "102400", 2024-09-13T14:39:27.278 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_scheduler_type": "throttler", 2024-09-13T14:39:27.278 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_script_uri": "", 2024-09-13T14:39:27.278 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_service_provider_name": "", 2024-09-13T14:39:27.278 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_shard_warning_threshold": "90.000000", 2024-09-13T14:39:27.278 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sts_client_id": "", 2024-09-13T14:39:27.278 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sts_client_secret": "", 2024-09-13T14:39:27.278 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sts_entry": "sts", 2024-09-13T14:39:27.278 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sts_key": "sts", 2024-09-13T14:39:27.278 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sts_max_session_duration": "43200", 2024-09-13T14:39:27.279 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sts_min_session_duration": "900", 2024-09-13T14:39:27.279 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sts_token_introspection_url": "", 2024-09-13T14:39:27.279 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_swift_account_in_url": "false", 2024-09-13T14:39:27.279 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_swift_auth_entry": "auth", 2024-09-13T14:39:27.279 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_swift_auth_url": "", 2024-09-13T14:39:27.279 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_swift_custom_header": "", 2024-09-13T14:39:27.279 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_swift_enforce_content_length": "false", 2024-09-13T14:39:27.279 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_swift_need_stats": "true", 2024-09-13T14:39:27.279 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_swift_tenant_name": "", 2024-09-13T14:39:27.279 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_swift_token_expiration": "86400", 2024-09-13T14:39:27.279 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_swift_url": "", 2024-09-13T14:39:27.279 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_swift_url_prefix": "swift", 2024-09-13T14:39:27.279 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_swift_versioning_enabled": "false", 2024-09-13T14:39:27.279 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2024-09-13T14:39:27.280 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2024-09-13T14:39:27.280 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sync_lease_period": "120", 2024-09-13T14:39:27.280 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2024-09-13T14:39:27.280 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sync_log_trim_interval": "1200", 2024-09-13T14:39:27.280 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sync_log_trim_max_buckets": "16", 2024-09-13T14:39:27.280 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2024-09-13T14:39:27.280 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2024-09-13T14:39:27.280 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sync_obj_etag_verify": "false", 2024-09-13T14:39:27.280 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sync_trace_history_size": "4096", 2024-09-13T14:39:27.280 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sync_trace_per_node_log_size": "32", 2024-09-13T14:39:27.280 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2024-09-13T14:39:27.280 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_thread_pool_size": "512", 2024-09-13T14:39:27.280 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_topic_persistency_max_retries": "0", 2024-09-13T14:39:27.281 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_topic_persistency_sleep_duration": "0", 2024-09-13T14:39:27.281 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_topic_persistency_time_to_live": "0", 2024-09-13T14:39:27.281 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_topic_require_publish_policy": "false", 2024-09-13T14:39:27.281 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_torrent_comment": "", 2024-09-13T14:39:27.281 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_torrent_createby": "", 2024-09-13T14:39:27.281 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_torrent_encoding": "", 2024-09-13T14:39:27.281 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_torrent_flag": "false", 2024-09-13T14:39:27.281 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_torrent_max_size": "5368709120", 2024-09-13T14:39:27.281 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_torrent_origin": "", 2024-09-13T14:39:27.281 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_torrent_sha_unit": "524288", 2024-09-13T14:39:27.281 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_torrent_tracker": "", 2024-09-13T14:39:27.281 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_trust_forwarded_https": "false", 2024-09-13T14:39:27.281 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_usage_log_flush_threshold": "1024", 2024-09-13T14:39:27.282 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_usage_log_tick_interval": "30", 2024-09-13T14:39:27.282 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_usage_max_shards": "32", 2024-09-13T14:39:27.282 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_usage_max_user_shards": "1", 2024-09-13T14:39:27.282 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_use_opa_authz": "false", 2024-09-13T14:39:27.282 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_user_counters_cache": "false", 2024-09-13T14:39:27.282 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_user_counters_cache_size": "10000", 2024-09-13T14:39:27.282 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_user_default_quota_max_objects": "-1", 2024-09-13T14:39:27.282 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_user_default_quota_max_size": "-1", 2024-09-13T14:39:27.282 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_user_max_buckets": "1000", 2024-09-13T14:39:27.282 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_user_policies_max_num": "100", 2024-09-13T14:39:27.282 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2024-09-13T14:39:27.282 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_user_quota_sync_idle_users": "false", 2024-09-13T14:39:27.282 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_user_quota_sync_interval": "86400", 2024-09-13T14:39:27.283 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_user_quota_sync_wait_time": "86400", 2024-09-13T14:39:27.283 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_user_unique_email": "true", 2024-09-13T14:39:27.283 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_verify_ssl": "true", 2024-09-13T14:39:27.283 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_website_routing_rules_max_num": "50", 2024-09-13T14:39:27.283 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_zone": "", 2024-09-13T14:39:27.283 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_zone_id": "", 2024-09-13T14:39:27.283 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_zone_root_pool": ".rgw.root", 2024-09-13T14:39:27.283 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_zonegroup": "", 2024-09-13T14:39:27.283 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_zonegroup_id": "", 2024-09-13T14:39:27.283 INFO:teuthology.orchestra.run.smithi096.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2024-09-13T14:39:27.284 INFO:teuthology.orchestra.run.smithi096.stdout: "rgwlc_auto_session_clear": "true", 2024-09-13T14:39:27.284 INFO:teuthology.orchestra.run.smithi096.stdout: "rgwlc_skip_bucket_step": "false", 2024-09-13T14:39:27.284 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_block_size": "4096", 2024-09-13T14:39:27.284 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_bloom_bits_per_key": "20", 2024-09-13T14:39:27.284 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2024-09-13T14:39:27.284 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2024-09-13T14:39:27.284 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_cache_row_ratio": "0.000000", 2024-09-13T14:39:27.284 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_cache_shard_bits": "4", 2024-09-13T14:39:27.284 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_cache_size": "536870912", 2024-09-13T14:39:27.284 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_cache_type": "binned_lru", 2024-09-13T14:39:27.284 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_cf_compact_on_deletion": "true", 2024-09-13T14:39:27.284 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2024-09-13T14:39:27.285 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2024-09-13T14:39:27.285 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_collect_compaction_stats": "false", 2024-09-13T14:39:27.285 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_collect_extended_stats": "false", 2024-09-13T14:39:27.285 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_collect_memory_stats": "false", 2024-09-13T14:39:27.285 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_delete_range_threshold": "1048576", 2024-09-13T14:39:27.285 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_index_type": "binary_search", 2024-09-13T14:39:27.285 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_log_to_ceph_log": "true", 2024-09-13T14:39:27.285 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_metadata_block_size": "4096", 2024-09-13T14:39:27.285 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_partition_filters": "false", 2024-09-13T14:39:27.285 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_perf": "false", 2024-09-13T14:39:27.285 INFO:teuthology.orchestra.run.smithi096.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2024-09-13T14:39:27.285 INFO:teuthology.orchestra.run.smithi096.stdout: "rotating_keys_bootstrap_timeout": "30", 2024-09-13T14:39:27.285 INFO:teuthology.orchestra.run.smithi096.stdout: "rotating_keys_renewal_timeout": "10", 2024-09-13T14:39:27.286 INFO:teuthology.orchestra.run.smithi096.stdout: "run_dir": "/var/run/ceph", 2024-09-13T14:39:27.286 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_block_create": "true", 2024-09-13T14:39:27.286 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_cache_lru_size": "67108864", 2024-09-13T14:39:27.286 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_cbjournal_size": "5368709120", 2024-09-13T14:39:27.286 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_data_delta_based_overwrite": "0", 2024-09-13T14:39:27.286 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_default_max_object_size": "16777216", 2024-09-13T14:39:27.286 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_default_object_metadata_reservation": "16777216", 2024-09-13T14:39:27.286 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_device_size": "53687091200", 2024-09-13T14:39:27.286 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_disable_end_to_end_data_protection": "true", 2024-09-13T14:39:27.286 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_full_integrity_check": "false", 2024-09-13T14:39:27.286 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_journal_batch_capacity": "16", 2024-09-13T14:39:27.286 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_journal_batch_flush_size": "16777216", 2024-09-13T14:39:27.286 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2024-09-13T14:39:27.287 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_journal_iodepth_limit": "5", 2024-09-13T14:39:27.287 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_main_device_type": "SSD", 2024-09-13T14:39:27.287 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_max_concurrent_transactions": "8", 2024-09-13T14:39:27.287 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_max_data_allocation_size": "32768", 2024-09-13T14:39:27.287 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2024-09-13T14:39:27.287 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2024-09-13T14:39:27.287 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2024-09-13T14:39:27.287 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_obj_data_write_amplification": "1.250000", 2024-09-13T14:39:27.287 INFO:teuthology.orchestra.run.smithi096.stdout: "seastore_segment_size": "67108864", 2024-09-13T14:39:27.288 INFO:teuthology.orchestra.run.smithi096.stdout: "set_keepcaps": "false", 2024-09-13T14:39:27.288 INFO:teuthology.orchestra.run.smithi096.stdout: "setgroup": "ceph", 2024-09-13T14:39:27.288 INFO:teuthology.orchestra.run.smithi096.stdout: "setuser": "ceph", 2024-09-13T14:39:27.288 INFO:teuthology.orchestra.run.smithi096.stdout: "setuser_match_path": "", 2024-09-13T14:39:27.288 INFO:teuthology.orchestra.run.smithi096.stdout: "target_max_misplaced_ratio": "0.050000", 2024-09-13T14:39:27.288 INFO:teuthology.orchestra.run.smithi096.stdout: "thp": "false", 2024-09-13T14:39:27.288 INFO:teuthology.orchestra.run.smithi096.stdout: "threadpool_default_timeout": "60", 2024-09-13T14:39:27.288 INFO:teuthology.orchestra.run.smithi096.stdout: "threadpool_empty_queue_max_wait": "2", 2024-09-13T14:39:27.288 INFO:teuthology.orchestra.run.smithi096.stdout: "throttler_perf_counter": "true", 2024-09-13T14:39:27.288 INFO:teuthology.orchestra.run.smithi096.stdout: "tmp_dir": "/tmp", 2024-09-13T14:39:27.288 INFO:teuthology.orchestra.run.smithi096.stdout: "tmp_file_template": "/tmp/ceph-mon.a.XXXXXX", 2024-09-13T14:39:27.288 INFO:teuthology.orchestra.run.smithi096.stdout: "uadk_compressor_enabled": "false", 2024-09-13T14:39:27.289 INFO:teuthology.orchestra.run.smithi096.stdout: "uadk_wd_sync_ctx_num": "2" 2024-09-13T14:39:27.289 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:39:27.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:27 smithi204 ceph-mon[87071]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:28.056 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:39:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:39:27] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.51.0" 2024-09-13T14:39:28.081 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2024-09-13T14:39:28.324 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:29 smithi096 ceph-mon[98901]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:29.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:29 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:29 smithi096 ceph-mon[102928]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:29 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:29.404 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:29 smithi204 ceph-mon[87071]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:29.404 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:29 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:30.254 INFO:teuthology.orchestra.run.smithi096.stdout:mon.a: {} 2024-09-13T14:39:30.254 INFO:teuthology.orchestra.run.smithi096.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T14:39:30.255 INFO:teuthology.orchestra.run.smithi096.stdout:mon.b: {} 2024-09-13T14:39:30.256 INFO:teuthology.orchestra.run.smithi096.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T14:39:30.264 INFO:teuthology.orchestra.run.smithi096.stdout:mon.c: {} 2024-09-13T14:39:30.264 INFO:teuthology.orchestra.run.smithi096.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T14:39:30.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:39:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:39:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:31.284 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:31 smithi096 ceph-mon[98901]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:31.284 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:31 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2583674697' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:39:31.284 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:31 smithi096 ceph-mon[102928]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:31.285 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:31 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2583674697' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:39:31.309 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2024-09-13T14:39:31.309 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-09-13T14:39:31.311 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:39:31.312 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:39:31.313 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-09-13T14:39:31.313 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_0 16 2024-09-13T14:39:31.314 INFO:tasks.thrashosds.thrasher:starting do_thrash 2024-09-13T14:39:31.314 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:39:31.314 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:39:31.314 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:39:31.314 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd out 6 2024-09-13T14:39:31.315 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2024-09-13T14:39:31.315 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:31.315 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2024-09-13T14:39:31.316 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:39:31.316 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2024-09-13T14:39:31.316 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:39:31.317 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2024-09-13T14:39:31.317 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:39:31.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:31.341 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:31.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:31.433 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:31 smithi204 ceph-mon[87071]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:31.433 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:31 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2583674697' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:39:31.442 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:31.465 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:31.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:31.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:39:31.566 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:31.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:31.590 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:31.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:31.691 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:31.712 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:31.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:31.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:31.814 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:31.835 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:31.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:31.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:31.937 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:31.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:31.959 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:31.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:32.060 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:32.082 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:32.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:32.184 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:32.205 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:32.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:32.294 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:32.307 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:32.324 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:32.328 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:32.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:39:32.430 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:32.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.452 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:32.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:32.553 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:32.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.577 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:32.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:32.678 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:32.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.700 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:32.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:32.801 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:32.824 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:32.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:32.926 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:32.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.951 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:32.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:33.052 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:33.074 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:33.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:33.176 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:33.197 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:33.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:33.299 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:33.314 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:33 smithi096 ceph-mon[98901]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:33.315 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:33.315 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:33.315 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:33 smithi096 ceph-mon[102928]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:33.315 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:33.315 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:33.322 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:33.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:33.424 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:33.445 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:33.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:33 smithi204 ceph-mon[87071]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:33.547 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:33.569 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:33.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:33.671 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:33.693 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:33.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:33.795 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:33.816 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:33.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:33.918 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:33.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.941 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:33.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:34.042 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:34.064 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:34.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:34.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:34.150 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:34.150 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:34.167 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:34.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:34.191 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:34.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:34.292 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:34.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:34.315 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:34.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:34.416 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:34.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:34.438 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:34.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:34.539 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:34.565 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:34.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:34.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:34.667 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:34.693 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:34.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:34.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:34.795 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:34.820 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:34.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:34.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:34.922 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:34.948 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:34.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:34.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:35.050 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:35.072 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:35.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:35.174 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:35.200 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:35.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:35.302 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:35.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:35 smithi096 ceph-mon[98901]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:35.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:35 smithi096 ceph-mon[102928]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:35.326 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:35.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:35.428 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:35.452 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:35.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:35.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:35 smithi204 ceph-mon[87071]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:35.554 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:35.579 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:35.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:35.681 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:35.703 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:35.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:35.805 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:35.826 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:35.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:35.927 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:35.950 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:35.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:35.981 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:39:35.981 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:39:35.982 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:39:35.982 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:39:35.982 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:39:35.982 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:39:35.982 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:39:35.982 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:39:35.982 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:39:35.982 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:39:35.983 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:39:35.983 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:39:35.983 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:39:35.983 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:39:35.983 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:39:35.983 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:39:36.052 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:36.076 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:36.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:36.178 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:36.200 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:36.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:36.301 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:36.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:36 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3625561722' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:36.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:36 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3625561722' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:36.328 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:36.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:36.429 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:36.451 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:36.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:36.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:36 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3625561722' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:36.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:36.554 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:36.579 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:36.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:39:36.681 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:36.707 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:36.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:36.809 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:36.835 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:36.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:36.937 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:36.959 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:36.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:37.061 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:37.087 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:37.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:37.189 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:37.210 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:37.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:37.312 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:37.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:37 smithi096 ceph-mon[98901]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:37.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:37 smithi096 ceph-mon[102928]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:37.334 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:37.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:37.435 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:37.461 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:37.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:37.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:37 smithi204 ceph-mon[87071]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:37.564 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:37.589 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:37.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:39:37.691 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:37.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.716 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:37.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:37.817 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:37.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.839 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:37.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:37.941 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:37.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.963 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:37.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:38.064 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:38.091 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:38.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:38.180 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:39:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:39:37] "GET /metrics HTTP/1.1" 200 34031 "" "Prometheus/2.51.0" 2024-09-13T14:39:38.193 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:38.214 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:38.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:38.316 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:38.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.338 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:38.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:38.440 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:38.465 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:38.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:38.568 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:38.589 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:38.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:38.691 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:38.716 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:38.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:39:38.818 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:38.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:38 smithi096 ceph-mon[98901]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:38.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:38.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:38.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:38.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:38.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:38.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:38 smithi096 ceph-mon[102928]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:38.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:38.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:38.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:38.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:38.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:38.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.840 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:38.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:38.945 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:38.967 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:38.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:39.068 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:39.090 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:39.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:39.192 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:39.213 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:39.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:38 smithi204 ceph-mon[87071]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:39.315 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:39.341 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:39.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:39.443 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:39.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.467 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:39.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:39.569 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:39.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.597 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:39.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:39.698 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:39.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.721 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:39.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:39.747 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:39:39.822 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:39.843 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:39.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:39.945 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:39.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.972 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:39.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:40.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:39 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3455248582' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:39:40.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:39 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:39:40.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:39 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3455248582' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:39:40.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:39 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:39:40.074 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:40.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.098 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:40.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:40.199 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:40.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:39 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3455248582' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:39:40.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:39 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:39:40.226 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:40.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:40.328 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:40.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.352 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:40.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:40.456 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:40.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:39:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:39:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:40.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.483 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:40.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:40.584 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:40.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.606 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:40.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:40.707 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:40.728 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:40.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:40.830 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:40.856 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:40.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:40.952 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:40 smithi096 ceph-mon[98901]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:40.952 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:40 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:40.952 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:40 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:39:40.952 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:40 smithi096 ceph-mon[98901]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T14:39:40.952 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:40 smithi096 ceph-mon[102928]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:40.952 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:40 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:40.952 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:40 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:39:40.952 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:40 smithi096 ceph-mon[102928]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T14:39:40.958 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:40.959 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:40.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.981 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:40.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:41.082 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:41.109 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:41.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:41.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:41.204 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:41 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a[98876]: 2024-09-13T14:39:41.133+0000 7f16aeda4640 -1 mon.a@0(leader).osd e106 definitely_dead 0 2024-09-13T14:39:41.210 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:41.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:40 smithi204 ceph-mon[87071]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:41.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:40 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:41.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:40 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:39:41.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:40 smithi204 ceph-mon[87071]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T14:39:41.231 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:41.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:41.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:41.333 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:41.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:41.361 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:41.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:41.462 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:41.488 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:41.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:41.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:41.590 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:41.616 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:41.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:41.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:41.718 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:41.742 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:41.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:41.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:41.756 INFO:teuthology.orchestra.run.smithi096.stderr:marked out osd.6. 2024-09-13T14:39:41.756 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'unique_pool_0' created 2024-09-13T14:39:41.843 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:41.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:41.866 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:41.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:41.929 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:39:41.967 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:41.995 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:41.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:41.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:42.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:41 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4233936385' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:39:42.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:41 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:39:42.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:41 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2899215439' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-13T14:39:42.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:41 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-13T14:39:42.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:41 smithi096 ceph-mon[98901]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-13T14:39:42.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:41 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4233936385' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:39:42.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:41 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:39:42.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:41 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2899215439' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-13T14:39:42.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:41 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-13T14:39:42.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:41 smithi096 ceph-mon[102928]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-13T14:39:42.098 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:42.122 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:42.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:42.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:42.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:41 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4233936385' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:39:42.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:41 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:39:42.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:41 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2899215439' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-13T14:39:42.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:41 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-13T14:39:42.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:41 smithi204 ceph-mon[87071]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-13T14:39:42.224 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:42.245 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:42.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:42.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:42.347 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:42.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:42.369 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:42.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:42.470 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:42.496 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:42.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:42.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:42.506 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:39:42.599 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:42.624 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:42.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:42.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:42.726 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:42.754 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:42.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:42.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:42.856 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:42.878 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:42.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:42.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:42.980 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:43.002 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:43.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:43.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:43.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:42 smithi096 ceph-mon[98901]: pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:43.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:42 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-13T14:39:43.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:42 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-13T14:39:43.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:42 smithi096 ceph-mon[98901]: osdmap e107: 8 total, 8 up, 7 in 2024-09-13T14:39:43.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:42 smithi096 ceph-mon[102928]: pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:43.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:42 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-13T14:39:43.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:42 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-13T14:39:43.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:42 smithi096 ceph-mon[102928]: osdmap e107: 8 total, 8 up, 7 in 2024-09-13T14:39:43.103 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:43.124 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:43.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:43.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:43.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:42 smithi204 ceph-mon[87071]: pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:39:43.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:42 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-13T14:39:43.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:42 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-13T14:39:43.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:42 smithi204 ceph-mon[87071]: osdmap e107: 8 total, 8 up, 7 in 2024-09-13T14:39:43.228 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:43.252 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:43.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:43.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:39:43.354 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:43.375 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:43.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:43.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:43.477 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:43.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:43.499 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:43.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:43.601 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:43.622 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:43.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:43.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:43.724 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:43.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:43.746 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:43.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:43.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:43 smithi096 ceph-mon[98901]: osdmap e108: 8 total, 8 up, 7 in 2024-09-13T14:39:43.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:43 smithi096 ceph-mon[102928]: osdmap e108: 8 total, 8 up, 7 in 2024-09-13T14:39:43.847 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:43.873 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-09-13T14:39:43.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:43.874 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:43.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:39:43.974 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:43.982 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:39:43.982 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:39:44.000 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:44.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:44.102 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:44.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.124 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:44.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:44.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:43 smithi204 ceph-mon[87071]: osdmap e108: 8 total, 8 up, 7 in 2024-09-13T14:39:44.226 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:44.247 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:44.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:44.348 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:44.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.370 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:44.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:44.471 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:44.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.493 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:44.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:44.551 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:39:44.552 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:44.595 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:44.616 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:44.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:44.717 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:44.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.741 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:44.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:44.811 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:44 smithi096 ceph-mon[102928]: pgmap v243: 89 pgs: 1 creating+peering, 4 peering, 7 unknown, 77 active+clean; 4.6 MiB data, 1000 MiB used, 625 GiB / 626 GiB avail; 790 B/s, 1 objects/s recovering 2024-09-13T14:39:44.811 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:44 smithi096 ceph-mon[102928]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:39:44.811 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:44 smithi096 ceph-mon[102928]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:39:44.811 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:44 smithi096 ceph-mon[102928]: osdmap e109: 8 total, 8 up, 7 in 2024-09-13T14:39:44.811 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:44 smithi096 ceph-mon[102928]: osdmap e110: 8 total, 8 up, 7 in 2024-09-13T14:39:44.811 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:44 smithi096 ceph-mon[98901]: pgmap v243: 89 pgs: 1 creating+peering, 4 peering, 7 unknown, 77 active+clean; 4.6 MiB data, 1000 MiB used, 625 GiB / 626 GiB avail; 790 B/s, 1 objects/s recovering 2024-09-13T14:39:44.811 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:44 smithi096 ceph-mon[98901]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:39:44.811 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:44 smithi096 ceph-mon[98901]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:39:44.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:44 smithi096 ceph-mon[98901]: osdmap e109: 8 total, 8 up, 7 in 2024-09-13T14:39:44.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:44 smithi096 ceph-mon[98901]: osdmap e110: 8 total, 8 up, 7 in 2024-09-13T14:39:44.842 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:44.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.863 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:44.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:44.965 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:44.988 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:44.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:45.089 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:45.116 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:45.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:45.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:45.217 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:45.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:44 smithi204 ceph-mon[87071]: pgmap v243: 89 pgs: 1 creating+peering, 4 peering, 7 unknown, 77 active+clean; 4.6 MiB data, 1000 MiB used, 625 GiB / 626 GiB avail; 790 B/s, 1 objects/s recovering 2024-09-13T14:39:45.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:44 smithi204 ceph-mon[87071]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:39:45.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:44 smithi204 ceph-mon[87071]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:39:45.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:44 smithi204 ceph-mon[87071]: osdmap e109: 8 total, 8 up, 7 in 2024-09-13T14:39:45.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:44 smithi204 ceph-mon[87071]: osdmap e110: 8 total, 8 up, 7 in 2024-09-13T14:39:45.243 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:45.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:45.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:45.252 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:45.345 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:45.366 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:45.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:45.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:45.468 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:45.494 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:45.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:45.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:45.596 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:45.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:45.618 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:45.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:45.719 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:45.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:45.741 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:45.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:45.843 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:45.865 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:45.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:45.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:45.967 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:45.994 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:45.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:45.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:39:46.095 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:46.118 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:46.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:46.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:46.219 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:46.242 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:46.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:46.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:46.243 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:46.248 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:45 smithi096 ceph-mon[98901]: osdmap e111: 8 total, 8 up, 7 in 2024-09-13T14:39:46.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:45 smithi096 ceph-mon[102928]: osdmap e111: 8 total, 8 up, 7 in 2024-09-13T14:39:46.344 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:46.365 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:46.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:46.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:46.467 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:46.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:45 smithi204 ceph-mon[87071]: osdmap e111: 8 total, 8 up, 7 in 2024-09-13T14:39:46.488 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:46.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:46.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:46.592 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:46.613 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:46.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:46.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:46.715 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:46.741 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:46.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:46.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:46.778 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:39:46.779 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:39:46.791 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:39:46.791 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:39:46.807 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:39:46.807 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:39:46.823 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:39:46.823 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:39:46.839 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:39:46.839 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:39:46.843 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:46.855 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:39:46.855 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:39:46.868 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:46.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:46.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:46.870 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:39:46.870 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:39:46.886 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:39:46.886 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:39:46.970 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:46.995 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:46.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:46.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:47.097 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:47.119 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:47.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:47.220 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:47.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.249 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:47.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:47.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:46 smithi096 ceph-mon[98901]: pgmap v247: 89 pgs: 1 creating+peering, 4 peering, 7 unknown, 77 active+clean; 4.6 MiB data, 1000 MiB used, 625 GiB / 626 GiB avail; 1.2 KiB/s, 2 objects/s recovering 2024-09-13T14:39:47.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:46 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1370612598' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:47.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:46 smithi096 ceph-mon[102928]: pgmap v247: 89 pgs: 1 creating+peering, 4 peering, 7 unknown, 77 active+clean; 4.6 MiB data, 1000 MiB used, 625 GiB / 626 GiB avail; 1.2 KiB/s, 2 objects/s recovering 2024-09-13T14:39:47.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:46 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1370612598' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:47.350 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:47.372 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:47.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:47.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:46 smithi204 ceph-mon[87071]: pgmap v247: 89 pgs: 1 creating+peering, 4 peering, 7 unknown, 77 active+clean; 4.6 MiB data, 1000 MiB used, 625 GiB / 626 GiB avail; 1.2 KiB/s, 2 objects/s recovering 2024-09-13T14:39:47.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:46 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1370612598' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:47.473 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:47.499 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:47.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:47.601 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:47.624 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:47.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:47.726 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:47.748 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:47.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:47.850 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:47.872 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:47.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:47.974 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:47.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.998 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:47.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:48.099 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:48.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:48.121 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:48.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:48.151 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:39:48.152 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:39:48.203 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:48 smithi096 ceph-mon[98901]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:39:48.204 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:48 smithi096 ceph-mon[102928]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:39:48.204 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:39:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:39:47] "GET /metrics HTTP/1.1" 200 34031 "" "Prometheus/2.51.0" 2024-09-13T14:39:48.222 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:48.249 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:48.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:48.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:48.351 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:48.376 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:48.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:48.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:48.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:48 smithi204 ceph-mon[87071]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:39:48.478 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:48.504 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:48.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:48.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:48.606 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:48.632 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:48.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:48.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:48.734 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:48.758 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:48.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:48.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:48.860 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:48.885 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:48.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:48.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:48.987 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:49.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.011 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:49.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:49.112 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:49.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.134 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:49.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:49.235 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:49.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.261 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:49.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:49.292 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:49 smithi096 ceph-mon[98901]: pgmap v248: 89 pgs: 89 active+clean; 4.6 MiB data, 1001 MiB used, 625 GiB / 626 GiB avail; 960 KiB/s, 8 objects/s recovering 2024-09-13T14:39:49.292 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:49 smithi096 ceph-mon[102928]: pgmap v248: 89 pgs: 89 active+clean; 4.6 MiB data, 1001 MiB used, 625 GiB / 626 GiB avail; 960 KiB/s, 8 objects/s recovering 2024-09-13T14:39:49.362 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:49.383 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:49.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:49 smithi204 ceph-mon[87071]: pgmap v248: 89 pgs: 89 active+clean; 4.6 MiB data, 1001 MiB used, 625 GiB / 626 GiB avail; 960 KiB/s, 8 objects/s recovering 2024-09-13T14:39:49.485 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:49.510 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:49.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:49.612 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:49.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.635 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:49.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:49.736 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:49.758 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:49.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:49.861 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:49.887 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:49.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:49.988 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:50.009 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:50.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:50.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:50.062 INFO:teuthology.orchestra.run.smithi096.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-09-13T14:39:50.111 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:50.135 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:50.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:50.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:50.206 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:39:50.229 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:50 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/984756399' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:50.229 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:50 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:50.229 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:50 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/984756399' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:50.229 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:50 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:50.237 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:50.258 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:50.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:50.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:50.360 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:50.369 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:50 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/984756399' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:50.369 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:50 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:50.369 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:39:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:39:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:50.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:50.387 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:50.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:39:50.488 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:50.509 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:50.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:50.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:50.589 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:39:50.589 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:39:50.589 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:39:50.589 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:39:50.589 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:39:50.611 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:50.633 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:50.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:50.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:50.735 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:50.756 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:50.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:50.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:50.858 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:50.880 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:50.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:50.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:50.982 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:51.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.006 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:51.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:39:51.107 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:51.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.130 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:51.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:51.161 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:51 smithi096 ceph-mon[102928]: pgmap v249: 89 pgs: 89 active+clean; 4.6 MiB data, 1005 MiB used, 625 GiB / 626 GiB avail; 779 KiB/s, 5 objects/s recovering 2024-09-13T14:39:51.161 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:51 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:51.161 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:51 smithi096 ceph-mon[102928]: osdmap e112: 8 total, 8 up, 7 in 2024-09-13T14:39:51.161 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:51.161 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:51.161 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:51 smithi096 ceph-mon[98901]: pgmap v249: 89 pgs: 89 active+clean; 4.6 MiB data, 1005 MiB used, 625 GiB / 626 GiB avail; 779 KiB/s, 5 objects/s recovering 2024-09-13T14:39:51.161 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:51 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:51.161 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:51 smithi096 ceph-mon[98901]: osdmap e112: 8 total, 8 up, 7 in 2024-09-13T14:39:51.162 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:51.162 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:51.187 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:39:51.230 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:51.254 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:51.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:51.356 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:51.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.379 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:51.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:51 smithi204 ceph-mon[87071]: pgmap v249: 89 pgs: 89 active+clean; 4.6 MiB data, 1005 MiB used, 625 GiB / 626 GiB avail; 779 KiB/s, 5 objects/s recovering 2024-09-13T14:39:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:51 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:51 smithi204 ceph-mon[87071]: osdmap e112: 8 total, 8 up, 7 in 2024-09-13T14:39:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:51.480 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:51.501 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:51.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:51.604 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:51.624 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:51.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:51.726 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:51.747 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:51.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:51.850 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:51.871 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:51.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:51.973 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:51.994 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:51.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:52.096 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:52.121 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:52.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:52.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:52.223 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:52.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:52.245 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:52.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:52.309 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:52.318 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:52 smithi096 ceph-mon[98901]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:39:52.319 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:52 smithi096 ceph-mon[102928]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:39:52.346 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:52.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:52.368 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:52.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:52.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:52 smithi204 ceph-mon[87071]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:39:52.469 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:52.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:52.492 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:52.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:52.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:52.615 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:52.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:52.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:52.654 DEBUG:teuthology.orchestra.run.smithi096:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-09-13T14:39:52.655 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-13T14:39:52.655 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:39:52.658 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:39:52.658 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:39:52.706 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:hints = 1 2024-09-13T14:39:52.706 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:39:52.707 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Object prefix: benchmark_data_smithi096_138841 2024-09-13T14:39:52.707 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:39:52.707 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:39:52.717 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:52.761 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:52.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:52.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:52.863 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:52.892 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:52.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:52.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:52.994 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:53.036 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:53.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:53.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:53.138 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:53.168 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:53.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:53.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:53.270 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:53.306 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:53.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:53.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:53.408 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:53.421 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:53 smithi096 ceph-mon[98901]: pgmap v251: 89 pgs: 89 active+clean; 4.6 MiB data, 1005 MiB used, 625 GiB / 626 GiB avail; 611 KiB/s, 4 objects/s recovering 2024-09-13T14:39:53.421 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:53 smithi096 ceph-mon[102928]: pgmap v251: 89 pgs: 89 active+clean; 4.6 MiB data, 1005 MiB used, 625 GiB / 626 GiB avail; 611 KiB/s, 4 objects/s recovering 2024-09-13T14:39:53.447 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:53.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:53.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:53.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:53 smithi204 ceph-mon[87071]: pgmap v251: 89 pgs: 89 active+clean; 4.6 MiB data, 1005 MiB used, 625 GiB / 626 GiB avail; 611 KiB/s, 4 objects/s recovering 2024-09-13T14:39:53.549 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:53.591 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:53.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:53.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:53.693 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:53.706 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 1 16 3656 3640 227.485 227.5 0.00414575 0.00437944 2024-09-13T14:39:53.722 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:53.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:53.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:53.824 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:53.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:53.861 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:53.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:53.962 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:54.006 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:54.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:54.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:54.108 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:54.132 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:39:54.178 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:54.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:54.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:54.295 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:54.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:54.336 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:54.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:54.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:54 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/808196474' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:54.428 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:54 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/808196474' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:54.437 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:54.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:54 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/808196474' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:54.474 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:54.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:54.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:54.502 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:39:54.502 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:39:54.541 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:39:54.541 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:39:54.576 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:54.577 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:39:54.577 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:39:54.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:54.614 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:54.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:54.616 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:39:54.616 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:39:54.649 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:39:54.650 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:39:54.681 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:39:54.681 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:39:54.706 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 2 16 6971 6955 217.327 207.188 0.00463792 0.0045883 2024-09-13T14:39:54.712 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:39:54.713 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:39:54.715 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:54.747 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:39:54.747 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:39:54.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:54.748 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:54.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:54.849 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:54.878 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:54.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:54.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:54.980 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:55.017 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:55.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:55.119 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:55.154 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:55.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:55.256 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:55.288 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:55.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:39:55.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:55 smithi096 ceph-mon[98901]: pgmap v252: 89 pgs: 89 active+clean; 79 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 8.6 MiB/s wr, 137 op/s; 541 KiB/s, 4 objects/s recovering 2024-09-13T14:39:55.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:55 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:55.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:55 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/808196474' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:39:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:55 smithi096 ceph-mon[98901]: osdmap e113: 8 total, 8 up, 7 in 2024-09-13T14:39:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:55 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3892766013' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:55.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:55 smithi096 ceph-mon[102928]: pgmap v252: 89 pgs: 89 active+clean; 79 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 8.6 MiB/s wr, 137 op/s; 541 KiB/s, 4 objects/s recovering 2024-09-13T14:39:55.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:55 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:55.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:55 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/808196474' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:39:55.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:55 smithi096 ceph-mon[102928]: osdmap e113: 8 total, 8 up, 7 in 2024-09-13T14:39:55.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:55 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3892766013' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:55.391 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:55.423 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:55.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:39:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:55 smithi204 ceph-mon[87071]: pgmap v252: 89 pgs: 89 active+clean; 79 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 8.6 MiB/s wr, 137 op/s; 541 KiB/s, 4 objects/s recovering 2024-09-13T14:39:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:55 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:55 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/808196474' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:39:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:55 smithi204 ceph-mon[87071]: osdmap e113: 8 total, 8 up, 7 in 2024-09-13T14:39:55.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:55 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3892766013' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:55.525 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:55.555 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:55.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:55.658 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:55.694 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:55.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:55.706 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 3 15 10170 10155 211.548 200 0.00322179 0.00472072 2024-09-13T14:39:55.797 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:55.828 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:55.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:55.911 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:55.930 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:55.970 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:55.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:56.071 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:56.102 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:56.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:56.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:56.204 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:56.240 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:56.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:56.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:56.315 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:56.342 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:56.383 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:56.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:56.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:56.485 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:56.516 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:56.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:56.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:56.618 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:56.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:56.666 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:56.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:56.706 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 4 16 13498 13482 210.64 207.938 0.00342402 0.00474133 2024-09-13T14:39:56.767 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:56.798 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:56.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:56.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:39:56.878 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:56 smithi096 ceph-mon[98901]: pgmap v254: 89 pgs: 89 active+clean; 79 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 9.3 MiB/s wr, 148 op/s 2024-09-13T14:39:56.878 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:56 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:56.878 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:56 smithi096 ceph-mon[102928]: pgmap v254: 89 pgs: 89 active+clean; 79 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 9.3 MiB/s wr, 148 op/s 2024-09-13T14:39:56.900 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:56.929 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:56.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:56.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:57.031 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:57.060 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:57.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:57.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:57.132 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:56 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:57.162 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:57.197 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:57.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:57.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:57.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:56 smithi204 ceph-mon[87071]: pgmap v254: 89 pgs: 89 active+clean; 79 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 9.3 MiB/s wr, 148 op/s 2024-09-13T14:39:57.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:56 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:39:57.299 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:57.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:57.333 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:57.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:57.434 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:57.473 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:57.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:57.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:57.563 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:39:57.574 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:57.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:57.654 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:57.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:57.707 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 5 16 16658 16642 207.996 197.5 0.00528377 0.00480115 2024-09-13T14:39:57.755 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:57.794 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:57.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:57.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:57.896 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:57.929 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:57.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:57.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:39:58.031 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:58.059 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:58.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:58.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:58.161 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:58.196 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:58.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:58.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:58.234 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:39:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:39:57] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.51.0" 2024-09-13T14:39:58.297 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:58.326 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:58.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:58.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:58.357 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:39:58.428 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:58.468 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:58.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:58.571 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:58.598 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:58.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:58.700 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:58.707 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 6 16 19411 19395 202.002 172.062 0.00509905 0.00494433 2024-09-13T14:39:58.751 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:58.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:58.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:58.853 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:58.880 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:58.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:58.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:58.982 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:59.013 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:59.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:59.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:39:59.115 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:59.151 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:59.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:59.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:59.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:39:58 smithi204 ceph-mon[87071]: pgmap v255: 89 pgs: 89 active+clean; 491 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 61 MiB/s wr, 971 op/s 2024-09-13T14:39:59.253 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:59.272 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:39:58 smithi096 ceph-mon[98901]: pgmap v255: 89 pgs: 89 active+clean; 491 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 61 MiB/s wr, 971 op/s 2024-09-13T14:39:59.273 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:39:58 smithi096 ceph-mon[102928]: pgmap v255: 89 pgs: 89 active+clean; 491 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 61 MiB/s wr, 971 op/s 2024-09-13T14:39:59.287 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:59.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:59.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:59.389 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:59.422 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:59.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:59.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:59.524 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:59.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:59.560 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:39:59.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:59.661 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:59.687 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:59.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:59.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:59.707 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 7 16 21856 21840 194.973 152.812 0.00507649 0.0051238 2024-09-13T14:39:59.789 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:59.819 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:59.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:59.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:59.921 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:39:59.951 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:39:59.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:59.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:00.053 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:00.089 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:00.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:00.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:00.191 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:00.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:00.216 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:00.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:00.317 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:00.350 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:00.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:00.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:00.451 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:00.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:40:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:40:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:00.482 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:00.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:00.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:00.584 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:00.617 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:00.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:00.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:00.707 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 8 16 24300 24284 189.694 152.75 0.00409764 0.00526609 2024-09-13T14:40:00.718 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:00.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:00.763 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:00.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:00.864 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:00.906 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:00.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:00.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:01.008 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:01.039 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:01.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:01.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:01.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:00 smithi096 ceph-mon[98901]: pgmap v256: 89 pgs: 89 active+clean; 990 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:40:01.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:00 smithi096 ceph-mon[98901]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T14:40:01.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:00 smithi096 ceph-mon[98901]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T14:40:01.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:00 smithi096 ceph-mon[102928]: pgmap v256: 89 pgs: 89 active+clean; 990 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:40:01.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:00 smithi096 ceph-mon[102928]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T14:40:01.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:00 smithi096 ceph-mon[102928]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T14:40:01.140 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:01.162 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:01.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:01.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:01.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:00 smithi204 ceph-mon[87071]: pgmap v256: 89 pgs: 89 active+clean; 990 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:40:01.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:00 smithi204 ceph-mon[87071]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T14:40:01.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:00 smithi204 ceph-mon[87071]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T14:40:01.264 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:01.299 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:01.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:01.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:01.401 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:01.450 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:01.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:01.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:01.552 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:01.580 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:01.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:01.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:01.682 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:01.709 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 9 16 26213 26197 181.893 119.562 0.00607989 0.00549101 2024-09-13T14:40:01.724 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:01.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:01.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:01.829 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:01.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:02.397 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:02.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:02.397 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:40:02.397 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:40:02.498 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:02.524 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:02.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:02.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:02.626 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:02.651 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:02.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:02.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:02.684 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:40:02.689 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:40:02.708 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 10 16 28229 28213 176.303 126 0.00420194 0.00566541 2024-09-13T14:40:02.753 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:02.785 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:02.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:02.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:02.887 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:02.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:02.929 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:02.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:03.027 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:02 smithi096 ceph-mon[98901]: pgmap v257: 89 pgs: 89 active+clean; 1.3 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-09-13T14:40:03.027 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:02 smithi096 ceph-mon[102928]: pgmap v257: 89 pgs: 89 active+clean; 1.3 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-09-13T14:40:03.030 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:03.058 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:03.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:03.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:03.070 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:40:03.161 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:03.204 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:03.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:03.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:03.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:02 smithi204 ceph-mon[87071]: pgmap v257: 89 pgs: 89 active+clean; 1.3 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-09-13T14:40:03.305 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:03.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:03.366 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:03.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:03.467 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:03.505 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:03.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:03.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:03.607 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:03.641 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:03.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:03.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:03.708 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 11 16 30541 30525 173.411 144.5 0.0146116 0.00576054 2024-09-13T14:40:03.722 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":257,"stamp":"2024-09-13T14:40:01.628111+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1408702413,"num_objects":21450,"num_object_clones":0,"num_object_copies":64350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21450,"num_whiteouts":0,"num_read":6304,"num_read_kb":13770,"num_write":22478,"num_write_kb":1385259,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":53,"num_bytes_recovered":5384707,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":22554,"ondisk_log_size":22554,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5053684,"kb_used_data":4030652,"kb_used_omap":34,"kb_used_meta":1022877,"kb_avail":651162380,"statfs":{"total":671965249536,"available":666790277120,"internally_reserved":0,"allocated":4127387648,"data_stored":4123043294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35126,"internal_metadata":1047426762},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1403912192,"num_objects":21422,"num_object_clones":0,"num_object_copies":64266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":21422,"num_write_kb":1371008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002536"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":89,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.143255+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:39:54.143255+0000","last_peered":"2024-09-13T14:39:54.143255+0000","last_clean":"2024-09-13T14:39:54.143255+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:39:54.143255+0000","last_undegraded":"2024-09-13T14:39:54.143255+0000","last_fullsized":"2024-09-13T14:39:54.143255+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":79,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.061358+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:39:50.061358+0000","last_peered":"2024-09-13T14:39:50.061358+0000","last_clean":"2024-09-13T14:39:50.061358+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:39:50.061358+0000","last_undegraded":"2024-09-13T14:39:50.061358+0000","last_fullsized":"2024-09-13T14:39:50.061358+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":77,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.059060+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:39:50.059060+0000","last_peered":"2024-09-13T14:39:50.059060+0000","last_clean":"2024-09-13T14:39:50.059060+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:39:50.059060+0000","last_undegraded":"2024-09-13T14:39:50.059060+0000","last_fullsized":"2024-09-13T14:39:50.059060+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":82,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.143409+0000","last_change":"2024-09-13T14:38:13.977954+0000","last_active":"2024-09-13T14:39:54.143409+0000","last_peered":"2024-09-13T14:39:54.143409+0000","last_clean":"2024-09-13T14:39:54.143409+0000","last_became_active":"2024-09-13T14:38:13.977619+0000","last_became_peered":"2024-09-13T14:38:13.977619+0000","last_unstale":"2024-09-13T14:39:54.143409+0000","last_undegraded":"2024-09-13T14:39:54.143409+0000","last_fullsized":"2024-09-13T14:39:54.143409+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:58:30.259967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.143210+0000","last_change":"2024-09-13T14:38:13.978232+0000","last_active":"2024-09-13T14:39:54.143210+0000","last_peered":"2024-09-13T14:39:54.143210+0000","last_clean":"2024-09-13T14:39:54.143210+0000","last_became_active":"2024-09-13T14:38:13.977921+0000","last_became_peered":"2024-09-13T14:38:13.977921+0000","last_unstale":"2024-09-13T14:39:54.143210+0000","last_undegraded":"2024-09-13T14:39:54.143210+0000","last_fullsized":"2024-09-13T14:39:54.143210+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:10.127626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":70,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.061453+0000","last_change":"2024-09-13T14:39:42.765301+0000","last_active":"2024-09-13T14:39:50.061453+0000","last_peered":"2024-09-13T14:39:50.061453+0000","last_clean":"2024-09-13T14:39:50.061453+0000","last_became_active":"2024-09-13T14:39:42.764977+0000","last_became_peered":"2024-09-13T14:39:42.764977+0000","last_unstale":"2024-09-13T14:39:50.061453+0000","last_undegraded":"2024-09-13T14:39:50.061453+0000","last_fullsized":"2024-09-13T14:39:50.061453+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:25.223972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":69,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.147889+0000","last_change":"2024-09-13T14:37:40.925691+0000","last_active":"2024-09-13T14:39:54.147889+0000","last_peered":"2024-09-13T14:39:54.147889+0000","last_clean":"2024-09-13T14:39:54.147889+0000","last_became_active":"2024-09-13T14:37:40.925360+0000","last_became_peered":"2024-09-13T14:37:40.925360+0000","last_unstale":"2024-09-13T14:39:54.147889+0000","last_undegraded":"2024-09-13T14:39:54.147889+0000","last_fullsized":"2024-09-13T14:39:54.147889+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:07.337967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":44,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.142916+0000","last_change":"2024-09-13T14:39:42.781872+0000","last_active":"2024-09-13T14:39:54.142916+0000","last_peered":"2024-09-13T14:39:54.142916+0000","last_clean":"2024-09-13T14:39:54.142916+0000","last_became_active":"2024-09-13T14:39:42.770254+0000","last_became_peered":"2024-09-13T14:39:42.770254+0000","last_unstale":"2024-09-13T14:39:54.142916+0000","last_undegraded":"2024-09-13T14:39:54.142916+0000","last_fullsized":"2024-09-13T14:39:54.142916+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:31.211081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.9","version":"113'1619","reported_seq":1633,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:40:01.551556+0000","last_change":"2024-09-13T14:39:42.764535+0000","last_active":"2024-09-13T14:40:01.551556+0000","last_peered":"2024-09-13T14:40:01.551556+0000","last_clean":"2024-09-13T14:40:01.551556+0000","last_became_active":"2024-09-13T14:39:42.764101+0000","last_became_peered":"2024-09-13T14:39:42.764101+0000","last_unstale":"2024-09-13T14:40:01.551556+0000","last_undegraded":"2024-09-13T14:40:01.551556+0000","last_fullsized":"2024-09-13T14:40:01.551556+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":1619,"log_dups_size":0,"ondisk_log_size":1619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:22.498969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106102784,"num_objects":1619,"num_object_clones":0,"num_object_copies":4857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1619,"num_write_kb":103616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":74,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.146496+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:39:54.146496+0000","last_peered":"2024-09-13T14:39:54.146496+0000","last_clean":"2024-09-13T14:39:54.146496+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:39:54.146496+0000","last_undegraded":"2024-09-13T14:39:54.146496+0000","last_fullsized":"2024-09-13T14:39:54.146496+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":70,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.058687+0000","last_change":"2024-09-13T14:39:42.764876+0000","last_active":"2024-09-13T14:39:50.058687+0000","last_peered":"2024-09-13T14:39:50.058687+0000","last_clean":"2024-09-13T14:39:50.058687+0000","last_became_active":"2024-09-13T14:39:42.764082+0000","last_became_peered":"2024-09-13T14:39:42.764082+0000","last_unstale":"2024-09-13T14:39:50.058687+0000","last_undegraded":"2024-09-13T14:39:50.058687+0000","last_fullsized":"2024-09-13T14:39:50.058687+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:10:07.458952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"113'1123","reported_seq":1146,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:58.525069+0000","last_change":"2024-09-13T14:39:42.767715+0000","last_active":"2024-09-13T14:39:58.525069+0000","last_peered":"2024-09-13T14:39:58.525069+0000","last_clean":"2024-09-13T14:39:58.525069+0000","last_became_active":"2024-09-13T14:39:42.766678+0000","last_became_peered":"2024-09-13T14:39:42.766678+0000","last_unstale":"2024-09-13T14:39:58.525069+0000","last_undegraded":"2024-09-13T14:39:58.525069+0000","last_fullsized":"2024-09-13T14:39:58.525069+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":1123,"log_dups_size":0,"ondisk_log_size":1123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:30:38.645181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73596928,"num_objects":1123,"num_object_clones":0,"num_object_copies":3369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1123,"num_write_kb":71872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":79,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.061745+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:39:50.061745+0000","last_peered":"2024-09-13T14:39:50.061745+0000","last_clean":"2024-09-13T14:39:50.061745+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:39:50.061745+0000","last_undegraded":"2024-09-13T14:39:50.061745+0000","last_fullsized":"2024-09-13T14:39:50.061745+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":104,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.171486+0000","last_change":"2024-09-13T14:38:13.979677+0000","last_active":"2024-09-13T14:39:50.171486+0000","last_peered":"2024-09-13T14:39:50.171486+0000","last_clean":"2024-09-13T14:39:50.171486+0000","last_became_active":"2024-09-13T14:38:13.979323+0000","last_became_peered":"2024-09-13T14:38:13.979323+0000","last_unstale":"2024-09-13T14:39:50.171486+0000","last_undegraded":"2024-09-13T14:39:50.171486+0000","last_fullsized":"2024-09-13T14:39:50.171486+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:29:03.491254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"113'1214","reported_seq":1237,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:58.522733+0000","last_change":"2024-09-13T14:39:42.766942+0000","last_active":"2024-09-13T14:39:58.522733+0000","last_peered":"2024-09-13T14:39:58.522733+0000","last_clean":"2024-09-13T14:39:58.522733+0000","last_became_active":"2024-09-13T14:39:42.766647+0000","last_became_peered":"2024-09-13T14:39:42.766647+0000","last_unstale":"2024-09-13T14:39:58.522733+0000","last_undegraded":"2024-09-13T14:39:58.522733+0000","last_fullsized":"2024-09-13T14:39:58.522733+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":1214,"log_dups_size":0,"ondisk_log_size":1214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:16.267901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79560704,"num_objects":1214,"num_object_clones":0,"num_object_copies":3642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1214,"num_write_kb":77696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":75,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.061692+0000","last_change":"2024-09-13T14:39:42.774547+0000","last_active":"2024-09-13T14:39:50.061692+0000","last_peered":"2024-09-13T14:39:50.061692+0000","last_clean":"2024-09-13T14:39:50.061692+0000","last_became_active":"2024-09-13T14:39:42.766006+0000","last_became_peered":"2024-09-13T14:39:42.766006+0000","last_unstale":"2024-09-13T14:39:50.061692+0000","last_undegraded":"2024-09-13T14:39:50.061692+0000","last_fullsized":"2024-09-13T14:39:50.061692+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:06:29.632163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":73,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.139622+0000","last_change":"2024-09-13T14:39:42.765763+0000","last_active":"2024-09-13T14:39:54.139622+0000","last_peered":"2024-09-13T14:39:54.139622+0000","last_clean":"2024-09-13T14:39:54.139622+0000","last_became_active":"2024-09-13T14:39:42.765441+0000","last_became_peered":"2024-09-13T14:39:42.765441+0000","last_unstale":"2024-09-13T14:39:54.139622+0000","last_undegraded":"2024-09-13T14:39:54.139622+0000","last_fullsized":"2024-09-13T14:39:54.139622+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:06.483638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.a","version":"113'1220","reported_seq":1243,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:58.512408+0000","last_change":"2024-09-13T14:39:42.763571+0000","last_active":"2024-09-13T14:39:58.512408+0000","last_peered":"2024-09-13T14:39:58.512408+0000","last_clean":"2024-09-13T14:39:58.512408+0000","last_became_active":"2024-09-13T14:39:42.763270+0000","last_became_peered":"2024-09-13T14:39:42.763270+0000","last_unstale":"2024-09-13T14:39:58.512408+0000","last_undegraded":"2024-09-13T14:39:58.512408+0000","last_fullsized":"2024-09-13T14:39:58.512408+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":1220,"log_dups_size":0,"ondisk_log_size":1220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:38:09.281466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79953920,"num_objects":1220,"num_object_clones":0,"num_object_copies":3660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1220,"num_write_kb":78080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":88,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.058993+0000","last_change":"2024-09-13T14:39:42.765558+0000","last_active":"2024-09-13T14:39:50.058993+0000","last_peered":"2024-09-13T14:39:50.058993+0000","last_clean":"2024-09-13T14:39:50.058993+0000","last_became_active":"2024-09-13T14:39:42.765246+0000","last_became_peered":"2024-09-13T14:39:42.765246+0000","last_unstale":"2024-09-13T14:39:50.058993+0000","last_undegraded":"2024-09-13T14:39:50.058993+0000","last_fullsized":"2024-09-13T14:39:50.058993+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:08.640610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.144302+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:39:54.144302+0000","last_peered":"2024-09-13T14:39:54.144302+0000","last_clean":"2024-09-13T14:39:54.144302+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:39:54.144302+0000","last_undegraded":"2024-09-13T14:39:54.144302+0000","last_fullsized":"2024-09-13T14:39:54.144302+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"113'1208","reported_seq":1231,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:58.518166+0000","last_change":"2024-09-13T14:39:42.759038+0000","last_active":"2024-09-13T14:39:58.518166+0000","last_peered":"2024-09-13T14:39:58.518166+0000","last_clean":"2024-09-13T14:39:58.518166+0000","last_became_active":"2024-09-13T14:39:42.758753+0000","last_became_peered":"2024-09-13T14:39:42.758753+0000","last_unstale":"2024-09-13T14:39:58.518166+0000","last_undegraded":"2024-09-13T14:39:58.518166+0000","last_fullsized":"2024-09-13T14:39:58.518166+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":1208,"log_dups_size":0,"ondisk_log_size":1208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:14:07.557396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79167488,"num_objects":1208,"num_object_clones":0,"num_object_copies":3624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1208,"num_write_kb":77312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":86,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.143211+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:39:54.143211+0000","last_peered":"2024-09-13T14:39:54.143211+0000","last_clean":"2024-09-13T14:39:54.143211+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:39:54.143211+0000","last_undegraded":"2024-09-13T14:39:54.143211+0000","last_fullsized":"2024-09-13T14:39:54.143211+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":87,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.171786+0000","last_change":"2024-09-13T14:39:42.775584+0000","last_active":"2024-09-13T14:39:50.171786+0000","last_peered":"2024-09-13T14:39:50.171786+0000","last_clean":"2024-09-13T14:39:50.171786+0000","last_became_active":"2024-09-13T14:39:42.775254+0000","last_became_peered":"2024-09-13T14:39:42.775254+0000","last_unstale":"2024-09-13T14:39:50.171786+0000","last_undegraded":"2024-09-13T14:39:50.171786+0000","last_fullsized":"2024-09-13T14:39:50.171786+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:44.607098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"113'877","reported_seq":900,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:56.863503+0000","last_change":"2024-09-13T14:39:42.772481+0000","last_active":"2024-09-13T14:39:56.863503+0000","last_peered":"2024-09-13T14:39:56.863503+0000","last_clean":"2024-09-13T14:39:56.863503+0000","last_became_active":"2024-09-13T14:39:42.772211+0000","last_became_peered":"2024-09-13T14:39:42.772211+0000","last_unstale":"2024-09-13T14:39:56.863503+0000","last_undegraded":"2024-09-13T14:39:56.863503+0000","last_fullsized":"2024-09-13T14:39:56.863503+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":877,"log_dups_size":0,"ondisk_log_size":877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:30:16.860807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57475072,"num_objects":877,"num_object_clones":0,"num_object_copies":2631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":877,"num_write_kb":56128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":69,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.171492+0000","last_change":"2024-09-13T14:39:42.776751+0000","last_active":"2024-09-13T14:39:50.171492+0000","last_peered":"2024-09-13T14:39:50.171492+0000","last_clean":"2024-09-13T14:39:50.171492+0000","last_became_active":"2024-09-13T14:39:42.774855+0000","last_became_peered":"2024-09-13T14:39:42.774855+0000","last_unstale":"2024-09-13T14:39:50.171492+0000","last_undegraded":"2024-09-13T14:39:50.171492+0000","last_fullsized":"2024-09-13T14:39:50.171492+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:11.022013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":94,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.171823+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:39:50.171823+0000","last_peered":"2024-09-13T14:39:50.171823+0000","last_clean":"2024-09-13T14:39:50.171823+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:39:50.171823+0000","last_undegraded":"2024-09-13T14:39:50.171823+0000","last_fullsized":"2024-09-13T14:39:50.171823+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"113'1573","reported_seq":1587,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:40:01.541941+0000","last_change":"2024-09-13T14:39:42.764380+0000","last_active":"2024-09-13T14:40:01.541941+0000","last_peered":"2024-09-13T14:40:01.541941+0000","last_clean":"2024-09-13T14:40:01.541941+0000","last_became_active":"2024-09-13T14:39:42.764127+0000","last_became_peered":"2024-09-13T14:39:42.764127+0000","last_unstale":"2024-09-13T14:40:01.541941+0000","last_undegraded":"2024-09-13T14:40:01.541941+0000","last_fullsized":"2024-09-13T14:40:01.541941+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":1573,"log_dups_size":0,"ondisk_log_size":1573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:37:32.713989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103088128,"num_objects":1573,"num_object_clones":0,"num_object_copies":4719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1573,"num_write_kb":100672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.143180+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:39:54.143180+0000","last_peered":"2024-09-13T14:39:54.143180+0000","last_clean":"2024-09-13T14:39:54.143180+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:39:54.143180+0000","last_undegraded":"2024-09-13T14:39:54.143180+0000","last_fullsized":"2024-09-13T14:39:54.143180+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":87,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.171821+0000","last_change":"2024-09-13T14:39:42.776187+0000","last_active":"2024-09-13T14:39:50.171821+0000","last_peered":"2024-09-13T14:39:50.171821+0000","last_clean":"2024-09-13T14:39:50.171821+0000","last_became_active":"2024-09-13T14:39:42.775751+0000","last_became_peered":"2024-09-13T14:39:42.775751+0000","last_unstale":"2024-09-13T14:39:50.171821+0000","last_undegraded":"2024-09-13T14:39:50.171821+0000","last_fullsized":"2024-09-13T14:39:50.171821+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:35.155717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"113'1698","reported_seq":1712,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:40:01.539308+0000","last_change":"2024-09-13T14:39:42.772007+0000","last_active":"2024-09-13T14:40:01.539308+0000","last_peered":"2024-09-13T14:40:01.539308+0000","last_clean":"2024-09-13T14:40:01.539308+0000","last_became_active":"2024-09-13T14:39:42.771672+0000","last_became_peered":"2024-09-13T14:39:42.771672+0000","last_unstale":"2024-09-13T14:40:01.539308+0000","last_undegraded":"2024-09-13T14:40:01.539308+0000","last_fullsized":"2024-09-13T14:40:01.539308+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":1698,"log_dups_size":0,"ondisk_log_size":1698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:26:12.134635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111280128,"num_objects":1698,"num_object_clones":0,"num_object_copies":5094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1698,"num_write_kb":108672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":90,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.144216+0000","last_change":"2024-09-13T14:38:13.981576+0000","last_active":"2024-09-13T14:39:54.144216+0000","last_peered":"2024-09-13T14:39:54.144216+0000","last_clean":"2024-09-13T14:39:54.144216+0000","last_became_active":"2024-09-13T14:38:13.981262+0000","last_became_peered":"2024-09-13T14:38:13.981262+0000","last_unstale":"2024-09-13T14:39:54.144216+0000","last_undegraded":"2024-09-13T14:39:54.144216+0000","last_fullsized":"2024-09-13T14:39:54.144216+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:20.284250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":71,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.140584+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:39:54.140584+0000","last_peered":"2024-09-13T14:39:54.140584+0000","last_clean":"2024-09-13T14:39:54.140584+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:39:54.140584+0000","last_undegraded":"2024-09-13T14:39:54.140584+0000","last_fullsized":"2024-09-13T14:39:54.140584+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.4","version":"113'1643","reported_seq":1657,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:40:01.556807+0000","last_change":"2024-09-13T14:39:42.765514+0000","last_active":"2024-09-13T14:40:01.556807+0000","last_peered":"2024-09-13T14:40:01.556807+0000","last_clean":"2024-09-13T14:40:01.556807+0000","last_became_active":"2024-09-13T14:39:42.765209+0000","last_became_peered":"2024-09-13T14:39:42.765209+0000","last_unstale":"2024-09-13T14:40:01.556807+0000","last_undegraded":"2024-09-13T14:40:01.556807+0000","last_fullsized":"2024-09-13T14:40:01.556807+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":1643,"log_dups_size":0,"ondisk_log_size":1643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:59.785672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107675648,"num_objects":1643,"num_object_clones":0,"num_object_copies":4929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1643,"num_write_kb":105152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":210,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.061287+0000","last_change":"2024-09-13T14:39:42.784223+0000","last_active":"2024-09-13T14:39:50.061287+0000","last_peered":"2024-09-13T14:39:50.061287+0000","last_clean":"2024-09-13T14:39:50.061287+0000","last_became_active":"2024-09-13T14:39:42.771429+0000","last_became_peered":"2024-09-13T14:39:42.771429+0000","last_unstale":"2024-09-13T14:39:50.061287+0000","last_undegraded":"2024-09-13T14:39:50.061287+0000","last_fullsized":"2024-09-13T14:39:50.061287+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:24:20.875473+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:19.239670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":640,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.059001+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:39:50.059001+0000","last_peered":"2024-09-13T14:39:50.059001+0000","last_clean":"2024-09-13T14:39:50.059001+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:39:50.059001+0000","last_undegraded":"2024-09-13T14:39:50.059001+0000","last_fullsized":"2024-09-13T14:39:50.059001+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":47,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.144959+0000","last_change":"2024-09-13T14:39:42.768341+0000","last_active":"2024-09-13T14:39:54.144959+0000","last_peered":"2024-09-13T14:39:54.144959+0000","last_clean":"2024-09-13T14:39:54.144959+0000","last_became_active":"2024-09-13T14:39:42.768034+0000","last_became_peered":"2024-09-13T14:39:42.768034+0000","last_unstale":"2024-09-13T14:39:54.144959+0000","last_undegraded":"2024-09-13T14:39:54.144959+0000","last_fullsized":"2024-09-13T14:39:54.144959+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:28:55.502108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.171657+0000","last_change":"2024-09-13T14:39:42.772096+0000","last_active":"2024-09-13T14:39:50.171657+0000","last_peered":"2024-09-13T14:39:50.171657+0000","last_clean":"2024-09-13T14:39:50.171657+0000","last_became_active":"2024-09-13T14:39:42.771762+0000","last_became_peered":"2024-09-13T14:39:42.771762+0000","last_unstale":"2024-09-13T14:39:50.171657+0000","last_undegraded":"2024-09-13T14:39:50.171657+0000","last_fullsized":"2024-09-13T14:39:50.171657+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:16.172571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.7","version":"113'1655","reported_seq":1669,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:40:01.544741+0000","last_change":"2024-09-13T14:39:42.773740+0000","last_active":"2024-09-13T14:40:01.544741+0000","last_peered":"2024-09-13T14:40:01.544741+0000","last_clean":"2024-09-13T14:40:01.544741+0000","last_became_active":"2024-09-13T14:39:42.773444+0000","last_became_peered":"2024-09-13T14:39:42.773444+0000","last_unstale":"2024-09-13T14:40:01.544741+0000","last_undegraded":"2024-09-13T14:40:01.544741+0000","last_fullsized":"2024-09-13T14:40:01.544741+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":1655,"log_dups_size":0,"ondisk_log_size":1655,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:09:16.690630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108462080,"num_objects":1655,"num_object_clones":0,"num_object_copies":4965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1655,"num_write_kb":105920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":324,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.061213+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:39:50.061213+0000","last_peered":"2024-09-13T14:39:50.061213+0000","last_clean":"2024-09-13T14:39:50.061213+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:39:50.061213+0000","last_undegraded":"2024-09-13T14:39:50.061213+0000","last_fullsized":"2024-09-13T14:39:50.061213+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":91,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.143295+0000","last_change":"2024-09-13T14:38:13.977962+0000","last_active":"2024-09-13T14:39:54.143295+0000","last_peered":"2024-09-13T14:39:54.143295+0000","last_clean":"2024-09-13T14:39:54.143295+0000","last_became_active":"2024-09-13T14:38:13.977638+0000","last_became_peered":"2024-09-13T14:38:13.977638+0000","last_unstale":"2024-09-13T14:39:54.143295+0000","last_undegraded":"2024-09-13T14:39:54.143295+0000","last_fullsized":"2024-09-13T14:39:54.143295+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:53:16.902157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":94,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.143370+0000","last_change":"2024-09-13T14:37:40.927444+0000","last_active":"2024-09-13T14:39:54.143370+0000","last_peered":"2024-09-13T14:39:54.143370+0000","last_clean":"2024-09-13T14:39:54.143370+0000","last_became_active":"2024-09-13T14:37:40.927140+0000","last_became_peered":"2024-09-13T14:37:40.927140+0000","last_unstale":"2024-09-13T14:39:54.143370+0000","last_undegraded":"2024-09-13T14:39:54.143370+0000","last_fullsized":"2024-09-13T14:39:54.143370+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:32:54.207568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"113'1193","reported_seq":1216,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:58.527399+0000","last_change":"2024-09-13T14:39:42.759808+0000","last_active":"2024-09-13T14:39:58.527399+0000","last_peered":"2024-09-13T14:39:58.527399+0000","last_clean":"2024-09-13T14:39:58.527399+0000","last_became_active":"2024-09-13T14:39:42.759199+0000","last_became_peered":"2024-09-13T14:39:42.759199+0000","last_unstale":"2024-09-13T14:39:58.527399+0000","last_undegraded":"2024-09-13T14:39:58.527399+0000","last_fullsized":"2024-09-13T14:39:58.527399+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":1193,"log_dups_size":0,"ondisk_log_size":1193,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:55:55.573511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78184448,"num_objects":1193,"num_object_clones":0,"num_object_copies":3579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1193,"num_write_kb":76352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":495,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.140590+0000","last_change":"2024-09-13T14:38:13.978679+0000","last_active":"2024-09-13T14:39:54.140590+0000","last_peered":"2024-09-13T14:39:54.140590+0000","last_clean":"2024-09-13T14:39:54.140590+0000","last_became_active":"2024-09-13T14:38:13.978171+0000","last_became_peered":"2024-09-13T14:38:13.978171+0000","last_unstale":"2024-09-13T14:39:54.140590+0000","last_undegraded":"2024-09-13T14:39:54.140590+0000","last_fullsized":"2024-09-13T14:39:54.140590+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:57.691198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":78,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.140946+0000","last_change":"2024-09-13T14:39:42.769691+0000","last_active":"2024-09-13T14:39:54.140946+0000","last_peered":"2024-09-13T14:39:54.140946+0000","last_clean":"2024-09-13T14:39:54.140946+0000","last_became_active":"2024-09-13T14:39:42.769266+0000","last_became_peered":"2024-09-13T14:39:42.769266+0000","last_unstale":"2024-09-13T14:39:54.140946+0000","last_undegraded":"2024-09-13T14:39:54.140946+0000","last_fullsized":"2024-09-13T14:39:54.140946+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:11.637732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":104,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.140419+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:39:54.140419+0000","last_peered":"2024-09-13T14:39:54.140419+0000","last_clean":"2024-09-13T14:39:54.140419+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:39:54.140419+0000","last_undegraded":"2024-09-13T14:39:54.140419+0000","last_fullsized":"2024-09-13T14:39:54.140419+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"113'1060","reported_seq":1083,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:57.997498+0000","last_change":"2024-09-13T14:39:42.776210+0000","last_active":"2024-09-13T14:39:57.997498+0000","last_peered":"2024-09-13T14:39:57.997498+0000","last_clean":"2024-09-13T14:39:57.997498+0000","last_became_active":"2024-09-13T14:39:42.775823+0000","last_became_peered":"2024-09-13T14:39:42.775823+0000","last_unstale":"2024-09-13T14:39:57.997498+0000","last_undegraded":"2024-09-13T14:39:57.997498+0000","last_fullsized":"2024-09-13T14:39:57.997498+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":1060,"log_dups_size":0,"ondisk_log_size":1060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:39.020681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69468160,"num_objects":1060,"num_object_clones":0,"num_object_copies":3180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1060,"num_write_kb":67840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":704,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.058816+0000","last_change":"2024-09-13T14:39:42.805055+0000","last_active":"2024-09-13T14:39:50.058816+0000","last_peered":"2024-09-13T14:39:50.058816+0000","last_clean":"2024-09-13T14:39:50.058816+0000","last_became_active":"2024-09-13T14:39:42.762456+0000","last_became_peered":"2024-09-13T14:39:42.762456+0000","last_unstale":"2024-09-13T14:39:50.058816+0000","last_undegraded":"2024-09-13T14:39:50.058816+0000","last_fullsized":"2024-09-13T14:39:50.058816+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:54:40.839991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":88,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.171728+0000","last_change":"2024-09-13T14:39:42.775159+0000","last_active":"2024-09-13T14:39:50.171728+0000","last_peered":"2024-09-13T14:39:50.171728+0000","last_clean":"2024-09-13T14:39:50.171728+0000","last_became_active":"2024-09-13T14:39:42.774841+0000","last_became_peered":"2024-09-13T14:39:42.774841+0000","last_unstale":"2024-09-13T14:39:50.171728+0000","last_undegraded":"2024-09-13T14:39:50.171728+0000","last_fullsized":"2024-09-13T14:39:50.171728+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:35:45.579397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":82,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.142025+0000","last_change":"2024-09-13T14:38:13.981294+0000","last_active":"2024-09-13T14:39:54.142025+0000","last_peered":"2024-09-13T14:39:54.142025+0000","last_clean":"2024-09-13T14:39:54.142025+0000","last_became_active":"2024-09-13T14:38:13.980975+0000","last_became_peered":"2024-09-13T14:38:13.980975+0000","last_unstale":"2024-09-13T14:39:54.142025+0000","last_undegraded":"2024-09-13T14:39:54.142025+0000","last_fullsized":"2024-09-13T14:39:54.142025+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:17.545179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"113'1064","reported_seq":1086,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:57.990976+0000","last_change":"2024-09-13T14:39:42.770363+0000","last_active":"2024-09-13T14:39:57.990976+0000","last_peered":"2024-09-13T14:39:57.990976+0000","last_clean":"2024-09-13T14:39:57.990976+0000","last_became_active":"2024-09-13T14:39:42.770092+0000","last_became_peered":"2024-09-13T14:39:42.770092+0000","last_unstale":"2024-09-13T14:39:57.990976+0000","last_undegraded":"2024-09-13T14:39:57.990976+0000","last_fullsized":"2024-09-13T14:39:57.990976+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":1064,"log_dups_size":0,"ondisk_log_size":1064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:20:20.375252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69730304,"num_objects":1064,"num_object_clones":0,"num_object_copies":3192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1064,"num_write_kb":68096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":95,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.059138+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:39:50.059138+0000","last_peered":"2024-09-13T14:39:50.059138+0000","last_clean":"2024-09-13T14:39:50.059138+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:39:50.059138+0000","last_undegraded":"2024-09-13T14:39:50.059138+0000","last_fullsized":"2024-09-13T14:39:50.059138+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":588,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.142190+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:39:54.142190+0000","last_peered":"2024-09-13T14:39:54.142190+0000","last_clean":"2024-09-13T14:39:54.142190+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:39:54.142190+0000","last_undegraded":"2024-09-13T14:39:54.142190+0000","last_fullsized":"2024-09-13T14:39:54.142190+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":60,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.139878+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:39:54.139878+0000","last_peered":"2024-09-13T14:39:54.139878+0000","last_clean":"2024-09-13T14:39:54.139878+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:39:54.139878+0000","last_undegraded":"2024-09-13T14:39:54.139878+0000","last_fullsized":"2024-09-13T14:39:54.139878+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"113'1103","reported_seq":1126,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:57.983114+0000","last_change":"2024-09-13T14:39:42.770399+0000","last_active":"2024-09-13T14:39:57.983114+0000","last_peered":"2024-09-13T14:39:57.983114+0000","last_clean":"2024-09-13T14:39:57.983114+0000","last_became_active":"2024-09-13T14:39:42.770074+0000","last_became_peered":"2024-09-13T14:39:42.770074+0000","last_unstale":"2024-09-13T14:39:57.983114+0000","last_undegraded":"2024-09-13T14:39:57.983114+0000","last_fullsized":"2024-09-13T14:39:57.983114+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":1103,"log_dups_size":0,"ondisk_log_size":1103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:28:03.030813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72286208,"num_objects":1103,"num_object_clones":0,"num_object_copies":3309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1103,"num_write_kb":70592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":71,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.142104+0000","last_change":"2024-09-13T14:37:40.921861+0000","last_active":"2024-09-13T14:39:54.142104+0000","last_peered":"2024-09-13T14:39:54.142104+0000","last_clean":"2024-09-13T14:39:54.142104+0000","last_became_active":"2024-09-13T14:37:40.921587+0000","last_became_peered":"2024-09-13T14:37:40.921587+0000","last_unstale":"2024-09-13T14:39:54.142104+0000","last_undegraded":"2024-09-13T14:39:54.142104+0000","last_fullsized":"2024-09-13T14:39:54.142104+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:24.619352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":326,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.061698+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:39:50.061698+0000","last_peered":"2024-09-13T14:39:50.061698+0000","last_clean":"2024-09-13T14:39:50.061698+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:39:50.061698+0000","last_undegraded":"2024-09-13T14:39:50.061698+0000","last_fullsized":"2024-09-13T14:39:50.061698+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":73,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.144259+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:39:54.144259+0000","last_peered":"2024-09-13T14:39:54.144259+0000","last_clean":"2024-09-13T14:39:54.144259+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:39:54.144259+0000","last_undegraded":"2024-09-13T14:39:54.144259+0000","last_fullsized":"2024-09-13T14:39:54.144259+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"113'1573","reported_seq":1587,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:40:01.557773+0000","last_change":"2024-09-13T14:39:42.772158+0000","last_active":"2024-09-13T14:40:01.557773+0000","last_peered":"2024-09-13T14:40:01.557773+0000","last_clean":"2024-09-13T14:40:01.557773+0000","last_became_active":"2024-09-13T14:39:42.771189+0000","last_became_peered":"2024-09-13T14:39:42.771189+0000","last_unstale":"2024-09-13T14:40:01.557773+0000","last_undegraded":"2024-09-13T14:40:01.557773+0000","last_fullsized":"2024-09-13T14:40:01.557773+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":1573,"log_dups_size":0,"ondisk_log_size":1573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:58:08.882288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103088128,"num_objects":1573,"num_object_clones":0,"num_object_copies":4719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1573,"num_write_kb":100672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":86,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.144698+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:39:54.144698+0000","last_peered":"2024-09-13T14:39:54.144698+0000","last_clean":"2024-09-13T14:39:54.144698+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:39:54.144698+0000","last_undegraded":"2024-09-13T14:39:54.144698+0000","last_fullsized":"2024-09-13T14:39:54.144698+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":409,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.140382+0000","last_change":"2024-09-13T14:39:44.990216+0000","last_active":"2024-09-13T14:39:54.140382+0000","last_peered":"2024-09-13T14:39:54.140382+0000","last_clean":"2024-09-13T14:39:54.140382+0000","last_became_active":"2024-09-13T14:39:44.989433+0000","last_became_peered":"2024-09-13T14:39:44.989433+0000","last_unstale":"2024-09-13T14:39:54.140382+0000","last_undegraded":"2024-09-13T14:39:54.140382+0000","last_fullsized":"2024-09-13T14:39:54.140382+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:00.426042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.144374+0000","last_change":"2024-09-13T14:38:13.979255+0000","last_active":"2024-09-13T14:39:54.144374+0000","last_peered":"2024-09-13T14:39:54.144374+0000","last_clean":"2024-09-13T14:39:54.144374+0000","last_became_active":"2024-09-13T14:38:13.978868+0000","last_became_peered":"2024-09-13T14:38:13.978868+0000","last_unstale":"2024-09-13T14:39:54.144374+0000","last_undegraded":"2024-09-13T14:39:54.144374+0000","last_fullsized":"2024-09-13T14:39:54.144374+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:15:26.003538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"113'1599","reported_seq":1622,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:40:01.590046+0000","last_change":"2024-09-13T14:39:42.770912+0000","last_active":"2024-09-13T14:40:01.590046+0000","last_peered":"2024-09-13T14:40:01.590046+0000","last_clean":"2024-09-13T14:40:01.590046+0000","last_became_active":"2024-09-13T14:39:42.770646+0000","last_became_peered":"2024-09-13T14:39:42.770646+0000","last_unstale":"2024-09-13T14:40:01.590046+0000","last_undegraded":"2024-09-13T14:40:01.590046+0000","last_fullsized":"2024-09-13T14:40:01.590046+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":1599,"log_dups_size":0,"ondisk_log_size":1599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:54:13.100820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104792064,"num_objects":1599,"num_object_clones":0,"num_object_copies":4797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1599,"num_write_kb":102336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":94,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.143117+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:39:54.143117+0000","last_peered":"2024-09-13T14:39:54.143117+0000","last_clean":"2024-09-13T14:39:54.143117+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:39:54.143117+0000","last_undegraded":"2024-09-13T14:39:54.143117+0000","last_fullsized":"2024-09-13T14:39:54.143117+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1030,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.061684+0000","last_change":"2024-09-13T14:39:44.990389+0000","last_active":"2024-09-13T14:39:50.061684+0000","last_peered":"2024-09-13T14:39:50.061684+0000","last_clean":"2024-09-13T14:39:50.061684+0000","last_became_active":"2024-09-13T14:39:44.989805+0000","last_became_peered":"2024-09-13T14:39:44.989805+0000","last_unstale":"2024-09-13T14:39:50.061684+0000","last_undegraded":"2024-09-13T14:39:50.061684+0000","last_fullsized":"2024-09-13T14:39:50.061684+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:15.392445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":70,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.171388+0000","last_change":"2024-09-13T14:39:42.769954+0000","last_active":"2024-09-13T14:39:50.171388+0000","last_peered":"2024-09-13T14:39:50.171388+0000","last_clean":"2024-09-13T14:39:50.171388+0000","last_became_active":"2024-09-13T14:39:42.769635+0000","last_became_peered":"2024-09-13T14:39:42.769635+0000","last_unstale":"2024-09-13T14:39:50.171388+0000","last_undegraded":"2024-09-13T14:39:50.171388+0000","last_fullsized":"2024-09-13T14:39:50.171388+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:51.588071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":87,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.171695+0000","last_change":"2024-09-13T14:39:42.775357+0000","last_active":"2024-09-13T14:39:50.171695+0000","last_peered":"2024-09-13T14:39:50.171695+0000","last_clean":"2024-09-13T14:39:50.171695+0000","last_became_active":"2024-09-13T14:39:42.775041+0000","last_became_peered":"2024-09-13T14:39:42.775041+0000","last_unstale":"2024-09-13T14:39:50.171695+0000","last_undegraded":"2024-09-13T14:39:50.171695+0000","last_fullsized":"2024-09-13T14:39:50.171695+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:44.768910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":85,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.143519+0000","last_change":"2024-09-13T14:39:42.787151+0000","last_active":"2024-09-13T14:39:54.143519+0000","last_peered":"2024-09-13T14:39:54.143519+0000","last_clean":"2024-09-13T14:39:54.143519+0000","last_became_active":"2024-09-13T14:39:42.771712+0000","last_became_peered":"2024-09-13T14:39:42.771712+0000","last_unstale":"2024-09-13T14:39:54.143519+0000","last_undegraded":"2024-09-13T14:39:54.143519+0000","last_fullsized":"2024-09-13T14:39:54.143519+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:54.216663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":78,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.139759+0000","last_change":"2024-09-13T14:39:42.772974+0000","last_active":"2024-09-13T14:39:54.139759+0000","last_peered":"2024-09-13T14:39:54.139759+0000","last_clean":"2024-09-13T14:39:54.139759+0000","last_became_active":"2024-09-13T14:39:42.771449+0000","last_became_peered":"2024-09-13T14:39:42.771449+0000","last_unstale":"2024-09-13T14:39:54.139759+0000","last_undegraded":"2024-09-13T14:39:54.139759+0000","last_fullsized":"2024-09-13T14:39:54.139759+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:46.441079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":93,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.061582+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:39:50.061582+0000","last_peered":"2024-09-13T14:39:50.061582+0000","last_clean":"2024-09-13T14:39:50.061582+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:39:50.061582+0000","last_undegraded":"2024-09-13T14:39:50.061582+0000","last_fullsized":"2024-09-13T14:39:50.061582+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":78,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.059099+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:39:50.059099+0000","last_peered":"2024-09-13T14:39:50.059099+0000","last_clean":"2024-09-13T14:39:50.059099+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:39:50.059099+0000","last_undegraded":"2024-09-13T14:39:50.059099+0000","last_fullsized":"2024-09-13T14:39:50.059099+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":47,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.143052+0000","last_change":"2024-09-13T14:39:42.767764+0000","last_active":"2024-09-13T14:39:54.143052+0000","last_peered":"2024-09-13T14:39:54.143052+0000","last_clean":"2024-09-13T14:39:54.143052+0000","last_became_active":"2024-09-13T14:39:42.767450+0000","last_became_peered":"2024-09-13T14:39:42.767450+0000","last_unstale":"2024-09-13T14:39:54.143052+0000","last_undegraded":"2024-09-13T14:39:54.143052+0000","last_fullsized":"2024-09-13T14:39:54.143052+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:47.729382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":82,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.148119+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:39:54.148119+0000","last_peered":"2024-09-13T14:39:54.148119+0000","last_clean":"2024-09-13T14:39:54.148119+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:39:54.148119+0000","last_undegraded":"2024-09-13T14:39:54.148119+0000","last_fullsized":"2024-09-13T14:39:54.148119+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":96,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.144454+0000","last_change":"2024-09-13T14:39:42.772266+0000","last_active":"2024-09-13T14:39:54.144454+0000","last_peered":"2024-09-13T14:39:54.144454+0000","last_clean":"2024-09-13T14:39:54.144454+0000","last_became_active":"2024-09-13T14:39:42.771880+0000","last_became_peered":"2024-09-13T14:39:42.771880+0000","last_unstale":"2024-09-13T14:39:54.144454+0000","last_undegraded":"2024-09-13T14:39:54.144454+0000","last_fullsized":"2024-09-13T14:39:54.144454+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:41:01.160105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.059164+0000","last_change":"2024-09-13T14:39:42.773890+0000","last_active":"2024-09-13T14:39:50.059164+0000","last_peered":"2024-09-13T14:39:50.059164+0000","last_clean":"2024-09-13T14:39:50.059164+0000","last_became_active":"2024-09-13T14:39:42.773523+0000","last_became_peered":"2024-09-13T14:39:42.773523+0000","last_unstale":"2024-09-13T14:39:50.059164+0000","last_undegraded":"2024-09-13T14:39:50.059164+0000","last_fullsized":"2024-09-13T14:39:50.059164+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:13.156261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":107,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.144296+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:39:54.144296+0000","last_peered":"2024-09-13T14:39:54.144296+0000","last_clean":"2024-09-13T14:39:54.144296+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:39:54.144296+0000","last_undegraded":"2024-09-13T14:39:54.144296+0000","last_fullsized":"2024-09-13T14:39:54.144296+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":84,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.143291+0000","last_change":"2024-09-13T14:38:13.979604+0000","last_active":"2024-09-13T14:39:54.143291+0000","last_peered":"2024-09-13T14:39:54.143291+0000","last_clean":"2024-09-13T14:39:54.143291+0000","last_became_active":"2024-09-13T14:38:13.979288+0000","last_became_peered":"2024-09-13T14:38:13.979288+0000","last_unstale":"2024-09-13T14:39:54.143291+0000","last_undegraded":"2024-09-13T14:39:54.143291+0000","last_fullsized":"2024-09-13T14:39:54.143291+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:21.031527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":71,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.139864+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:39:54.139864+0000","last_peered":"2024-09-13T14:39:54.139864+0000","last_clean":"2024-09-13T14:39:54.139864+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:39:54.139864+0000","last_undegraded":"2024-09-13T14:39:54.139864+0000","last_fullsized":"2024-09-13T14:39:54.139864+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":103,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.143612+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:39:54.143612+0000","last_peered":"2024-09-13T14:39:54.143612+0000","last_clean":"2024-09-13T14:39:54.143612+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:39:54.143612+0000","last_undegraded":"2024-09-13T14:39:54.143612+0000","last_fullsized":"2024-09-13T14:39:54.143612+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":96,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.171595+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:39:50.171595+0000","last_peered":"2024-09-13T14:39:50.171595+0000","last_clean":"2024-09-13T14:39:50.171595+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:39:50.171595+0000","last_undegraded":"2024-09-13T14:39:50.171595+0000","last_fullsized":"2024-09-13T14:39:50.171595+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":109,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.140888+0000","last_change":"2024-09-13T14:38:13.976950+0000","last_active":"2024-09-13T14:39:54.140888+0000","last_peered":"2024-09-13T14:39:54.140888+0000","last_clean":"2024-09-13T14:39:54.140888+0000","last_became_active":"2024-09-13T14:38:13.976639+0000","last_became_peered":"2024-09-13T14:38:13.976639+0000","last_unstale":"2024-09-13T14:39:54.140888+0000","last_undegraded":"2024-09-13T14:39:54.140888+0000","last_fullsized":"2024-09-13T14:39:54.140888+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:24.126890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":75,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.144135+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:39:54.144135+0000","last_peered":"2024-09-13T14:39:54.144135+0000","last_clean":"2024-09-13T14:39:54.144135+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:39:54.144135+0000","last_undegraded":"2024-09-13T14:39:54.144135+0000","last_fullsized":"2024-09-13T14:39:54.144135+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":94,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.171603+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:39:50.171603+0000","last_peered":"2024-09-13T14:39:50.171603+0000","last_clean":"2024-09-13T14:39:50.171603+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:39:50.171603+0000","last_undegraded":"2024-09-13T14:39:50.171603+0000","last_fullsized":"2024-09-13T14:39:50.171603+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":80,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.058980+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:39:50.058980+0000","last_peered":"2024-09-13T14:39:50.058980+0000","last_clean":"2024-09-13T14:39:50.058980+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:39:50.058980+0000","last_undegraded":"2024-09-13T14:39:50.058980+0000","last_fullsized":"2024-09-13T14:39:50.058980+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":94,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.171505+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:39:50.171505+0000","last_peered":"2024-09-13T14:39:50.171505+0000","last_clean":"2024-09-13T14:39:50.171505+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:39:50.171505+0000","last_undegraded":"2024-09-13T14:39:50.171505+0000","last_fullsized":"2024-09-13T14:39:50.171505+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":105,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.059059+0000","last_change":"2024-09-13T14:39:42.778413+0000","last_active":"2024-09-13T14:39:50.059059+0000","last_peered":"2024-09-13T14:39:50.059059+0000","last_clean":"2024-09-13T14:39:50.059059+0000","last_became_active":"2024-09-13T14:39:42.766624+0000","last_became_peered":"2024-09-13T14:39:42.766624+0000","last_unstale":"2024-09-13T14:39:50.059059+0000","last_undegraded":"2024-09-13T14:39:50.059059+0000","last_fullsized":"2024-09-13T14:39:50.059059+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:13.435604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":69,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.058657+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:39:50.058657+0000","last_peered":"2024-09-13T14:39:50.058657+0000","last_clean":"2024-09-13T14:39:50.058657+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:39:50.058657+0000","last_undegraded":"2024-09-13T14:39:50.058657+0000","last_fullsized":"2024-09-13T14:39:50.058657+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":102,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.144399+0000","last_change":"2024-09-13T14:39:42.767451+0000","last_active":"2024-09-13T14:39:54.144399+0000","last_peered":"2024-09-13T14:39:54.144399+0000","last_clean":"2024-09-13T14:39:54.144399+0000","last_became_active":"2024-09-13T14:39:42.767086+0000","last_became_peered":"2024-09-13T14:39:42.767086+0000","last_unstale":"2024-09-13T14:39:54.144399+0000","last_undegraded":"2024-09-13T14:39:54.144399+0000","last_fullsized":"2024-09-13T14:39:54.144399+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:02.674069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":80,"reported_epoch":112,"state":"active+clean","last_fresh":"2024-09-13T14:39:50.171300+0000","last_change":"2024-09-13T14:39:42.768653+0000","last_active":"2024-09-13T14:39:50.171300+0000","last_peered":"2024-09-13T14:39:50.171300+0000","last_clean":"2024-09-13T14:39:50.171300+0000","last_became_active":"2024-09-13T14:39:42.768333+0000","last_became_peered":"2024-09-13T14:39:42.768333+0000","last_unstale":"2024-09-13T14:39:50.171300+0000","last_undegraded":"2024-09-13T14:39:50.171300+0000","last_fullsized":"2024-09-13T14:39:50.171300+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:25.655793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":87,"reported_epoch":113,"state":"active+clean","last_fresh":"2024-09-13T14:39:54.148074+0000","last_change":"2024-09-13T14:37:40.924860+0000","last_active":"2024-09-13T14:39:54.148074+0000","last_peered":"2024-09-13T14:39:54.148074+0000","last_clean":"2024-09-13T14:39:54.148074+0000","last_became_active":"2024-09-13T14:37:40.924522+0000","last_became_peered":"2024-09-13T14:37:40.924522+0000","last_unstale":"2024-09-13T14:39:54.148074+0000","last_undegraded":"2024-09-13T14:39:54.148074+0000","last_fullsized":"2024-09-13T14:39:54.148074+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:10.340511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":1403912192,"num_objects":21422,"num_object_clones":0,"num_object_copies":64266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":21422,"num_write_kb":1371008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4309909504,"data_stored":4309909504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":21422,"ondisk_log_size":21422,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":9648,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397673,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":813972,"kb_used_data":516620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":92931180,"statfs":{"total":95995035648,"available":95161528320,"internally_reserved":0,"allocated":529018880,"data_stored":528388351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":184683593902,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790131,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1182208,"kb_used_data":884856,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":92562944,"statfs":{"total":95995035648,"available":94784454656,"internally_reserved":0,"allocated":906092544,"data_stored":905477420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986360,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1119364,"kb_used_data":822012,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":92625788,"statfs":{"total":95995035648,"available":94848806912,"internally_reserved":0,"allocated":841740288,"data_stored":841120799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598808,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":432440,"kb_used_data":396592,"kb_used_omap":6,"kb_used_meta":35833,"kb_avail":93312712,"statfs":{"total":95995035648,"available":95552217088,"internally_reserved":0,"allocated":406110208,"data_stored":405502715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":36693984},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762334,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":628108,"kb_used_data":592260,"kb_used_omap":8,"kb_used_meta":35831,"kb_avail":93117044,"statfs":{"total":95995035648,"available":95351853056,"internally_reserved":0,"allocated":606474240,"data_stored":605851567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":36691402},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":94,"seq":403726925860,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":358464,"kb_used_data":330872,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93386688,"statfs":{"total":95995035648,"available":95627968512,"internally_reserved":0,"allocated":338812928,"data_stored":338197980,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":28173001},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122100,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":519128,"kb_used_data":487440,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93226024,"statfs":{"total":95995035648,"available":95463448576,"internally_reserved":0,"allocated":499138560,"data_stored":498504462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":32432719},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":569442304,"data_stored":569442304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":339279872,"data_stored":339279872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":625475584,"data_stored":625475584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":422576128,"data_stored":422576128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":847446016,"data_stored":847446016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":970194944,"data_stored":970194944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":535494656,"data_stored":535494656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:40:03.727 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:40:03.727 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pg_num 11 2024-09-13T14:40:03.743 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:03.791 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:03.796 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:03.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:03.858 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:03.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:03.937 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:03 smithi096 ceph-mon[98901]: from='client.35061 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:40:03.937 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:03 smithi096 ceph-mon[102928]: from='client.35061 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:40:03.960 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:04.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:04.000 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:04.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:04.101 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:04.129 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:04.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:04.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:04.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:03 smithi204 ceph-mon[87071]: from='client.35061 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:40:04.232 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:04.265 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:04.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:04.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:04.367 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:04.398 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:04.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:04.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:04.500 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:04.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:04.531 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:04.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:04.632 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:04.666 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:04.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:04.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:04.708 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 12 16 32696 32680 170.183 134.688 0.00360866 0.00587115 2024-09-13T14:40:04.768 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:04.801 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:04.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:04.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:04.903 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:04.935 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:04.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:04.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:05.037 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:05.069 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:05.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:05.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:05.172 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:05.207 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:05.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:05.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:05.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:04 smithi204 ceph-mon[87071]: pgmap v258: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 167 MiB/s wr, 2.67k op/s 2024-09-13T14:40:05.309 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:05.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:04 smithi096 ceph-mon[98901]: pgmap v258: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 167 MiB/s wr, 2.67k op/s 2024-09-13T14:40:05.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:04 smithi096 ceph-mon[102928]: pgmap v258: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 167 MiB/s wr, 2.67k op/s 2024-09-13T14:40:05.339 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:05.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:05.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:05.441 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:05.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:05.474 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:05.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:05.575 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:05.612 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:05.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:05.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:05.709 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 13 16 35087 35071 168.586 149.438 0.00552752 0.00592736 2024-09-13T14:40:05.714 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:05.745 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:05.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:05.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:05.847 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:05.884 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:05.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:05.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:05.986 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:06.019 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:06.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:06.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:06.121 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:06.159 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:06.186 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:06.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:06.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:06.287 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:06.325 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:06.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:06.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:06.428 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:06.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:06.463 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:06.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:06.564 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:06.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:06.595 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:06.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:06.697 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:06.708 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 14 16 37226 37210 166.092 133.688 0.0127319 0.00601481 2024-09-13T14:40:06.740 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:06.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:06.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:06.757 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:06.843 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:06.874 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:06.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:06.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:06.911 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:06 smithi096 ceph-mon[98901]: pgmap v259: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 145 MiB/s wr, 2.33k op/s 2024-09-13T14:40:06.911 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:06.911 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:06.911 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:06 smithi096 ceph-mon[102928]: pgmap v259: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 145 MiB/s wr, 2.33k op/s 2024-09-13T14:40:06.911 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:06.911 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:06.976 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:07.005 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:07.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:07.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:07.107 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:07.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:07.137 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:07.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:07.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:06 smithi204 ceph-mon[87071]: pgmap v259: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 145 MiB/s wr, 2.33k op/s 2024-09-13T14:40:07.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:07.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:07.239 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:07.267 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:07.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:07.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:07.369 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:07.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:07.391 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:07.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:07.492 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:07.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:07.526 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:07.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:07.628 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:07.658 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:07.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:07.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:07.708 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 15 16 39156 39140 163.06 120.625 0.00346635 0.00612798 2024-09-13T14:40:07.760 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:07.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:07.796 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:07.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:07.897 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:07.933 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:07.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:07.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:08.035 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:08.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:08.068 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:08.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:08.169 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:08.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:08.203 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:08.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:08.304 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:08.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:40:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:40:07] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.51.0" 2024-09-13T14:40:08.344 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:08.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:08.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:08.447 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:08.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:08.478 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:08.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:08.580 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:08.604 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:08.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:08.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:08.706 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:08.708 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 16 16 41098 41082 160.454 121.375 0.0106956 0.00622774 2024-09-13T14:40:08.739 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:08.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:08.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:08.841 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:08.863 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:08.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:08.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:08.965 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:08.999 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:08.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:09.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:09.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:08 smithi096 ceph-mon[98901]: pgmap v260: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 167 MiB/s wr, 2.68k op/s 2024-09-13T14:40:09.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:08 smithi096 ceph-mon[102928]: pgmap v260: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 167 MiB/s wr, 2.68k op/s 2024-09-13T14:40:09.101 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:09.133 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:09.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:09.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:09.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:08 smithi204 ceph-mon[87071]: pgmap v260: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 167 MiB/s wr, 2.68k op/s 2024-09-13T14:40:09.235 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:09.267 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:09.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:09.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:09.369 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:09.405 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:09.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:09.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:09.507 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:09.541 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:09.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:09.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:09.643 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:09.673 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:09.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:09.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:09.708 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 17 16 42943 42927 157.799 115.312 0.00842707 0.00633295 2024-09-13T14:40:09.775 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:09.810 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:09.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:09.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:09.912 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:09.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:09.943 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:09.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:10.043 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:10.075 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:10.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:10.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:10.177 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:10.201 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:10.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:10.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:10.303 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:10.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:10.342 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:10.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:10.444 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:10.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:40:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:40:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:10.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:10.485 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:10.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:10.586 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:10.624 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:10.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:10.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:10.708 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 18 16 44919 44903 155.893 123.5 0.0374665 0.00640559 2024-09-13T14:40:10.727 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:10.759 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:10.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:10.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:10.861 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:10.896 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:10.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:10.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:10.998 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:11.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:11.032 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:11.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:11.133 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:11.165 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:11.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:11.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:11.190 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:10 smithi096 ceph-mon[98901]: pgmap v261: 89 pgs: 89 active+clean; 2.4 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 161 MiB/s wr, 2.58k op/s 2024-09-13T14:40:11.190 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:10 smithi096 ceph-mon[102928]: pgmap v261: 89 pgs: 89 active+clean; 2.4 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 161 MiB/s wr, 2.58k op/s 2024-09-13T14:40:11.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:10 smithi204 ceph-mon[87071]: pgmap v261: 89 pgs: 89 active+clean; 2.4 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 161 MiB/s wr, 2.58k op/s 2024-09-13T14:40:11.266 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:11.295 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:11.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:11.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:11.397 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:11.418 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:11.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:11.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:11.520 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:11.549 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:11.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:11.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:11.651 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:11.686 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:11.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:11.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:11.708 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 19 16 46530 46514 152.987 100.688 0.00441044 0.00653156 2024-09-13T14:40:11.788 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:11.825 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:11.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:11.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:11.927 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:11.963 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:11.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:11.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:12.066 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:12.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:12.099 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:12.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:12.200 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:12.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:12.261 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:12.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:12.362 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:12.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:12.390 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:12.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:12.432 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:40:12.432 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:40:12.478 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:40:12.478 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:40:12.491 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:12.522 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:40:12.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:12.523 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:40:12.523 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:12.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:12.558 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:40:12.558 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:40:12.596 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:40:12.597 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:40:12.624 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:12.628 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:40:12.629 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:40:12.662 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:12.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:12.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:12.665 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:40:12.666 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:40:12.698 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:40:12.698 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:40:12.708 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:40:12.708906+0000 min lat: 0.000998702 max lat: 0.13403 avg lat: 0.00664146 2024-09-13T14:40:12.709 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:40:12.709 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 20 16 48171 48155 150.465 102.562 0.0136016 0.00664146 2024-09-13T14:40:12.764 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:12.803 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:12.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:12.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:12.839 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:40:12.905 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:12.942 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:12.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:12.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:13.045 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:13.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:13.083 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:13.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:13.107 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:12 smithi096 ceph-mon[98901]: pgmap v262: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-13T14:40:13.107 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:12 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/739753415' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:13.107 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:12 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4156387483' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:40:13.108 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:12 smithi096 ceph-mon[102928]: pgmap v262: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-13T14:40:13.108 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:12 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/739753415' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:13.108 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:12 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4156387483' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:40:13.184 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:13.218 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:13.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:12 smithi204 ceph-mon[87071]: pgmap v262: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-13T14:40:13.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:12 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/739753415' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:13.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:12 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4156387483' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:40:13.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:13.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:13.320 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:13.352 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:13.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:13.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:13.454 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:13.490 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:13.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:13.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:13.592 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:13.623 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:13.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:13.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:13.709 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 21 16 49968 49952 148.648 112.312 0.0160918 0.00672231 2024-09-13T14:40:13.724 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:13.761 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:13.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:13.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:13.863 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:13.890 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:13.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:13.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:13.991 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:14.005 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:13 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:40:14.005 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:13 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4156387483' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:40:14.005 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:13 smithi096 ceph-mon[98901]: osdmap e114: 8 total, 8 up, 7 in 2024-09-13T14:40:14.005 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:13 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:40:14.005 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:13 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4156387483' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:40:14.005 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:13 smithi096 ceph-mon[102928]: osdmap e114: 8 total, 8 up, 7 in 2024-09-13T14:40:14.027 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:14.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:14.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:14.129 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:14.171 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:14.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:14.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:14.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:13 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:40:14.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:13 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4156387483' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:40:14.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:13 smithi204 ceph-mon[87071]: osdmap e114: 8 total, 8 up, 7 in 2024-09-13T14:40:14.273 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:14.300 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:14.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:14.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:14.401 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:14.430 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:14.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:14.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:14.443 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:40:14.443 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:40:14.443 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.444 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:51257 5.7 e114/107 5:ec3fb5dc:::benchmark_data_smithi096_138841_object51256:head v 114'3231, mlcod=114'3229)", 2024-09-13T14:40:14.444 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:14.441164+0000", 2024-09-13T14:40:14.444 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.001458724, 2024-09-13T14:40:14.444 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001485423, 2024-09-13T14:40:14.444 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:40:14.444 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:40:14.444 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:40:14.444 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:40:14.444 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.444 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:40:14.444 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441164+0000", 2024-09-13T14:40:14.444 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:14.445 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:14.445 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.445 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:40:14.445 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441164+0000", 2024-09-13T14:40:14.445 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:14.445 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:14.445 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.445 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:40:14.445 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441165+0000", 2024-09-13T14:40:14.445 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7300000000000004e-07 2024-09-13T14:40:14.445 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:14.445 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.445 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:40:14.446 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441250+0000", 2024-09-13T14:40:14.446 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.4896999999999997e-05 2024-09-13T14:40:14.446 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:14.446 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.446 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:40:14.446 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441266+0000", 2024-09-13T14:40:14.446 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5349000000000002e-05 2024-09-13T14:40:14.446 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:14.446 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.446 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:40:14.446 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441274+0000", 2024-09-13T14:40:14.446 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.8120000000000008e-06 2024-09-13T14:40:14.446 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:14.446 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.447 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:40:14.447 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441347+0000", 2024-09-13T14:40:14.447 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3417999999999994e-05 2024-09-13T14:40:14.447 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:14.447 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.447 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:40:14.447 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441365+0000", 2024-09-13T14:40:14.447 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.857e-05 2024-09-13T14:40:14.447 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:14.447 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:40:14.447 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:14.447 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:14.447 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.447 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:51261 5.0 e114/107 5:01296361:::benchmark_data_smithi096_138841_object51260:head v 114'3159, mlcod=114'3157)", 2024-09-13T14:40:14.448 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:14.441312+0000", 2024-09-13T14:40:14.448 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0013111279999999999, 2024-09-13T14:40:14.448 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0013944210000000001, 2024-09-13T14:40:14.448 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:40:14.448 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:40:14.448 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:40:14.448 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:40:14.448 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.448 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:40:14.448 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441312+0000", 2024-09-13T14:40:14.448 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:14.448 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:14.449 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.449 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:40:14.449 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441312+0000", 2024-09-13T14:40:14.449 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:14.449 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:14.449 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.449 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:40:14.449 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441313+0000", 2024-09-13T14:40:14.449 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7199999999999999e-07 2024-09-13T14:40:14.449 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:14.449 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.449 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:40:14.449 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441332+0000", 2024-09-13T14:40:14.449 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9842000000000001e-05 2024-09-13T14:40:14.450 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:14.450 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.450 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:40:14.450 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441340+0000", 2024-09-13T14:40:14.450 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1100000000000003e-06 2024-09-13T14:40:14.450 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:14.450 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.450 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:40:14.450 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441346+0000", 2024-09-13T14:40:14.450 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6169999999999998e-06 2024-09-13T14:40:14.450 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:14.450 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.450 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:40:14.451 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441415+0000", 2024-09-13T14:40:14.451 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8936999999999994e-05 2024-09-13T14:40:14.451 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:14.451 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:14.451 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:40:14.451 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:14.441432+0000", 2024-09-13T14:40:14.451 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6844999999999999e-05 2024-09-13T14:40:14.451 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:14.451 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:40:14.451 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:14.451 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:14.451 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2024-09-13T14:40:14.451 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 2 2024-09-13T14:40:14.451 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:40:14.532 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:14.563 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:14.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:14.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:14.665 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:14.700 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:14.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:14.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:14.708 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 22 16 51740 51724 146.925 110.75 0.00617382 0.00680188 2024-09-13T14:40:14.802 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:14.828 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:14.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:14.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:14.930 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:14.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:14.961 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:14.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:15.062 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:15.095 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:15.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:15.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:15.197 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:15.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:14 smithi204 ceph-mon[87071]: pgmap v264: 89 pgs: 89 active+clean; 2.9 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 125 MiB/s wr, 1.99k op/s 2024-09-13T14:40:15.235 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:14 smithi096 ceph-mon[98901]: pgmap v264: 89 pgs: 89 active+clean; 2.9 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 125 MiB/s wr, 1.99k op/s 2024-09-13T14:40:15.235 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:14 smithi096 ceph-mon[102928]: pgmap v264: 89 pgs: 89 active+clean; 2.9 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 125 MiB/s wr, 1.99k op/s 2024-09-13T14:40:15.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:15.238 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:15.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:15.276 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:40:15.340 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:15.367 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:15.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:15.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:15.469 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:15.498 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:15.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:15.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:15.600 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:15.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:15.631 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:15.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:15.709 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 23 16 53499 53483 145.316 109.938 0.00530894 0.00687646 2024-09-13T14:40:15.731 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:15.749 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:40:15.760 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:15.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:15.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:15.862 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:15.889 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:15.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:15.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:15.991 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:16.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:16.039 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:16.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:16.140 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:16.164 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:16.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:16.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:16.266 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:16.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:16.297 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:16.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:16.398 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:16.441 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:16.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:16.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:16.542 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:40:16.542 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:16.588 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:16.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:16.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:16.665 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:16.670 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:16.690 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:16.711 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 24 16 54929 54913 142.971 89.375 0.0154206 0.00698827 2024-09-13T14:40:16.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:16.757 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:16.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:16.858 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:16.884 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:16.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:16.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:16.986 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:17.012 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:17.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:17.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:17.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:16 smithi096 ceph-mon[98901]: pgmap v265: 89 pgs: 89 active+clean; 2.9 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 125 MiB/s wr, 1.99k op/s 2024-09-13T14:40:17.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:16 smithi096 ceph-mon[102928]: pgmap v265: 89 pgs: 89 active+clean; 2.9 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 125 MiB/s wr, 1.99k op/s 2024-09-13T14:40:17.114 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:17.134 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:17.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:17.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:16 smithi204 ceph-mon[87071]: pgmap v265: 89 pgs: 89 active+clean; 2.9 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 125 MiB/s wr, 1.99k op/s 2024-09-13T14:40:17.236 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:17.271 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:17.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:17.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:17.373 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:17.403 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:17.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:17.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:17.505 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:17.530 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:17.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:17.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:17.631 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:17.662 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:17.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:17.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:17.711 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 25 16 56137 56121 140.272 75.5 0.0646379 0.00711491 2024-09-13T14:40:17.763 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:17.791 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:17.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:17.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:17.893 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:17.895 INFO:teuthology.orchestra.run.smithi096.stderr:set pool 1 pg_num to 11 2024-09-13T14:40:17.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:17.919 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:17.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:18.020 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:18.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:18.062 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:18.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:18.163 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:18.194 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:18.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:18.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:18.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:17 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3865510436' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:40:18.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:17 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:40:18.296 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:18.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:17 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3865510436' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:40:18.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:17 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:40:18.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:40:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:40:17] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.51.0" 2024-09-13T14:40:18.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:17 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3865510436' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:40:18.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:17 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:40:18.337 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:18.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:18.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:18.439 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:18.460 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:18.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:18.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:18.562 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:18.599 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:18.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:18.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:18.701 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:18.711 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 26 16 57735 57719 138.718 99.875 0.0110044 0.00720387 2024-09-13T14:40:18.738 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:18.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:18.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:18.840 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:18.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:18.873 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:18.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:18.974 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:19.011 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:19.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:19.113 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:19.141 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:18 smithi096 ceph-mon[98901]: pgmap v266: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-09-13T14:40:19.141 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:18 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-13T14:40:19.141 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:18 smithi096 ceph-mon[98901]: osdmap e115: 8 total, 8 up, 7 in 2024-09-13T14:40:19.142 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:18 smithi096 ceph-mon[102928]: pgmap v266: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-09-13T14:40:19.142 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:18 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-13T14:40:19.142 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:18 smithi096 ceph-mon[102928]: osdmap e115: 8 total, 8 up, 7 in 2024-09-13T14:40:19.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.148 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:19.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:19.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:18 smithi204 ceph-mon[87071]: pgmap v266: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-09-13T14:40:19.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:18 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-13T14:40:19.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:18 smithi204 ceph-mon[87071]: osdmap e115: 8 total, 8 up, 7 in 2024-09-13T14:40:19.249 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:19.281 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:19.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:19.383 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:19.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.416 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:19.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:19.517 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:19.548 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:19.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:19.650 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:19.689 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:19.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:19.712 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 27 16 59407 59391 137.45 104.5 0.00723496 0.00727067 2024-09-13T14:40:19.790 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:19.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.830 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:19.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:19.931 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:19.964 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:19.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:20.066 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:20.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:19 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-13T14:40:20.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:19 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-13T14:40:20.101 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:20.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:20.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:20.203 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:20.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:19 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-13T14:40:20.238 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:20.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:20.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:20.340 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:20.375 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:20.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:20.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:20.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:40:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:40:20.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:20.477 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:20.515 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:20.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:20.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:20.616 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:20.654 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:20.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:20.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:20.712 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 28 16 61052 61036 136.213 102.812 0.0119922 0.0073366 2024-09-13T14:40:20.756 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:20.791 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:20.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:20.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:20.892 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:20.925 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:20.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:20.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:21.027 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:21.060 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:21.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:21.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:21.162 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:21.193 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:21.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:21.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:21.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:20 smithi204 ceph-mon[87071]: pgmap v268: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 102 MiB/s wr, 1.64k op/s 2024-09-13T14:40:21.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:20 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-13T14:40:21.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:20 smithi204 ceph-mon[87071]: osdmap e116: 8 total, 8 up, 7 in 2024-09-13T14:40:21.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:20 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:21.295 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:21.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:20 smithi096 ceph-mon[98901]: pgmap v268: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 102 MiB/s wr, 1.64k op/s 2024-09-13T14:40:21.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:20 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-13T14:40:21.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:20 smithi096 ceph-mon[98901]: osdmap e116: 8 total, 8 up, 7 in 2024-09-13T14:40:21.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:20 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:21.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:20 smithi096 ceph-mon[102928]: pgmap v268: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 102 MiB/s wr, 1.64k op/s 2024-09-13T14:40:21.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:20 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-13T14:40:21.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:20 smithi096 ceph-mon[102928]: osdmap e116: 8 total, 8 up, 7 in 2024-09-13T14:40:21.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:20 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:21.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:21.349 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:21.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:21.450 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:21.485 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:21.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:21.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:21.587 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:21.621 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:21.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:21.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:21.712 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 29 16 62648 62632 134.956 99.75 0.00869018 0.00740489 2024-09-13T14:40:21.723 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:21.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:21.764 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:21.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:21.865 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:21.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:21.912 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:21.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:21.936 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:22.013 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:22.054 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:22.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:22.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:22.156 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:22.186 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:22.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:22.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:22.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:21 smithi204 ceph-mon[87071]: osdmap e117: 8 total, 8 up, 7 in 2024-09-13T14:40:22.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-13T14:40:22.288 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:22.321 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:22.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:22.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:22.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:21 smithi096 ceph-mon[98901]: osdmap e117: 8 total, 8 up, 7 in 2024-09-13T14:40:22.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-13T14:40:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:21 smithi096 ceph-mon[102928]: osdmap e117: 8 total, 8 up, 7 in 2024-09-13T14:40:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-13T14:40:22.422 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:22.471 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:22.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:22.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:22.573 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:22.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:22.618 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:22.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:22.712 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 30 16 64176 64160 133.64 95.5 0.0194769 0.00747736 2024-09-13T14:40:22.719 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:22.755 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:22.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:22.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:22.857 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:22.893 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:22.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:22.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:22.995 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:23.040 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:23.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:23.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:23.142 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:23.162 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:40:23.162 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:40:23.163 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:40:23.163 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:40:23.163 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:40:23.182 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:23.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:23.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:23.211 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:22 smithi096 ceph-mon[98901]: pgmap v271: 99 pgs: 99 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-13T14:40:23.211 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:22 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-13T14:40:23.211 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:22 smithi096 ceph-mon[98901]: osdmap e118: 8 total, 8 up, 7 in 2024-09-13T14:40:23.212 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:22 smithi096 ceph-mon[102928]: pgmap v271: 99 pgs: 99 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-13T14:40:23.212 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:22 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-13T14:40:23.212 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:22 smithi096 ceph-mon[102928]: osdmap e118: 8 total, 8 up, 7 in 2024-09-13T14:40:23.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:22 smithi204 ceph-mon[87071]: pgmap v271: 99 pgs: 99 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-13T14:40:23.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:22 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-13T14:40:23.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:22 smithi204 ceph-mon[87071]: osdmap e118: 8 total, 8 up, 7 in 2024-09-13T14:40:23.285 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:23.318 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:23.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:23.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:23.420 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:23.455 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:23.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:23.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:23.557 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:23.593 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:23.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:23.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:23.695 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:23.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 31 15 65631 65616 132.265 91 0.0093834 0.00755591 2024-09-13T14:40:23.735 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:23.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:23.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:23.838 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:23.874 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:23.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:23.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:23.976 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:24.009 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:24.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:24.111 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:24.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.148 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:24.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:24.162 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:23 smithi096 ceph-mon[98901]: 1.0 scrub starts 2024-09-13T14:40:24.162 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:23 smithi096 ceph-mon[98901]: 1.0 scrub ok 2024-09-13T14:40:24.162 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:23 smithi096 ceph-mon[98901]: osdmap e119: 8 total, 8 up, 7 in 2024-09-13T14:40:24.163 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:23 smithi096 ceph-mon[102928]: 1.0 scrub starts 2024-09-13T14:40:24.163 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:23 smithi096 ceph-mon[102928]: 1.0 scrub ok 2024-09-13T14:40:24.163 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:23 smithi096 ceph-mon[102928]: osdmap e119: 8 total, 8 up, 7 in 2024-09-13T14:40:24.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:23 smithi204 ceph-mon[87071]: 1.0 scrub starts 2024-09-13T14:40:24.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:23 smithi204 ceph-mon[87071]: 1.0 scrub ok 2024-09-13T14:40:24.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:23 smithi204 ceph-mon[87071]: osdmap e119: 8 total, 8 up, 7 in 2024-09-13T14:40:24.249 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:24.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.294 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:24.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:24.395 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:24.431 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:24.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:24.533 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:24.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.563 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:24.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:24.664 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:24.698 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:24.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:24.712 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 32 16 67045 67029 130.891 88.3125 0.0156423 0.00763507 2024-09-13T14:40:24.801 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:24.833 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:24.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:24.935 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:24.971 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:24.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:25.072 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:25.106 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:25.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:25.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:25.175 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:40:25.175 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:40:25.176 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T14:40:25.176 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:40:25.208 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:25.211 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:24 smithi096 ceph-mon[98901]: 1.7 scrub starts 2024-09-13T14:40:25.211 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:24 smithi096 ceph-mon[98901]: 1.7 scrub ok 2024-09-13T14:40:25.211 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:24 smithi096 ceph-mon[98901]: 1.6 scrub starts 2024-09-13T14:40:25.211 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:24 smithi096 ceph-mon[98901]: 1.6 scrub ok 2024-09-13T14:40:25.211 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:24 smithi096 ceph-mon[98901]: pgmap v274: 99 pgs: 99 active+clean; 3.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T14:40:25.211 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:24 smithi096 ceph-mon[98901]: 1.3 scrub starts 2024-09-13T14:40:25.211 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:24 smithi096 ceph-mon[98901]: 1.3 scrub ok 2024-09-13T14:40:25.211 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:24 smithi096 ceph-mon[102928]: 1.7 scrub starts 2024-09-13T14:40:25.211 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:24 smithi096 ceph-mon[102928]: 1.7 scrub ok 2024-09-13T14:40:25.211 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:24 smithi096 ceph-mon[102928]: 1.6 scrub starts 2024-09-13T14:40:25.212 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:24 smithi096 ceph-mon[102928]: 1.6 scrub ok 2024-09-13T14:40:25.212 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:24 smithi096 ceph-mon[102928]: pgmap v274: 99 pgs: 99 active+clean; 3.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T14:40:25.212 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:24 smithi096 ceph-mon[102928]: 1.3 scrub starts 2024-09-13T14:40:25.212 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:24 smithi096 ceph-mon[102928]: 1.3 scrub ok 2024-09-13T14:40:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:24 smithi204 ceph-mon[87071]: 1.7 scrub starts 2024-09-13T14:40:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:24 smithi204 ceph-mon[87071]: 1.7 scrub ok 2024-09-13T14:40:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:24 smithi204 ceph-mon[87071]: 1.6 scrub starts 2024-09-13T14:40:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:24 smithi204 ceph-mon[87071]: 1.6 scrub ok 2024-09-13T14:40:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:24 smithi204 ceph-mon[87071]: pgmap v274: 99 pgs: 99 active+clean; 3.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T14:40:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:24 smithi204 ceph-mon[87071]: 1.3 scrub starts 2024-09-13T14:40:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:24 smithi204 ceph-mon[87071]: 1.3 scrub ok 2024-09-13T14:40:25.229 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:25.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:25.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:25.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:25.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:25.368 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:25.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:25.469 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:25.505 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:25.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:25.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:25.607 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:25.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:25.629 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:25.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:25.712 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 33 15 68144 68129 129.008 68.75 0.0147351 0.00774632 2024-09-13T14:40:25.731 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:25.762 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:25.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:25.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:25.864 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:25.892 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:25.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:25.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:25.994 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:26.015 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:26.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:26.117 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:26.143 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:26.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:26.245 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:26.279 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:26.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:26.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:25 smithi096 ceph-mon[98901]: 1.4 scrub starts 2024-09-13T14:40:26.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:26 smithi096 ceph-mon[98901]: 1.4 scrub ok 2024-09-13T14:40:26.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:25 smithi096 ceph-mon[102928]: 1.4 scrub starts 2024-09-13T14:40:26.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:26 smithi096 ceph-mon[102928]: 1.4 scrub ok 2024-09-13T14:40:26.381 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:26.409 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:26.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:26.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:25 smithi204 ceph-mon[87071]: 1.4 scrub starts 2024-09-13T14:40:26.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:25 smithi204 ceph-mon[87071]: 1.4 scrub ok 2024-09-13T14:40:26.511 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:26.542 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:26.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:26.644 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:26.684 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:26.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:26.712 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 34 16 69454 69438 127.62 81.8125 0.00617414 0.00783041 2024-09-13T14:40:26.787 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:26.818 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:26.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:26.920 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:26.950 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:26.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:27.052 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:27.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:27.092 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:27.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:27.190 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:40:27.192 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:27.231 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:26 smithi096 ceph-mon[98901]: pgmap v275: 99 pgs: 99 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:40:27.232 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:26 smithi096 ceph-mon[98901]: 1.a scrub starts 2024-09-13T14:40:27.232 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:26 smithi096 ceph-mon[98901]: 1.a scrub ok 2024-09-13T14:40:27.233 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:26 smithi096 ceph-mon[102928]: pgmap v275: 99 pgs: 99 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:40:27.234 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:26 smithi096 ceph-mon[102928]: 1.a scrub starts 2024-09-13T14:40:27.234 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:26 smithi096 ceph-mon[102928]: 1.a scrub ok 2024-09-13T14:40:27.271 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:27.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:27.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:27.373 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:27.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:27.397 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:27.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:27.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:26 smithi204 ceph-mon[87071]: pgmap v275: 99 pgs: 99 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:40:27.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:26 smithi204 ceph-mon[87071]: 1.a scrub starts 2024-09-13T14:40:27.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:26 smithi204 ceph-mon[87071]: 1.a scrub ok 2024-09-13T14:40:27.498 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:27.528 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:27.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:27.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:27.630 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:27.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:27.667 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:27.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:27.712 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 35 16 70809 70793 126.393 84.6875 0.00955521 0.00790708 2024-09-13T14:40:27.768 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:27.808 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:27.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:27.910 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:27.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:27.939 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:27.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:28.040 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:28.071 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:40:28.072 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:40:28.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:28.091 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:28.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:28.106 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:40:28.107 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:40:28.142 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:40:28.142 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:40:28.174 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:40:28.175 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:40:28.175 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:27 smithi096 ceph-mon[98901]: 1.1 scrub starts 2024-09-13T14:40:28.175 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:27 smithi096 ceph-mon[98901]: 1.1 scrub ok 2024-09-13T14:40:28.175 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:27 smithi096 ceph-mon[102928]: 1.1 scrub starts 2024-09-13T14:40:28.176 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:27 smithi096 ceph-mon[102928]: 1.1 scrub ok 2024-09-13T14:40:28.176 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:40:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:40:27] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-13T14:40:28.192 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:28.204 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:40:28.204 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:40:28.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:28.234 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:28.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:28.236 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:40:28.236 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:40:28.275 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:40:28.276 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:40:28.307 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:40:28.307 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:40:28.335 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:28.372 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:28.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:28.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:28.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:27 smithi204 ceph-mon[87071]: 1.1 scrub starts 2024-09-13T14:40:28.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:27 smithi204 ceph-mon[87071]: 1.1 scrub ok 2024-09-13T14:40:28.474 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:28.513 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:28.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:28.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:28.614 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:28.641 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:28.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:28.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:28.712 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 36 16 72106 72090 125.134 81.0625 0.00899849 0.00798648 2024-09-13T14:40:28.743 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:28.783 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:28.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:28.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:28.885 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:28.919 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:28.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:28.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:29.001 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:40:29.021 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:29.053 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:29.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:29.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:29.155 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:29.190 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:29.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:29.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:29.292 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:29 smithi096 ceph-mon[98901]: pgmap v276: 99 pgs: 99 active+clean; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-13T14:40:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:29 smithi096 ceph-mon[98901]: 1.8 scrub starts 2024-09-13T14:40:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:29 smithi096 ceph-mon[98901]: 1.8 scrub ok 2024-09-13T14:40:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:29 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3989448479' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:29.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:29 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3747725473' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:40:29.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:29 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:40:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:29 smithi096 ceph-mon[102928]: pgmap v276: 99 pgs: 99 active+clean; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-13T14:40:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:29 smithi096 ceph-mon[102928]: 1.8 scrub starts 2024-09-13T14:40:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:29 smithi096 ceph-mon[102928]: 1.8 scrub ok 2024-09-13T14:40:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:29 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3989448479' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:29 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3747725473' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:40:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:29 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:40:29.330 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:29.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:29.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:29.433 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:29.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:29 smithi204 ceph-mon[87071]: pgmap v276: 99 pgs: 99 active+clean; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-13T14:40:29.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:29 smithi204 ceph-mon[87071]: 1.8 scrub starts 2024-09-13T14:40:29.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:29 smithi204 ceph-mon[87071]: 1.8 scrub ok 2024-09-13T14:40:29.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:29 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3989448479' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:29.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:29 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3747725473' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:40:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:29 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:40:29.471 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:29.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:29.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:29.573 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:29.606 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:29.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:29.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:29.709 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:29.712 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 37 16 73553 73537 124.196 90.4375 0.00724477 0.0080469 2024-09-13T14:40:29.744 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:29.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:29.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:29.846 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:29.870 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:29.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:29.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:29.901 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:29.911 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:29.972 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:30.002 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:30.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:30.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:30.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:30 smithi096 ceph-mon[98901]: 1.2 scrub starts 2024-09-13T14:40:30.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:30 smithi096 ceph-mon[98901]: 1.2 scrub ok 2024-09-13T14:40:30.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:30 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:40:30.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:30 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:40:30.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:30 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:40:30.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:30 smithi096 ceph-mon[98901]: osdmap e120: 8 total, 8 up, 7 in 2024-09-13T14:40:30.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:30 smithi096 ceph-mon[102928]: 1.2 scrub starts 2024-09-13T14:40:30.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:30 smithi096 ceph-mon[102928]: 1.2 scrub ok 2024-09-13T14:40:30.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:30 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:40:30.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:30 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:40:30.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:30 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:40:30.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:30 smithi096 ceph-mon[102928]: osdmap e120: 8 total, 8 up, 7 in 2024-09-13T14:40:30.104 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:30.136 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:30.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:30.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:30.238 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:30.267 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:30.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:30.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:30.369 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:30.402 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:30.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:30.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:30.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:30 smithi204 ceph-mon[87071]: 1.2 scrub starts 2024-09-13T14:40:30.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:30 smithi204 ceph-mon[87071]: 1.2 scrub ok 2024-09-13T14:40:30.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:30 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:40:30.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:30 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:40:30.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:30 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:40:30.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:30 smithi204 ceph-mon[87071]: osdmap e120: 8 total, 8 up, 7 in 2024-09-13T14:40:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:40:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:40:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:30.505 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:30.547 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:30.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:30.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:30.649 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:30.685 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:30.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:30.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:30.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 38 16 74949 74933 123.224 87.25 0.00685317 0.00811043 2024-09-13T14:40:30.787 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:30.814 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:30.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:30.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:30.916 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:30.949 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:30.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:30.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:31.051 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:31.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:31.081 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:31.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:31.123 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:40:31.182 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:31.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:31.211 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:31.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:31.312 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:31.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:31 smithi096 ceph-mon[98901]: pgmap v278: 99 pgs: 99 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-13T14:40:31.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:31 smithi096 ceph-mon[98901]: 1.5 deep-scrub starts 2024-09-13T14:40:31.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:31 smithi096 ceph-mon[98901]: 1.5 deep-scrub ok 2024-09-13T14:40:31.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:31 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:31.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:31 smithi096 ceph-mon[102928]: pgmap v278: 99 pgs: 99 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-13T14:40:31.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:31 smithi096 ceph-mon[102928]: 1.5 deep-scrub starts 2024-09-13T14:40:31.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:31 smithi096 ceph-mon[102928]: 1.5 deep-scrub ok 2024-09-13T14:40:31.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:31 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:31.339 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:31.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:31.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:31.441 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:31.468 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:31.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:31 smithi204 ceph-mon[87071]: pgmap v278: 99 pgs: 99 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-13T14:40:31.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:31 smithi204 ceph-mon[87071]: 1.5 deep-scrub starts 2024-09-13T14:40:31.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:31 smithi204 ceph-mon[87071]: 1.5 deep-scrub ok 2024-09-13T14:40:31.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:31 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:31.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:31.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:31.570 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:31.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:31.614 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:31.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:31.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 39 16 76337 76321 122.288 86.75 0.00572185 0.00817296 2024-09-13T14:40:31.715 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:31.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:31.746 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:31.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:31.848 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:31.879 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:40:31.884 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:31.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:31.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:31.985 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:32.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:32.014 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:32.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:32.115 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:32.143 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:32.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:32.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:32.245 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:32.278 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:32.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:32.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:32.380 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:32.416 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:32.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:32.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:32.518 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:32.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:32.556 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:32.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:32.657 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:32.684 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:32.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:32.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:32.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:40:32.713185+0000 min lat: 0.000998702 max lat: 0.18515 avg lat: 0.00824057 2024-09-13T14:40:32.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:40:32.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 40 16 77589 77573 121.187 78.25 0.00628676 0.00824057 2024-09-13T14:40:32.786 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:32.824 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:32.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:32.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:32.928 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:32.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:32.958 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:32.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:33.059 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:33.098 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:33.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:33.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:33.285 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:33.323 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:33.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:33 smithi096 ceph-mon[98901]: pgmap v279: 99 pgs: 99 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-13T14:40:33.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:33 smithi096 ceph-mon[102928]: pgmap v279: 99 pgs: 99 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-13T14:40:33.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:33.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:33.425 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:33.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:33.447 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:33.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:33 smithi204 ceph-mon[87071]: pgmap v279: 99 pgs: 99 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-13T14:40:33.549 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:33.585 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:33.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:33.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:33.687 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:33.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 41 16 78736 78720 119.98 71.6875 0.012099 0.00832992 2024-09-13T14:40:33.719 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:33.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:33.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:33.821 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:33.850 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:33.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:33.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:33.952 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:33.973 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:33.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:34.075 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:34.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:34.114 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:34.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:34.215 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:34.247 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:34.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:34.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:34.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:34 smithi096 ceph-mon[98901]: 1.9 scrub starts 2024-09-13T14:40:34.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:34 smithi096 ceph-mon[98901]: 1.9 scrub ok 2024-09-13T14:40:34.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:34 smithi096 ceph-mon[102928]: 1.9 scrub starts 2024-09-13T14:40:34.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:34 smithi096 ceph-mon[102928]: 1.9 scrub ok 2024-09-13T14:40:34.349 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:34.385 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:34.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:34.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:34.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:34 smithi204 ceph-mon[87071]: 1.9 scrub starts 2024-09-13T14:40:34.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:34 smithi204 ceph-mon[87071]: 1.9 scrub ok 2024-09-13T14:40:34.488 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:34.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:34.512 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:34.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:34.614 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:34.635 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:34.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:34.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:34.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 42 16 79674 79658 118.519 58.625 0.00410721 0.00843258 2024-09-13T14:40:34.737 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:34.771 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:34.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:34.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:34.873 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:34.904 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:34.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:34.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:35.006 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:35.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:35.082 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:35.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:35.183 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:35.226 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:35.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:35.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:35.231 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:35 smithi096 ceph-mon[98901]: pgmap v280: 99 pgs: 99 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 87 MiB/s wr, 1.38k op/s 2024-09-13T14:40:35.231 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:35 smithi096 ceph-mon[102928]: pgmap v280: 99 pgs: 99 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 87 MiB/s wr, 1.38k op/s 2024-09-13T14:40:35.328 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:35.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:35.363 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:35.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:35.464 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:35.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:35 smithi204 ceph-mon[87071]: pgmap v280: 99 pgs: 99 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 87 MiB/s wr, 1.38k op/s 2024-09-13T14:40:35.495 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:35.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:35.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:35.597 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:35.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:35.635 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:35.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:35.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 43 16 80911 80895 117.561 77.3125 0.00881868 0.00850114 2024-09-13T14:40:35.737 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:35.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:35.774 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:35.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:35.875 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:35.903 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:35.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:35.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:36.005 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:36.030 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:36.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:36.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:36.061 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:36.066 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:36.132 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:36.171 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:36.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:36.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:36.272 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:36.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:36.333 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:36.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:36.434 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:36.465 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:36.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:36.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:36.567 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:36.594 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:36.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:36.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:36.696 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:36.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 44 16 81993 81977 116.425 67.625 0.0103649 0.0085839 2024-09-13T14:40:36.726 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:36.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:36.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:36.827 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:36.852 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:36.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:36.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:36.954 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:36.993 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:36.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:36.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:37.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:36 smithi096 ceph-mon[98901]: pgmap v281: 99 pgs: 99 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 84 MiB/s wr, 1.33k op/s 2024-09-13T14:40:37.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:37.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:37.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:36 smithi096 ceph-mon[102928]: pgmap v281: 99 pgs: 99 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 84 MiB/s wr, 1.33k op/s 2024-09-13T14:40:37.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:37.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:37.095 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:37.130 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:37.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:37.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:37.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:36 smithi204 ceph-mon[87071]: pgmap v281: 99 pgs: 99 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 84 MiB/s wr, 1.33k op/s 2024-09-13T14:40:37.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:37.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:37.232 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:37.258 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:37.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:37.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:37.360 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:37.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:37.396 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:37.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:37.497 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:37.524 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:37.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:37.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:37.556 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:40:37.558 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:40:37.626 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:37.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:37.666 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:37.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:37.714 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 45 16 83087 83071 115.357 68.375 0.0119601 0.00866288 2024-09-13T14:40:37.766 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:37.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:37.817 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:37.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:37.918 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:37.951 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:37.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:37.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:38.053 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:38.082 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:38.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:38.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:38.184 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:38.217 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:38.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:38.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:38.320 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:38.321 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:40:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:40:37] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.51.0" 2024-09-13T14:40:38.364 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":281,"stamp":"2024-09-13T14:40:35.639613+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5069924723,"num_objects":77315,"num_object_clones":0,"num_object_copies":231945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":77315,"num_whiteouts":0,"num_read":6026,"num_read_kb":13080,"num_write":78207,"num_write_kb":4959139,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":4311310,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79419,"ondisk_log_size":79419,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15814920,"kb_used_data":14650380,"kb_used_omap":34,"kb_used_meta":1164445,"kb_avail":640401144,"statfs":{"total":671965249536,"available":655770771456,"internally_reserved":0,"allocated":15001989120,"data_stored":14997270147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35126,"internal_metadata":1192392394},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":882823590,"num_objects":13470,"num_object_clones":0,"num_object_copies":40410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13470,"num_whiteouts":0,"num_read":-277,"num_read_kb":-689,"num_write":13334,"num_write_kb":860600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-4,"num_bytes_recovered":-1019727,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005127"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":104,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.012747+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:40:29.012747+0000","last_peered":"2024-09-13T14:40:29.012747+0000","last_clean":"2024-09-13T14:40:29.012747+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:40:29.012747+0000","last_undegraded":"2024-09-13T14:40:29.012747+0000","last_fullsized":"2024-09-13T14:40:29.012747+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":86,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.940615+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:40:22.940615+0000","last_peered":"2024-09-13T14:40:22.940615+0000","last_clean":"2024-09-13T14:40:22.940615+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:40:22.940615+0000","last_undegraded":"2024-09-13T14:40:22.940615+0000","last_fullsized":"2024-09-13T14:40:22.940615+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":84,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.938603+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:40:22.938603+0000","last_peered":"2024-09-13T14:40:22.938603+0000","last_clean":"2024-09-13T14:40:22.938603+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:40:22.938603+0000","last_undegraded":"2024-09-13T14:40:22.938603+0000","last_fullsized":"2024-09-13T14:40:22.938603+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":97,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.003975+0000","last_change":"2024-09-13T14:38:13.977954+0000","last_active":"2024-09-13T14:40:29.003975+0000","last_peered":"2024-09-13T14:40:29.003975+0000","last_clean":"2024-09-13T14:40:29.003975+0000","last_became_active":"2024-09-13T14:38:13.977619+0000","last_became_peered":"2024-09-13T14:38:13.977619+0000","last_unstale":"2024-09-13T14:40:29.003975+0000","last_undegraded":"2024-09-13T14:40:29.003975+0000","last_fullsized":"2024-09-13T14:40:29.003975+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:58:30.259967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":97,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.007797+0000","last_change":"2024-09-13T14:38:13.978232+0000","last_active":"2024-09-13T14:40:29.007797+0000","last_peered":"2024-09-13T14:40:29.007797+0000","last_clean":"2024-09-13T14:40:29.007797+0000","last_became_active":"2024-09-13T14:38:13.977921+0000","last_became_peered":"2024-09-13T14:38:13.977921+0000","last_unstale":"2024-09-13T14:40:29.007797+0000","last_undegraded":"2024-09-13T14:40:29.007797+0000","last_fullsized":"2024-09-13T14:40:29.007797+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:10.127626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":77,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.939872+0000","last_change":"2024-09-13T14:39:42.765301+0000","last_active":"2024-09-13T14:40:22.939872+0000","last_peered":"2024-09-13T14:40:22.939872+0000","last_clean":"2024-09-13T14:40:22.939872+0000","last_became_active":"2024-09-13T14:39:42.764977+0000","last_became_peered":"2024-09-13T14:39:42.764977+0000","last_unstale":"2024-09-13T14:40:22.939872+0000","last_undegraded":"2024-09-13T14:40:22.939872+0000","last_fullsized":"2024-09-13T14:40:22.939872+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:25.223972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.007660+0000","last_change":"2024-09-13T14:37:40.925691+0000","last_active":"2024-09-13T14:40:29.007660+0000","last_peered":"2024-09-13T14:40:29.007660+0000","last_clean":"2024-09-13T14:40:29.007660+0000","last_became_active":"2024-09-13T14:37:40.925360+0000","last_became_peered":"2024-09-13T14:37:40.925360+0000","last_unstale":"2024-09-13T14:40:29.007660+0000","last_undegraded":"2024-09-13T14:40:29.007660+0000","last_fullsized":"2024-09-13T14:40:29.007660+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:07.337967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":58,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.001596+0000","last_change":"2024-09-13T14:39:42.781872+0000","last_active":"2024-09-13T14:40:29.001596+0000","last_peered":"2024-09-13T14:40:29.001596+0000","last_clean":"2024-09-13T14:40:29.001596+0000","last_became_active":"2024-09-13T14:39:42.770254+0000","last_became_peered":"2024-09-13T14:39:42.770254+0000","last_unstale":"2024-09-13T14:40:29.001596+0000","last_undegraded":"2024-09-13T14:40:29.001596+0000","last_fullsized":"2024-09-13T14:40:29.001596+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:31.211081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.9","version":"120'4711","reported_seq":4732,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:31.573185+0000","last_change":"2024-09-13T14:39:42.764535+0000","last_active":"2024-09-13T14:40:31.573185+0000","last_peered":"2024-09-13T14:40:31.573185+0000","last_clean":"2024-09-13T14:40:31.573185+0000","last_became_active":"2024-09-13T14:39:42.764101+0000","last_became_peered":"2024-09-13T14:39:42.764101+0000","last_unstale":"2024-09-13T14:40:31.573185+0000","last_undegraded":"2024-09-13T14:40:31.573185+0000","last_fullsized":"2024-09-13T14:40:31.573185+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":4711,"log_dups_size":0,"ondisk_log_size":4711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:22.498969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308740096,"num_objects":4711,"num_object_clones":0,"num_object_copies":14133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4711,"num_write_kb":301504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":88,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.006225+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:40:29.006225+0000","last_peered":"2024-09-13T14:40:29.006225+0000","last_clean":"2024-09-13T14:40:29.006225+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:40:29.006225+0000","last_undegraded":"2024-09-13T14:40:29.006225+0000","last_fullsized":"2024-09-13T14:40:29.006225+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":77,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.938763+0000","last_change":"2024-09-13T14:39:42.764876+0000","last_active":"2024-09-13T14:40:22.938763+0000","last_peered":"2024-09-13T14:40:22.938763+0000","last_clean":"2024-09-13T14:40:22.938763+0000","last_became_active":"2024-09-13T14:39:42.764082+0000","last_became_peered":"2024-09-13T14:39:42.764082+0000","last_unstale":"2024-09-13T14:40:22.938763+0000","last_undegraded":"2024-09-13T14:40:22.938763+0000","last_fullsized":"2024-09-13T14:40:22.938763+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:10:07.458952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"120'4825","reported_seq":4862,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:33.644704+0000","last_change":"2024-09-13T14:39:42.767715+0000","last_active":"2024-09-13T14:40:33.644704+0000","last_peered":"2024-09-13T14:40:33.644704+0000","last_clean":"2024-09-13T14:40:33.644704+0000","last_became_active":"2024-09-13T14:39:42.766678+0000","last_became_peered":"2024-09-13T14:39:42.766678+0000","last_unstale":"2024-09-13T14:40:33.644704+0000","last_undegraded":"2024-09-13T14:40:33.644704+0000","last_fullsized":"2024-09-13T14:40:33.644704+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":4825,"log_dups_size":0,"ondisk_log_size":4825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:30:38.645181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316211200,"num_objects":4825,"num_object_clones":0,"num_object_copies":14475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4825,"num_write_kb":308800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":86,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.941955+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:40:22.941955+0000","last_peered":"2024-09-13T14:40:22.941955+0000","last_clean":"2024-09-13T14:40:22.941955+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:40:22.941955+0000","last_undegraded":"2024-09-13T14:40:22.941955+0000","last_fullsized":"2024-09-13T14:40:22.941955+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":111,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.937490+0000","last_change":"2024-09-13T14:38:13.979677+0000","last_active":"2024-09-13T14:40:22.937490+0000","last_peered":"2024-09-13T14:40:22.937490+0000","last_clean":"2024-09-13T14:40:22.937490+0000","last_became_active":"2024-09-13T14:38:13.979323+0000","last_became_peered":"2024-09-13T14:38:13.979323+0000","last_unstale":"2024-09-13T14:40:22.937490+0000","last_undegraded":"2024-09-13T14:40:22.937490+0000","last_fullsized":"2024-09-13T14:40:22.937490+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:29:03.491254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"120'4947","reported_seq":4984,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:33.623337+0000","last_change":"2024-09-13T14:39:42.766942+0000","last_active":"2024-09-13T14:40:33.623337+0000","last_peered":"2024-09-13T14:40:33.623337+0000","last_clean":"2024-09-13T14:40:33.623337+0000","last_became_active":"2024-09-13T14:39:42.766647+0000","last_became_peered":"2024-09-13T14:39:42.766647+0000","last_unstale":"2024-09-13T14:40:33.623337+0000","last_undegraded":"2024-09-13T14:40:33.623337+0000","last_fullsized":"2024-09-13T14:40:33.623337+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":4947,"log_dups_size":0,"ondisk_log_size":4947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:16.267901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324206592,"num_objects":4947,"num_object_clones":0,"num_object_copies":14841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4947,"num_write_kb":316608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":82,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.941545+0000","last_change":"2024-09-13T14:39:42.774547+0000","last_active":"2024-09-13T14:40:22.941545+0000","last_peered":"2024-09-13T14:40:22.941545+0000","last_clean":"2024-09-13T14:40:22.941545+0000","last_became_active":"2024-09-13T14:39:42.766006+0000","last_became_peered":"2024-09-13T14:39:42.766006+0000","last_unstale":"2024-09-13T14:40:22.941545+0000","last_undegraded":"2024-09-13T14:40:22.941545+0000","last_fullsized":"2024-09-13T14:40:22.941545+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:06:29.632163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":87,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.001792+0000","last_change":"2024-09-13T14:39:42.765763+0000","last_active":"2024-09-13T14:40:29.001792+0000","last_peered":"2024-09-13T14:40:29.001792+0000","last_clean":"2024-09-13T14:40:29.001792+0000","last_became_active":"2024-09-13T14:39:42.765441+0000","last_became_peered":"2024-09-13T14:39:42.765441+0000","last_unstale":"2024-09-13T14:40:29.001792+0000","last_undegraded":"2024-09-13T14:40:29.001792+0000","last_fullsized":"2024-09-13T14:40:29.001792+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:06.483638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.a","version":"120'5041","reported_seq":5077,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:33.653742+0000","last_change":"2024-09-13T14:39:42.763571+0000","last_active":"2024-09-13T14:40:33.653742+0000","last_peered":"2024-09-13T14:40:33.653742+0000","last_clean":"2024-09-13T14:40:33.653742+0000","last_became_active":"2024-09-13T14:39:42.763270+0000","last_became_peered":"2024-09-13T14:39:42.763270+0000","last_unstale":"2024-09-13T14:40:33.653742+0000","last_undegraded":"2024-09-13T14:40:33.653742+0000","last_fullsized":"2024-09-13T14:40:33.653742+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":5041,"log_dups_size":0,"ondisk_log_size":5041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:38:09.281466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330366976,"num_objects":5041,"num_object_clones":0,"num_object_copies":15123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5041,"num_write_kb":322624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":95,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.938806+0000","last_change":"2024-09-13T14:39:42.765558+0000","last_active":"2024-09-13T14:40:22.938806+0000","last_peered":"2024-09-13T14:40:22.938806+0000","last_clean":"2024-09-13T14:40:22.938806+0000","last_became_active":"2024-09-13T14:39:42.765246+0000","last_became_peered":"2024-09-13T14:39:42.765246+0000","last_unstale":"2024-09-13T14:40:22.938806+0000","last_undegraded":"2024-09-13T14:40:22.938806+0000","last_fullsized":"2024-09-13T14:40:22.938806+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:08.640610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":98,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.007475+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:40:29.007475+0000","last_peered":"2024-09-13T14:40:29.007475+0000","last_clean":"2024-09-13T14:40:29.007475+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:40:29.007475+0000","last_undegraded":"2024-09-13T14:40:29.007475+0000","last_fullsized":"2024-09-13T14:40:29.007475+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":237,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.950975+0000","last_change":"2024-09-13T14:40:22.950254+0000","last_active":"2024-09-13T14:40:22.950975+0000","last_peered":"2024-09-13T14:40:22.950975+0000","last_clean":"2024-09-13T14:40:22.950975+0000","last_became_active":"2024-09-13T14:40:22.949830+0000","last_became_peered":"2024-09-13T14:40:22.949830+0000","last_unstale":"2024-09-13T14:40:22.950975+0000","last_undegraded":"2024-09-13T14:40:22.950975+0000","last_fullsized":"2024-09-13T14:40:22.950975+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:24:20.875473+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":28,"num_read_kb":69,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":107339,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.d","version":"120'4917","reported_seq":4954,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:33.648784+0000","last_change":"2024-09-13T14:39:42.759038+0000","last_active":"2024-09-13T14:40:33.648784+0000","last_peered":"2024-09-13T14:40:33.648784+0000","last_clean":"2024-09-13T14:40:33.648784+0000","last_became_active":"2024-09-13T14:39:42.758753+0000","last_became_peered":"2024-09-13T14:39:42.758753+0000","last_unstale":"2024-09-13T14:40:33.648784+0000","last_undegraded":"2024-09-13T14:40:33.648784+0000","last_fullsized":"2024-09-13T14:40:33.648784+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":4917,"log_dups_size":0,"ondisk_log_size":4917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:14:07.557396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322240512,"num_objects":4917,"num_object_clones":0,"num_object_copies":14751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4917,"num_write_kb":314688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":101,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.012283+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:40:29.012283+0000","last_peered":"2024-09-13T14:40:29.012283+0000","last_clean":"2024-09-13T14:40:29.012283+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:40:29.012283+0000","last_undegraded":"2024-09-13T14:40:29.012283+0000","last_fullsized":"2024-09-13T14:40:29.012283+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":94,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.938035+0000","last_change":"2024-09-13T14:39:42.775584+0000","last_active":"2024-09-13T14:40:22.938035+0000","last_peered":"2024-09-13T14:40:22.938035+0000","last_clean":"2024-09-13T14:40:22.938035+0000","last_became_active":"2024-09-13T14:39:42.775254+0000","last_became_peered":"2024-09-13T14:39:42.775254+0000","last_unstale":"2024-09-13T14:40:22.938035+0000","last_undegraded":"2024-09-13T14:40:22.938035+0000","last_fullsized":"2024-09-13T14:40:22.938035+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:44.607098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":28,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:27.868475+0000","last_change":"2024-09-13T14:40:27.868402+0000","last_active":"2024-09-13T14:40:27.868475+0000","last_peered":"2024-09-13T14:40:27.868475+0000","last_clean":"2024-09-13T14:40:27.868475+0000","last_became_active":"2024-09-13T14:40:22.966301+0000","last_became_peered":"2024-09-13T14:40:22.966301+0000","last_unstale":"2024-09-13T14:40:27.868475+0000","last_undegraded":"2024-09-13T14:40:27.868475+0000","last_fullsized":"2024-09-13T14:40:27.868475+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:18.660186+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.c","version":"120'4809","reported_seq":4846,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:31.883496+0000","last_change":"2024-09-13T14:39:42.772481+0000","last_active":"2024-09-13T14:40:31.883496+0000","last_peered":"2024-09-13T14:40:31.883496+0000","last_clean":"2024-09-13T14:40:31.883496+0000","last_became_active":"2024-09-13T14:39:42.772211+0000","last_became_peered":"2024-09-13T14:39:42.772211+0000","last_unstale":"2024-09-13T14:40:31.883496+0000","last_undegraded":"2024-09-13T14:40:31.883496+0000","last_fullsized":"2024-09-13T14:40:31.883496+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":4809,"log_dups_size":0,"ondisk_log_size":4809,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:30:16.860807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315162624,"num_objects":4809,"num_object_clones":0,"num_object_copies":14427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4809,"num_write_kb":307776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":76,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.939235+0000","last_change":"2024-09-13T14:39:42.776751+0000","last_active":"2024-09-13T14:40:22.939235+0000","last_peered":"2024-09-13T14:40:22.939235+0000","last_clean":"2024-09-13T14:40:22.939235+0000","last_became_active":"2024-09-13T14:39:42.774855+0000","last_became_peered":"2024-09-13T14:39:42.774855+0000","last_unstale":"2024-09-13T14:40:22.939235+0000","last_undegraded":"2024-09-13T14:40:22.939235+0000","last_fullsized":"2024-09-13T14:40:22.939235+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:11.022013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":101,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.937990+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:40:22.937990+0000","last_peered":"2024-09-13T14:40:22.937990+0000","last_clean":"2024-09-13T14:40:22.937990+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:40:22.937990+0000","last_undegraded":"2024-09-13T14:40:22.937990+0000","last_fullsized":"2024-09-13T14:40:22.937990+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"120'4722","reported_seq":4743,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:31.565393+0000","last_change":"2024-09-13T14:39:42.764380+0000","last_active":"2024-09-13T14:40:31.565393+0000","last_peered":"2024-09-13T14:40:31.565393+0000","last_clean":"2024-09-13T14:40:31.565393+0000","last_became_active":"2024-09-13T14:39:42.764127+0000","last_became_peered":"2024-09-13T14:39:42.764127+0000","last_unstale":"2024-09-13T14:40:31.565393+0000","last_undegraded":"2024-09-13T14:40:31.565393+0000","last_fullsized":"2024-09-13T14:40:31.565393+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":4722,"log_dups_size":0,"ondisk_log_size":4722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:37:32.713989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309460992,"num_objects":4722,"num_object_clones":0,"num_object_copies":14166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4722,"num_write_kb":302208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":84,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.002187+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:40:29.002187+0000","last_peered":"2024-09-13T14:40:29.002187+0000","last_clean":"2024-09-13T14:40:29.002187+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:40:29.002187+0000","last_undegraded":"2024-09-13T14:40:29.002187+0000","last_fullsized":"2024-09-13T14:40:29.002187+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":94,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.938237+0000","last_change":"2024-09-13T14:39:42.776187+0000","last_active":"2024-09-13T14:40:22.938237+0000","last_peered":"2024-09-13T14:40:22.938237+0000","last_clean":"2024-09-13T14:40:22.938237+0000","last_became_active":"2024-09-13T14:39:42.775751+0000","last_became_peered":"2024-09-13T14:39:42.775751+0000","last_unstale":"2024-09-13T14:40:22.938237+0000","last_undegraded":"2024-09-13T14:40:22.938237+0000","last_fullsized":"2024-09-13T14:40:22.938237+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:35.155717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":21,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:25.861330+0000","last_change":"2024-09-13T14:40:25.861254+0000","last_active":"2024-09-13T14:40:25.861330+0000","last_peered":"2024-09-13T14:40:25.861330+0000","last_clean":"2024-09-13T14:40:25.861330+0000","last_became_active":"2024-09-13T14:40:22.957614+0000","last_became_peered":"2024-09-13T14:40:22.957614+0000","last_unstale":"2024-09-13T14:40:25.861330+0000","last_undegraded":"2024-09-13T14:40:25.861330+0000","last_fullsized":"2024-09-13T14:40:25.861330+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:53.388190+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.e","version":"120'4884","reported_seq":4903,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:31.550632+0000","last_change":"2024-09-13T14:39:42.772007+0000","last_active":"2024-09-13T14:40:31.550632+0000","last_peered":"2024-09-13T14:40:31.550632+0000","last_clean":"2024-09-13T14:40:31.550632+0000","last_became_active":"2024-09-13T14:39:42.771672+0000","last_became_peered":"2024-09-13T14:39:42.771672+0000","last_unstale":"2024-09-13T14:40:31.550632+0000","last_undegraded":"2024-09-13T14:40:31.550632+0000","last_fullsized":"2024-09-13T14:40:31.550632+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":4884,"log_dups_size":0,"ondisk_log_size":4884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:26:12.134635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320077824,"num_objects":4884,"num_object_clones":0,"num_object_copies":14652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4884,"num_write_kb":312576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":105,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.017628+0000","last_change":"2024-09-13T14:38:13.981576+0000","last_active":"2024-09-13T14:40:29.017628+0000","last_peered":"2024-09-13T14:40:29.017628+0000","last_clean":"2024-09-13T14:40:29.017628+0000","last_became_active":"2024-09-13T14:38:13.981262+0000","last_became_peered":"2024-09-13T14:38:13.981262+0000","last_unstale":"2024-09-13T14:40:29.017628+0000","last_undegraded":"2024-09-13T14:40:29.017628+0000","last_fullsized":"2024-09-13T14:40:29.017628+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:20.284250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":85,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.005307+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:40:29.005307+0000","last_peered":"2024-09-13T14:40:29.005307+0000","last_clean":"2024-09-13T14:40:29.005307+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:40:29.005307+0000","last_undegraded":"2024-09-13T14:40:29.005307+0000","last_fullsized":"2024-09-13T14:40:29.005307+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.4","version":"120'4779","reported_seq":4800,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:31.551125+0000","last_change":"2024-09-13T14:39:42.765514+0000","last_active":"2024-09-13T14:40:31.551125+0000","last_peered":"2024-09-13T14:40:31.551125+0000","last_clean":"2024-09-13T14:40:31.551125+0000","last_became_active":"2024-09-13T14:39:42.765209+0000","last_became_peered":"2024-09-13T14:39:42.765209+0000","last_unstale":"2024-09-13T14:40:31.551125+0000","last_undegraded":"2024-09-13T14:40:31.551125+0000","last_fullsized":"2024-09-13T14:40:31.551125+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":4779,"log_dups_size":0,"ondisk_log_size":4779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:59.785672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313196544,"num_objects":4779,"num_object_clones":0,"num_object_copies":14337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4779,"num_write_kb":305856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":238,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.939269+0000","last_change":"2024-09-13T14:40:22.639864+0000","last_active":"2024-09-13T14:40:22.939269+0000","last_peered":"2024-09-13T14:40:22.939269+0000","last_clean":"2024-09-13T14:40:22.939269+0000","last_became_active":"2024-09-13T14:40:20.926503+0000","last_became_peered":"2024-09-13T14:40:20.926503+0000","last_unstale":"2024-09-13T14:40:22.939269+0000","last_undegraded":"2024-09-13T14:40:22.939269+0000","last_fullsized":"2024-09-13T14:40:22.939269+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:22.639760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:22.639760+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:08.523563+0000","scrub_duration":0.0051120610000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":647,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.938769+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:40:22.938769+0000","last_peered":"2024-09-13T14:40:22.938769+0000","last_clean":"2024-09-13T14:40:22.938769+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:40:22.938769+0000","last_undegraded":"2024-09-13T14:40:22.938769+0000","last_fullsized":"2024-09-13T14:40:22.938769+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":62,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.012281+0000","last_change":"2024-09-13T14:39:42.768341+0000","last_active":"2024-09-13T14:40:29.012281+0000","last_peered":"2024-09-13T14:40:29.012281+0000","last_clean":"2024-09-13T14:40:29.012281+0000","last_became_active":"2024-09-13T14:39:42.768034+0000","last_became_peered":"2024-09-13T14:39:42.768034+0000","last_unstale":"2024-09-13T14:40:29.012281+0000","last_undegraded":"2024-09-13T14:40:29.012281+0000","last_fullsized":"2024-09-13T14:40:29.012281+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:28:55.502108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":26,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.937830+0000","last_change":"2024-09-13T14:39:42.772096+0000","last_active":"2024-09-13T14:40:22.937830+0000","last_peered":"2024-09-13T14:40:22.937830+0000","last_clean":"2024-09-13T14:40:22.937830+0000","last_became_active":"2024-09-13T14:39:42.771762+0000","last_became_peered":"2024-09-13T14:39:42.771762+0000","last_unstale":"2024-09-13T14:40:22.937830+0000","last_undegraded":"2024-09-13T14:40:22.937830+0000","last_fullsized":"2024-09-13T14:40:22.937830+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:16.172571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":213,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.001620+0000","last_change":"2024-09-13T14:40:23.784403+0000","last_active":"2024-09-13T14:40:29.001620+0000","last_peered":"2024-09-13T14:40:29.001620+0000","last_clean":"2024-09-13T14:40:29.001620+0000","last_became_active":"2024-09-13T14:40:22.951246+0000","last_became_peered":"2024-09-13T14:40:22.951246+0000","last_unstale":"2024-09-13T14:40:29.001620+0000","last_undegraded":"2024-09-13T14:40:29.001620+0000","last_fullsized":"2024-09-13T14:40:29.001620+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.784263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.784263+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:49.551336+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.7","version":"120'4704","reported_seq":4725,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:31.560125+0000","last_change":"2024-09-13T14:39:42.773740+0000","last_active":"2024-09-13T14:40:31.560125+0000","last_peered":"2024-09-13T14:40:31.560125+0000","last_clean":"2024-09-13T14:40:31.560125+0000","last_became_active":"2024-09-13T14:39:42.773444+0000","last_became_peered":"2024-09-13T14:39:42.773444+0000","last_unstale":"2024-09-13T14:40:31.560125+0000","last_undegraded":"2024-09-13T14:40:31.560125+0000","last_fullsized":"2024-09-13T14:40:31.560125+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":4704,"log_dups_size":0,"ondisk_log_size":4704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:09:16.690630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308281344,"num_objects":4704,"num_object_clones":0,"num_object_copies":14112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4704,"num_write_kb":301056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":331,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.939169+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:40:22.939169+0000","last_peered":"2024-09-13T14:40:22.939169+0000","last_clean":"2024-09-13T14:40:22.939169+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:40:22.939169+0000","last_undegraded":"2024-09-13T14:40:22.939169+0000","last_fullsized":"2024-09-13T14:40:22.939169+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":105,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.002622+0000","last_change":"2024-09-13T14:38:13.977962+0000","last_active":"2024-09-13T14:40:29.002622+0000","last_peered":"2024-09-13T14:40:29.002622+0000","last_clean":"2024-09-13T14:40:29.002622+0000","last_became_active":"2024-09-13T14:38:13.977638+0000","last_became_peered":"2024-09-13T14:38:13.977638+0000","last_unstale":"2024-09-13T14:40:29.002622+0000","last_undegraded":"2024-09-13T14:40:29.002622+0000","last_fullsized":"2024-09-13T14:40:29.002622+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:53:16.902157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":109,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.007962+0000","last_change":"2024-09-13T14:37:40.927444+0000","last_active":"2024-09-13T14:40:29.007962+0000","last_peered":"2024-09-13T14:40:29.007962+0000","last_clean":"2024-09-13T14:40:29.007962+0000","last_became_active":"2024-09-13T14:37:40.927140+0000","last_became_peered":"2024-09-13T14:37:40.927140+0000","last_unstale":"2024-09-13T14:40:29.007962+0000","last_undegraded":"2024-09-13T14:40:29.007962+0000","last_fullsized":"2024-09-13T14:40:29.007962+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:32:54.207568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":247,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:28.603036+0000","last_change":"2024-09-13T14:40:28.602999+0000","last_active":"2024-09-13T14:40:28.603036+0000","last_peered":"2024-09-13T14:40:28.603036+0000","last_clean":"2024-09-13T14:40:28.603036+0000","last_became_active":"2024-09-13T14:40:22.949002+0000","last_became_peered":"2024-09-13T14:40:22.949002+0000","last_unstale":"2024-09-13T14:40:28.603036+0000","last_undegraded":"2024-09-13T14:40:28.603036+0000","last_fullsized":"2024-09-13T14:40:28.603036+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:28.602942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:28.602942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:29:46.037596+0000","scrub_duration":0.00945176,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.6","version":"120'4871","reported_seq":4908,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:33.659826+0000","last_change":"2024-09-13T14:39:42.759808+0000","last_active":"2024-09-13T14:40:33.659826+0000","last_peered":"2024-09-13T14:40:33.659826+0000","last_clean":"2024-09-13T14:40:33.659826+0000","last_became_active":"2024-09-13T14:39:42.759199+0000","last_became_peered":"2024-09-13T14:39:42.759199+0000","last_unstale":"2024-09-13T14:40:33.659826+0000","last_undegraded":"2024-09-13T14:40:33.659826+0000","last_fullsized":"2024-09-13T14:40:33.659826+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":4871,"log_dups_size":0,"ondisk_log_size":4871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:55:55.573511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319225856,"num_objects":4871,"num_object_clones":0,"num_object_copies":14613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4871,"num_write_kb":311744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":509,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.005923+0000","last_change":"2024-09-13T14:38:13.978679+0000","last_active":"2024-09-13T14:40:29.005923+0000","last_peered":"2024-09-13T14:40:29.005923+0000","last_clean":"2024-09-13T14:40:29.005923+0000","last_became_active":"2024-09-13T14:38:13.978171+0000","last_became_peered":"2024-09-13T14:38:13.978171+0000","last_unstale":"2024-09-13T14:40:29.005923+0000","last_undegraded":"2024-09-13T14:40:29.005923+0000","last_fullsized":"2024-09-13T14:40:29.005923+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:57.691198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":92,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.001875+0000","last_change":"2024-09-13T14:39:42.769691+0000","last_active":"2024-09-13T14:40:29.001875+0000","last_peered":"2024-09-13T14:40:29.001875+0000","last_clean":"2024-09-13T14:40:29.001875+0000","last_became_active":"2024-09-13T14:39:42.769266+0000","last_became_peered":"2024-09-13T14:39:42.769266+0000","last_unstale":"2024-09-13T14:40:29.001875+0000","last_undegraded":"2024-09-13T14:40:29.001875+0000","last_fullsized":"2024-09-13T14:40:29.001875+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:11.637732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":118,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.005219+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:40:29.005219+0000","last_peered":"2024-09-13T14:40:29.005219+0000","last_clean":"2024-09-13T14:40:29.005219+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:40:29.005219+0000","last_undegraded":"2024-09-13T14:40:29.005219+0000","last_fullsized":"2024-09-13T14:40:29.005219+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":21,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:26.896755+0000","last_change":"2024-09-13T14:40:26.896691+0000","last_active":"2024-09-13T14:40:26.896755+0000","last_peered":"2024-09-13T14:40:26.896755+0000","last_clean":"2024-09-13T14:40:26.896755+0000","last_became_active":"2024-09-13T14:40:22.949772+0000","last_became_peered":"2024-09-13T14:40:22.949772+0000","last_unstale":"2024-09-13T14:40:26.896755+0000","last_undegraded":"2024-09-13T14:40:26.896755+0000","last_fullsized":"2024-09-13T14:40:26.896755+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:26.896613+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:26.896613+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:01.615433+0000","scrub_duration":0.004120828,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.5","version":"120'4844","reported_seq":4881,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:33.022683+0000","last_change":"2024-09-13T14:39:42.776210+0000","last_active":"2024-09-13T14:40:33.022683+0000","last_peered":"2024-09-13T14:40:33.022683+0000","last_clean":"2024-09-13T14:40:33.022683+0000","last_became_active":"2024-09-13T14:39:42.775823+0000","last_became_peered":"2024-09-13T14:39:42.775823+0000","last_unstale":"2024-09-13T14:40:33.022683+0000","last_undegraded":"2024-09-13T14:40:33.022683+0000","last_fullsized":"2024-09-13T14:40:33.022683+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":4844,"log_dups_size":0,"ondisk_log_size":4844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:39.020681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317456384,"num_objects":4844,"num_object_clones":0,"num_object_copies":14532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4844,"num_write_kb":310016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":711,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.939113+0000","last_change":"2024-09-13T14:39:42.805055+0000","last_active":"2024-09-13T14:40:22.939113+0000","last_peered":"2024-09-13T14:40:22.939113+0000","last_clean":"2024-09-13T14:40:22.939113+0000","last_became_active":"2024-09-13T14:39:42.762456+0000","last_became_peered":"2024-09-13T14:39:42.762456+0000","last_unstale":"2024-09-13T14:40:22.939113+0000","last_undegraded":"2024-09-13T14:40:22.939113+0000","last_fullsized":"2024-09-13T14:40:22.939113+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:54:40.839991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":95,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.937674+0000","last_change":"2024-09-13T14:39:42.775159+0000","last_active":"2024-09-13T14:40:22.937674+0000","last_peered":"2024-09-13T14:40:22.937674+0000","last_clean":"2024-09-13T14:40:22.937674+0000","last_became_active":"2024-09-13T14:39:42.774841+0000","last_became_peered":"2024-09-13T14:39:42.774841+0000","last_unstale":"2024-09-13T14:40:22.937674+0000","last_undegraded":"2024-09-13T14:40:22.937674+0000","last_fullsized":"2024-09-13T14:40:22.937674+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:35:45.579397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":96,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.002519+0000","last_change":"2024-09-13T14:38:13.981294+0000","last_active":"2024-09-13T14:40:29.002519+0000","last_peered":"2024-09-13T14:40:29.002519+0000","last_clean":"2024-09-13T14:40:29.002519+0000","last_became_active":"2024-09-13T14:38:13.980975+0000","last_became_peered":"2024-09-13T14:38:13.980975+0000","last_unstale":"2024-09-13T14:40:29.002519+0000","last_undegraded":"2024-09-13T14:40:29.002519+0000","last_fullsized":"2024-09-13T14:40:29.002519+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:17.545179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":22,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.012965+0000","last_change":"2024-09-13T14:40:23.093425+0000","last_active":"2024-09-13T14:40:29.012965+0000","last_peered":"2024-09-13T14:40:29.012965+0000","last_clean":"2024-09-13T14:40:29.012965+0000","last_became_active":"2024-09-13T14:40:22.947358+0000","last_became_peered":"2024-09-13T14:40:22.947358+0000","last_unstale":"2024-09-13T14:40:29.012965+0000","last_undegraded":"2024-09-13T14:40:29.012965+0000","last_fullsized":"2024-09-13T14:40:29.012965+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.093356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.093356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:11:36.050022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.3","version":"120'4850","reported_seq":4887,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:33.017456+0000","last_change":"2024-09-13T14:39:42.770363+0000","last_active":"2024-09-13T14:40:33.017456+0000","last_peered":"2024-09-13T14:40:33.017456+0000","last_clean":"2024-09-13T14:40:33.017456+0000","last_became_active":"2024-09-13T14:39:42.770092+0000","last_became_peered":"2024-09-13T14:39:42.770092+0000","last_unstale":"2024-09-13T14:40:33.017456+0000","last_undegraded":"2024-09-13T14:40:33.017456+0000","last_fullsized":"2024-09-13T14:40:33.017456+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":4850,"log_dups_size":0,"ondisk_log_size":4850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:20:20.375252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317849600,"num_objects":4850,"num_object_clones":0,"num_object_copies":14550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4850,"num_write_kb":310400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":102,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.938764+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:40:22.938764+0000","last_peered":"2024-09-13T14:40:22.938764+0000","last_clean":"2024-09-13T14:40:22.938764+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:40:22.938764+0000","last_undegraded":"2024-09-13T14:40:22.938764+0000","last_fullsized":"2024-09-13T14:40:22.938764+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":602,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.002685+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:40:29.002685+0000","last_peered":"2024-09-13T14:40:29.002685+0000","last_clean":"2024-09-13T14:40:29.002685+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:40:29.002685+0000","last_undegraded":"2024-09-13T14:40:29.002685+0000","last_fullsized":"2024-09-13T14:40:29.002685+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":74,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.002156+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:40:29.002156+0000","last_peered":"2024-09-13T14:40:29.002156+0000","last_clean":"2024-09-13T14:40:29.002156+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:40:29.002156+0000","last_undegraded":"2024-09-13T14:40:29.002156+0000","last_fullsized":"2024-09-13T14:40:29.002156+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":168,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.007467+0000","last_change":"2024-09-13T14:40:23.334441+0000","last_active":"2024-09-13T14:40:29.007467+0000","last_peered":"2024-09-13T14:40:29.007467+0000","last_clean":"2024-09-13T14:40:29.007467+0000","last_became_active":"2024-09-13T14:40:22.952908+0000","last_became_peered":"2024-09-13T14:40:22.952908+0000","last_unstale":"2024-09-13T14:40:29.007467+0000","last_undegraded":"2024-09-13T14:40:29.007467+0000","last_fullsized":"2024-09-13T14:40:29.007467+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.334379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.334379+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:23:47.680793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"120'4896","reported_seq":4933,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:33.034168+0000","last_change":"2024-09-13T14:39:42.770399+0000","last_active":"2024-09-13T14:40:33.034168+0000","last_peered":"2024-09-13T14:40:33.034168+0000","last_clean":"2024-09-13T14:40:33.034168+0000","last_became_active":"2024-09-13T14:39:42.770074+0000","last_became_peered":"2024-09-13T14:39:42.770074+0000","last_unstale":"2024-09-13T14:40:33.034168+0000","last_undegraded":"2024-09-13T14:40:33.034168+0000","last_fullsized":"2024-09-13T14:40:33.034168+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":4896,"log_dups_size":0,"ondisk_log_size":4896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:28:03.030813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320864256,"num_objects":4896,"num_object_clones":0,"num_object_copies":14688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4896,"num_write_kb":313344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":85,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.002607+0000","last_change":"2024-09-13T14:37:40.921861+0000","last_active":"2024-09-13T14:40:29.002607+0000","last_peered":"2024-09-13T14:40:29.002607+0000","last_clean":"2024-09-13T14:40:29.002607+0000","last_became_active":"2024-09-13T14:37:40.921587+0000","last_became_peered":"2024-09-13T14:37:40.921587+0000","last_unstale":"2024-09-13T14:40:29.002607+0000","last_undegraded":"2024-09-13T14:40:29.002607+0000","last_fullsized":"2024-09-13T14:40:29.002607+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:24.619352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":333,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.941622+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:40:22.941622+0000","last_peered":"2024-09-13T14:40:22.941622+0000","last_clean":"2024-09-13T14:40:22.941622+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:40:22.941622+0000","last_undegraded":"2024-09-13T14:40:22.941622+0000","last_fullsized":"2024-09-13T14:40:22.941622+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":87,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.007279+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:40:29.007279+0000","last_peered":"2024-09-13T14:40:29.007279+0000","last_clean":"2024-09-13T14:40:29.007279+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:40:29.007279+0000","last_undegraded":"2024-09-13T14:40:29.007279+0000","last_fullsized":"2024-09-13T14:40:29.007279+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":21,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:30.500145+0000","last_change":"2024-09-13T14:40:30.500108+0000","last_active":"2024-09-13T14:40:30.500145+0000","last_peered":"2024-09-13T14:40:30.500145+0000","last_clean":"2024-09-13T14:40:30.500145+0000","last_became_active":"2024-09-13T14:40:22.949882+0000","last_became_peered":"2024-09-13T14:40:22.949882+0000","last_unstale":"2024-09-13T14:40:30.500145+0000","last_undegraded":"2024-09-13T14:40:30.500145+0000","last_fullsized":"2024-09-13T14:40:30.500145+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:40:30.500046+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:22:58.480202+0000","scrub_duration":0.00248832,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1","version":"120'4698","reported_seq":4719,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:31.529523+0000","last_change":"2024-09-13T14:39:42.772158+0000","last_active":"2024-09-13T14:40:31.529523+0000","last_peered":"2024-09-13T14:40:31.529523+0000","last_clean":"2024-09-13T14:40:31.529523+0000","last_became_active":"2024-09-13T14:39:42.771189+0000","last_became_peered":"2024-09-13T14:39:42.771189+0000","last_unstale":"2024-09-13T14:40:31.529523+0000","last_undegraded":"2024-09-13T14:40:31.529523+0000","last_fullsized":"2024-09-13T14:40:31.529523+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":4698,"log_dups_size":0,"ondisk_log_size":4698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:58:08.882288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307888128,"num_objects":4698,"num_object_clones":0,"num_object_copies":14094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4698,"num_write_kb":300672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":101,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.013101+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:40:29.013101+0000","last_peered":"2024-09-13T14:40:29.013101+0000","last_clean":"2024-09-13T14:40:29.013101+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:40:29.013101+0000","last_undegraded":"2024-09-13T14:40:29.013101+0000","last_fullsized":"2024-09-13T14:40:29.013101+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":423,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.002738+0000","last_change":"2024-09-13T14:39:44.990216+0000","last_active":"2024-09-13T14:40:29.002738+0000","last_peered":"2024-09-13T14:40:29.002738+0000","last_clean":"2024-09-13T14:40:29.002738+0000","last_became_active":"2024-09-13T14:39:44.989433+0000","last_became_peered":"2024-09-13T14:39:44.989433+0000","last_unstale":"2024-09-13T14:40:29.002738+0000","last_undegraded":"2024-09-13T14:40:29.002738+0000","last_fullsized":"2024-09-13T14:40:29.002738+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:00.426042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":98,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.018254+0000","last_change":"2024-09-13T14:38:13.979255+0000","last_active":"2024-09-13T14:40:29.018254+0000","last_peered":"2024-09-13T14:40:29.018254+0000","last_clean":"2024-09-13T14:40:29.018254+0000","last_became_active":"2024-09-13T14:38:13.978868+0000","last_became_peered":"2024-09-13T14:38:13.978868+0000","last_unstale":"2024-09-13T14:40:29.018254+0000","last_undegraded":"2024-09-13T14:40:29.018254+0000","last_fullsized":"2024-09-13T14:40:29.018254+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:15:26.003538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":27,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.012172+0000","last_change":"2024-09-13T14:40:24.124750+0000","last_active":"2024-09-13T14:40:29.012172+0000","last_peered":"2024-09-13T14:40:29.012172+0000","last_clean":"2024-09-13T14:40:29.012172+0000","last_became_active":"2024-09-13T14:40:22.958820+0000","last_became_peered":"2024-09-13T14:40:22.958820+0000","last_unstale":"2024-09-13T14:40:29.012172+0000","last_undegraded":"2024-09-13T14:40:29.012172+0000","last_fullsized":"2024-09-13T14:40:29.012172+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:24.124531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:24.124531+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:36.414537+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"120'4789","reported_seq":4826,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:31.602448+0000","last_change":"2024-09-13T14:39:42.770912+0000","last_active":"2024-09-13T14:40:31.602448+0000","last_peered":"2024-09-13T14:40:31.602448+0000","last_clean":"2024-09-13T14:40:31.602448+0000","last_became_active":"2024-09-13T14:39:42.770646+0000","last_became_peered":"2024-09-13T14:39:42.770646+0000","last_unstale":"2024-09-13T14:40:31.602448+0000","last_undegraded":"2024-09-13T14:40:31.602448+0000","last_fullsized":"2024-09-13T14:40:31.602448+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":4789,"log_dups_size":0,"ondisk_log_size":4789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:54:13.100820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313851904,"num_objects":4789,"num_object_clones":0,"num_object_copies":14367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4789,"num_write_kb":306496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":109,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.007697+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:40:29.007697+0000","last_peered":"2024-09-13T14:40:29.007697+0000","last_clean":"2024-09-13T14:40:29.007697+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:40:29.007697+0000","last_undegraded":"2024-09-13T14:40:29.007697+0000","last_fullsized":"2024-09-13T14:40:29.007697+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1037,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.941618+0000","last_change":"2024-09-13T14:39:44.990389+0000","last_active":"2024-09-13T14:40:22.941618+0000","last_peered":"2024-09-13T14:40:22.941618+0000","last_clean":"2024-09-13T14:40:22.941618+0000","last_became_active":"2024-09-13T14:39:44.989805+0000","last_became_peered":"2024-09-13T14:39:44.989805+0000","last_unstale":"2024-09-13T14:40:22.941618+0000","last_undegraded":"2024-09-13T14:40:22.941618+0000","last_fullsized":"2024-09-13T14:40:22.941618+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:15.392445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":77,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.937367+0000","last_change":"2024-09-13T14:39:42.769954+0000","last_active":"2024-09-13T14:40:22.937367+0000","last_peered":"2024-09-13T14:40:22.937367+0000","last_clean":"2024-09-13T14:40:22.937367+0000","last_became_active":"2024-09-13T14:39:42.769635+0000","last_became_peered":"2024-09-13T14:39:42.769635+0000","last_unstale":"2024-09-13T14:40:22.937367+0000","last_undegraded":"2024-09-13T14:40:22.937367+0000","last_fullsized":"2024-09-13T14:40:22.937367+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:51.588071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":94,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.937604+0000","last_change":"2024-09-13T14:39:42.775357+0000","last_active":"2024-09-13T14:40:22.937604+0000","last_peered":"2024-09-13T14:40:22.937604+0000","last_clean":"2024-09-13T14:40:22.937604+0000","last_became_active":"2024-09-13T14:39:42.775041+0000","last_became_peered":"2024-09-13T14:39:42.775041+0000","last_unstale":"2024-09-13T14:40:22.937604+0000","last_undegraded":"2024-09-13T14:40:22.937604+0000","last_fullsized":"2024-09-13T14:40:22.937604+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:44.768910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":100,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.007083+0000","last_change":"2024-09-13T14:39:42.787151+0000","last_active":"2024-09-13T14:40:29.007083+0000","last_peered":"2024-09-13T14:40:29.007083+0000","last_clean":"2024-09-13T14:40:29.007083+0000","last_became_active":"2024-09-13T14:39:42.771712+0000","last_became_peered":"2024-09-13T14:39:42.771712+0000","last_unstale":"2024-09-13T14:40:29.007083+0000","last_undegraded":"2024-09-13T14:40:29.007083+0000","last_fullsized":"2024-09-13T14:40:29.007083+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:54.216663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":92,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.002011+0000","last_change":"2024-09-13T14:39:42.772974+0000","last_active":"2024-09-13T14:40:29.002011+0000","last_peered":"2024-09-13T14:40:29.002011+0000","last_clean":"2024-09-13T14:40:29.002011+0000","last_became_active":"2024-09-13T14:39:42.771449+0000","last_became_peered":"2024-09-13T14:39:42.771449+0000","last_unstale":"2024-09-13T14:40:29.002011+0000","last_undegraded":"2024-09-13T14:40:29.002011+0000","last_fullsized":"2024-09-13T14:40:29.002011+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:46.441079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":100,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.941399+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:40:22.941399+0000","last_peered":"2024-09-13T14:40:22.941399+0000","last_clean":"2024-09-13T14:40:22.941399+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:40:22.941399+0000","last_undegraded":"2024-09-13T14:40:22.941399+0000","last_fullsized":"2024-09-13T14:40:22.941399+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":85,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.939000+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:40:22.939000+0000","last_peered":"2024-09-13T14:40:22.939000+0000","last_clean":"2024-09-13T14:40:22.939000+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:40:22.939000+0000","last_undegraded":"2024-09-13T14:40:22.939000+0000","last_fullsized":"2024-09-13T14:40:22.939000+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":62,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.012114+0000","last_change":"2024-09-13T14:39:42.767764+0000","last_active":"2024-09-13T14:40:29.012114+0000","last_peered":"2024-09-13T14:40:29.012114+0000","last_clean":"2024-09-13T14:40:29.012114+0000","last_became_active":"2024-09-13T14:39:42.767450+0000","last_became_peered":"2024-09-13T14:39:42.767450+0000","last_unstale":"2024-09-13T14:40:29.012114+0000","last_undegraded":"2024-09-13T14:40:29.012114+0000","last_fullsized":"2024-09-13T14:40:29.012114+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:47.729382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":96,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.007920+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:40:29.007920+0000","last_peered":"2024-09-13T14:40:29.007920+0000","last_clean":"2024-09-13T14:40:29.007920+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:40:29.007920+0000","last_undegraded":"2024-09-13T14:40:29.007920+0000","last_fullsized":"2024-09-13T14:40:29.007920+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":111,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.018363+0000","last_change":"2024-09-13T14:39:42.772266+0000","last_active":"2024-09-13T14:40:29.018363+0000","last_peered":"2024-09-13T14:40:29.018363+0000","last_clean":"2024-09-13T14:40:29.018363+0000","last_became_active":"2024-09-13T14:39:42.771880+0000","last_became_peered":"2024-09-13T14:39:42.771880+0000","last_unstale":"2024-09-13T14:40:29.018363+0000","last_undegraded":"2024-09-13T14:40:29.018363+0000","last_fullsized":"2024-09-13T14:40:29.018363+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:41:01.160105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":75,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.939005+0000","last_change":"2024-09-13T14:39:42.773890+0000","last_active":"2024-09-13T14:40:22.939005+0000","last_peered":"2024-09-13T14:40:22.939005+0000","last_clean":"2024-09-13T14:40:22.939005+0000","last_became_active":"2024-09-13T14:39:42.773523+0000","last_became_peered":"2024-09-13T14:39:42.773523+0000","last_unstale":"2024-09-13T14:40:22.939005+0000","last_undegraded":"2024-09-13T14:40:22.939005+0000","last_fullsized":"2024-09-13T14:40:22.939005+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:13.156261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":121,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.017727+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:40:29.017727+0000","last_peered":"2024-09-13T14:40:29.017727+0000","last_clean":"2024-09-13T14:40:29.017727+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:40:29.017727+0000","last_undegraded":"2024-09-13T14:40:29.017727+0000","last_fullsized":"2024-09-13T14:40:29.017727+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":98,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.007880+0000","last_change":"2024-09-13T14:38:13.979604+0000","last_active":"2024-09-13T14:40:29.007880+0000","last_peered":"2024-09-13T14:40:29.007880+0000","last_clean":"2024-09-13T14:40:29.007880+0000","last_became_active":"2024-09-13T14:38:13.979288+0000","last_became_peered":"2024-09-13T14:38:13.979288+0000","last_unstale":"2024-09-13T14:40:29.007880+0000","last_undegraded":"2024-09-13T14:40:29.007880+0000","last_fullsized":"2024-09-13T14:40:29.007880+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:21.031527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":85,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.001892+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:40:29.001892+0000","last_peered":"2024-09-13T14:40:29.001892+0000","last_clean":"2024-09-13T14:40:29.001892+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:40:29.001892+0000","last_undegraded":"2024-09-13T14:40:29.001892+0000","last_fullsized":"2024-09-13T14:40:29.001892+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":117,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.012499+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:40:29.012499+0000","last_peered":"2024-09-13T14:40:29.012499+0000","last_clean":"2024-09-13T14:40:29.012499+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:40:29.012499+0000","last_undegraded":"2024-09-13T14:40:29.012499+0000","last_fullsized":"2024-09-13T14:40:29.012499+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":103,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.939462+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:40:22.939462+0000","last_peered":"2024-09-13T14:40:22.939462+0000","last_clean":"2024-09-13T14:40:22.939462+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:40:22.939462+0000","last_undegraded":"2024-09-13T14:40:22.939462+0000","last_fullsized":"2024-09-13T14:40:22.939462+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":124,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.005974+0000","last_change":"2024-09-13T14:38:13.976950+0000","last_active":"2024-09-13T14:40:29.005974+0000","last_peered":"2024-09-13T14:40:29.005974+0000","last_clean":"2024-09-13T14:40:29.005974+0000","last_became_active":"2024-09-13T14:38:13.976639+0000","last_became_peered":"2024-09-13T14:38:13.976639+0000","last_unstale":"2024-09-13T14:40:29.005974+0000","last_undegraded":"2024-09-13T14:40:29.005974+0000","last_fullsized":"2024-09-13T14:40:29.005974+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:24.126890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":89,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.007182+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:40:29.007182+0000","last_peered":"2024-09-13T14:40:29.007182+0000","last_clean":"2024-09-13T14:40:29.007182+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:40:29.007182+0000","last_undegraded":"2024-09-13T14:40:29.007182+0000","last_fullsized":"2024-09-13T14:40:29.007182+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":101,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.937530+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:40:22.937530+0000","last_peered":"2024-09-13T14:40:22.937530+0000","last_clean":"2024-09-13T14:40:22.937530+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:40:22.937530+0000","last_undegraded":"2024-09-13T14:40:22.937530+0000","last_fullsized":"2024-09-13T14:40:22.937530+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":87,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.938664+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:40:22.938664+0000","last_peered":"2024-09-13T14:40:22.938664+0000","last_clean":"2024-09-13T14:40:22.938664+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:40:22.938664+0000","last_undegraded":"2024-09-13T14:40:22.938664+0000","last_fullsized":"2024-09-13T14:40:22.938664+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":101,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.939362+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:40:22.939362+0000","last_peered":"2024-09-13T14:40:22.939362+0000","last_clean":"2024-09-13T14:40:22.939362+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:40:22.939362+0000","last_undegraded":"2024-09-13T14:40:22.939362+0000","last_fullsized":"2024-09-13T14:40:22.939362+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":112,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.938682+0000","last_change":"2024-09-13T14:39:42.778413+0000","last_active":"2024-09-13T14:40:22.938682+0000","last_peered":"2024-09-13T14:40:22.938682+0000","last_clean":"2024-09-13T14:40:22.938682+0000","last_became_active":"2024-09-13T14:39:42.766624+0000","last_became_peered":"2024-09-13T14:39:42.766624+0000","last_unstale":"2024-09-13T14:40:22.938682+0000","last_undegraded":"2024-09-13T14:40:22.938682+0000","last_fullsized":"2024-09-13T14:40:22.938682+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:13.435604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":76,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.938069+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:40:22.938069+0000","last_peered":"2024-09-13T14:40:22.938069+0000","last_clean":"2024-09-13T14:40:22.938069+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:40:22.938069+0000","last_undegraded":"2024-09-13T14:40:22.938069+0000","last_fullsized":"2024-09-13T14:40:22.938069+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":117,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.013682+0000","last_change":"2024-09-13T14:39:42.767451+0000","last_active":"2024-09-13T14:40:29.013682+0000","last_peered":"2024-09-13T14:40:29.013682+0000","last_clean":"2024-09-13T14:40:29.013682+0000","last_became_active":"2024-09-13T14:39:42.767086+0000","last_became_peered":"2024-09-13T14:39:42.767086+0000","last_unstale":"2024-09-13T14:40:29.013682+0000","last_undegraded":"2024-09-13T14:40:29.013682+0000","last_fullsized":"2024-09-13T14:40:29.013682+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:02.674069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":87,"reported_epoch":119,"state":"active+clean","last_fresh":"2024-09-13T14:40:22.937203+0000","last_change":"2024-09-13T14:39:42.768653+0000","last_active":"2024-09-13T14:40:22.937203+0000","last_peered":"2024-09-13T14:40:22.937203+0000","last_clean":"2024-09-13T14:40:22.937203+0000","last_became_active":"2024-09-13T14:39:42.768333+0000","last_became_peered":"2024-09-13T14:39:42.768333+0000","last_unstale":"2024-09-13T14:40:22.937203+0000","last_undegraded":"2024-09-13T14:40:22.937203+0000","last_fullsized":"2024-09-13T14:40:22.937203+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:25.655793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":101,"reported_epoch":120,"state":"active+clean","last_fresh":"2024-09-13T14:40:29.007813+0000","last_change":"2024-09-13T14:37:40.924860+0000","last_active":"2024-09-13T14:40:29.007813+0000","last_peered":"2024-09-13T14:40:29.007813+0000","last_clean":"2024-09-13T14:40:29.007813+0000","last_became_active":"2024-09-13T14:37:40.924522+0000","last_became_peered":"2024-09-13T14:37:40.924522+0000","last_unstale":"2024-09-13T14:40:29.007813+0000","last_undegraded":"2024-09-13T14:40:29.007813+0000","last_fullsized":"2024-09-13T14:40:29.007813+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:10.340511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":5065080832,"num_objects":77287,"num_object_clones":0,"num_object_copies":231861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":77287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":77287,"num_write_kb":4946368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15122825216,"data_stored":15122825216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77287,"ondisk_log_size":77287,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":9648,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":28,"num_read_kb":69,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":107339,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397679,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1800960,"kb_used_data":1491128,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91944192,"statfs":{"total":95995035648,"available":94150852608,"internally_reserved":0,"allocated":1526915072,"data_stored":1526234578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":184683593908,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790137,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3018360,"kb_used_data":2712684,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":90726792,"statfs":{"total":95995035648,"available":92904235008,"internally_reserved":0,"allocated":2777788416,"data_stored":2777116223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986366,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2753716,"kb_used_data":2443884,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":90991436,"statfs":{"total":95995035648,"available":93175230464,"internally_reserved":0,"allocated":2502537216,"data_stored":2501864210,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598814,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1269100,"kb_used_data":1215204,"kb_used_omap":6,"kb_used_meta":53881,"kb_avail":92476052,"statfs":{"total":95995035648,"available":94695477248,"internally_reserved":0,"allocated":1244368896,"data_stored":1243704334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":55175136},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762341,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2586432,"kb_used_data":2514488,"kb_used_omap":8,"kb_used_meta":71927,"kb_avail":91158720,"statfs":{"total":95995035648,"available":93346529280,"internally_reserved":0,"allocated":2574835712,"data_stored":2574155970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":73653706},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":94,"seq":403726925867,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1846508,"kb_used_data":1800932,"kb_used_omap":7,"kb_used_meta":45560,"kb_avail":91898644,"statfs":{"total":95995035648,"available":94104211456,"internally_reserved":0,"allocated":1844154368,"data_stored":1843489455,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":46654153},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122107,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2539844,"kb_used_data":2472060,"kb_used_omap":7,"kb_used_meta":67768,"kb_avail":91205308,"statfs":{"total":95995035648,"available":93394235392,"internally_reserved":0,"allocated":2531389440,"data_stored":2530705377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":69395023},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2556166144,"data_stored":2556166144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1870921728,"data_stored":1870921728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2579103744,"data_stored":2579103744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1252261888,"data_stored":1252261888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2503344128,"data_stored":2503344128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2809331712,"data_stored":2809331712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1551695872,"data_stored":1551695872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:40:38.369 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 11 2024-09-13T14:40:38.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:38.385 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:38.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:38.430 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:40:38.430 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:40:38.487 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:38.565 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:38.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:38.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:38.666 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:38.696 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:38.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:38.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:38.714 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 46 16 84299 84283 114.496 75.75 0.0224358 0.00872913 2024-09-13T14:40:38.798 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:38.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:38.841 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:38.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:38.942 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:38.983 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:38.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:38.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:39.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:38 smithi096 ceph-mon[98901]: from='client.44960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:40:39.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:38 smithi096 ceph-mon[98901]: pgmap v282: 99 pgs: 99 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-13T14:40:39.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:39.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:38 smithi096 ceph-mon[102928]: from='client.44960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:40:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:38 smithi096 ceph-mon[102928]: pgmap v282: 99 pgs: 99 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-13T14:40:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:39.085 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:39.115 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:39.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:39.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:39.131 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:38 smithi204 ceph-mon[87071]: from='client.44960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:40:39.131 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:38 smithi204 ceph-mon[87071]: pgmap v282: 99 pgs: 99 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-13T14:40:39.131 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:39.135 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:40:39.217 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:39.257 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:39.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:39.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:39.359 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:39.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:39.386 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:39.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:39.487 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:39.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:39.535 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:39.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:39.636 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:39.687 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:39.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:39.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:39.714 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 47 16 85472 85456 113.619 73.3125 0.00913769 0.00879614 2024-09-13T14:40:39.788 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:39.826 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:39.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:39.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:39.927 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:39.949 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:39.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:40.050 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:40.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:40.087 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:40.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:40.188 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:40.214 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:40.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:40.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:40.316 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:40.360 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:40.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:40.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:40.462 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:40.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:40:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:40:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:40.499 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:40.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:40.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:40.601 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:40.653 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:40.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:40.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:40.715 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 48 16 86602 86586 112.724 70.625 0.0149442 0.00886662 2024-09-13T14:40:40.756 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:40.793 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:40.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:40.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:40.895 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:40.923 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:40.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:40.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:41.025 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:41.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:41.067 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:41.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:41.116 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:40 smithi204 ceph-mon[87071]: pgmap v283: 99 pgs: 99 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:40:41.168 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:41.202 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:41.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:41.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:41.304 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:41.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:40 smithi096 ceph-mon[98901]: pgmap v283: 99 pgs: 99 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:40:41.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:40 smithi096 ceph-mon[102928]: pgmap v283: 99 pgs: 99 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:40:41.332 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:41.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:41.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:41.434 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:41.445 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:41.458 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:41.464 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:41.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:41.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:41.566 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:41.610 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:41.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:41.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:41.712 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:41.714 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 49 16 87800 87784 111.951 74.875 0.0298309 0.00892742 2024-09-13T14:40:41.738 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:41.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:41.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:41.840 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:41.874 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:41.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:41.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:41.976 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:42.010 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:42.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:42.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:42.112 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:42.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:42.146 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:42.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:42.247 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:42.284 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:42.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:42.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:42.386 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:42.423 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:42.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:42.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:42.525 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:42.563 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:42.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:42.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:42.665 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:42.701 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:42.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:42.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:42.713 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:40:42.713 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:40:42.714 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 50 16 88998 88982 111.209 74.875 0.017036 0.0089871 2024-09-13T14:40:42.755 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:40:42.756 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:40:42.794 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:40:42.794 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:40:42.802 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:42.828 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:40:42.828 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:40:42.845 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:42.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:42.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:42.865 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:40:42.865 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:40:42.900 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:40:42.901 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:40:42.941 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:40:42.941 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:40:42.941 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:42 smithi096 ceph-mon[98901]: pgmap v284: 99 pgs: 99 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:40:42.942 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:42 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2978805812' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:42.942 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:42 smithi096 ceph-mon[102928]: pgmap v284: 99 pgs: 99 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:40:42.942 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:42 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2978805812' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:42.947 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:42.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:42 smithi204 ceph-mon[87071]: pgmap v284: 99 pgs: 99 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:40:42.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:42 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2978805812' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:42.972 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:40:42.972 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:40:42.978 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:42.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:42.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:43.080 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:43.110 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:43.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:43.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:43.212 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:43.249 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:43.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:43.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:43.351 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:43.388 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:43.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:43.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:43.490 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:43.520 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:43.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:43.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:43.622 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:43.658 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:43.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:43.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:43.714 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 51 15 90146 90131 110.436 71.8125 0.0100472 0.00904996 2024-09-13T14:40:43.760 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:43.789 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:43.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:43.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:43.891 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:43.940 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:43.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:43.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:44.042 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:44.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:44.075 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:44.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:44.176 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:44.214 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:44.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:44.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:44.300 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:40:44.316 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:44.360 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:44.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:44.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:44.463 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:44.501 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:44.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:44.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:44.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:44 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:44.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:44 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:44.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:44 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1838938644' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:40:44.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:44 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:40:44.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:44 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:44.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:44 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:44.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:44 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1838938644' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:40:44.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:44 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:40:44.603 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:44.637 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:44.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:44.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:44.717 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 52 16 91308 91292 109.703 72.5625 0.0103672 0.00910945 2024-09-13T14:40:44.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:44 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:44.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:44 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:44.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:44 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1838938644' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:40:44.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:44 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:40:44.738 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:44.787 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:44.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:44.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:44.888 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:44.918 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:44.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:44.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:45.020 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:45.065 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:45.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:45.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:45.167 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:45.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:45.206 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:45.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:45.307 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:45.348 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:45.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:45.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:45.449 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:45.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:45.493 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:45.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:45.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:45 smithi096 ceph-mon[98901]: pgmap v285: 99 pgs: 99 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T14:40:45.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:45 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:40:45.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:45 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:40:45.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:45 smithi096 ceph-mon[98901]: osdmap e121: 8 total, 8 up, 7 in 2024-09-13T14:40:45.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:45 smithi096 ceph-mon[102928]: pgmap v285: 99 pgs: 99 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T14:40:45.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:45 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:40:45.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:45 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:40:45.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:45 smithi096 ceph-mon[102928]: osdmap e121: 8 total, 8 up, 7 in 2024-09-13T14:40:45.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:45.633 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:45.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:45.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:45.717 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 53 16 92509 92493 109.05 75.0625 0.0368721 0.00916553 2024-09-13T14:40:45.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:45 smithi204 ceph-mon[87071]: pgmap v285: 99 pgs: 99 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T14:40:45.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:45 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:40:45.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:45 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:40:45.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:45 smithi204 ceph-mon[87071]: osdmap e121: 8 total, 8 up, 7 in 2024-09-13T14:40:45.735 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:45.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:45.768 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:45.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:45.869 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:45.908 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:45.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:45.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:46.009 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:46.042 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:46.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:46.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:46.144 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:46.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:46.181 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:46.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:46.282 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:46.304 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:46.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:46.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:46.406 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:46.432 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:46.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:46.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:46.506 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:40:46.506 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:40:46.506 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.506 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:93309 5.b 5:d3ced6b6:::benchmark_data_smithi096_138841_object93308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-09-13T14:40:46.506 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:46.433919+0000", 2024-09-13T14:40:46.507 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.013316062, 2024-09-13T14:40:46.507 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.013348466, 2024-09-13T14:40:46.507 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:40:46.507 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:40:46.507 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:40:46.507 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:40:46.507 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:40:46.507 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:40:46.507 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 93309 2024-09-13T14:40:46.507 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.507 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:40:46.507 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.508 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:40:46.508 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.433919+0000", 2024-09-13T14:40:46.508 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.508 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.508 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.508 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:40:46.508 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.433919+0000", 2024-09-13T14:40:46.508 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.508 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.508 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.508 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:40:46.508 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.433920+0000", 2024-09-13T14:40:46.508 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.671e-06 2024-09-13T14:40:46.509 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.509 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.509 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:40:46.509 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.433947+0000", 2024-09-13T14:40:46.509 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6251999999999999e-05 2024-09-13T14:40:46.509 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.509 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.509 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:40:46.509 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.433962+0000", 2024-09-13T14:40:46.509 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5102e-05 2024-09-13T14:40:46.509 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.509 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.509 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:40:46.510 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.433969+0000", 2024-09-13T14:40:46.510 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.5399999999999998e-06 2024-09-13T14:40:46.510 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.510 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.510 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:40:46.510 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.434050+0000", 2024-09-13T14:40:46.510 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0587000000000006e-05 2024-09-13T14:40:46.510 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.510 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.510 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:40:46.510 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.434185+0000", 2024-09-13T14:40:46.510 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013482300000000001 2024-09-13T14:40:46.510 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.510 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.510 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:40:46.511 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.434313+0000", 2024-09-13T14:40:46.511 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012843199999999999 2024-09-13T14:40:46.511 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.511 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.511 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:40:46.511 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441792+0000", 2024-09-13T14:40:46.511 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0074793749999999999 2024-09-13T14:40:46.511 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.511 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:40:46.511 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.511 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.511 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.512 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:93313 5.b 5:dd56ff68:::benchmark_data_smithi096_138841_object93312:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-09-13T14:40:46.512 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:46.438328+0000", 2024-09-13T14:40:46.512 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.008906654, 2024-09-13T14:40:46.512 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0090265829999999995, 2024-09-13T14:40:46.512 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:40:46.512 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:40:46.512 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:40:46.512 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:40:46.512 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:40:46.512 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:40:46.512 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 93313 2024-09-13T14:40:46.513 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.513 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:40:46.513 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.513 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:40:46.513 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.438328+0000", 2024-09-13T14:40:46.513 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.513 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.513 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.513 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:40:46.513 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.438328+0000", 2024-09-13T14:40:46.513 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.513 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.513 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.514 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:40:46.514 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.438329+0000", 2024-09-13T14:40:46.514 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3659999999999999e-06 2024-09-13T14:40:46.514 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.514 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.514 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:40:46.514 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.438353+0000", 2024-09-13T14:40:46.514 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3904999999999999e-05 2024-09-13T14:40:46.514 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.514 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.514 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:40:46.514 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.438368+0000", 2024-09-13T14:40:46.515 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.506e-05 2024-09-13T14:40:46.515 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.515 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.515 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:40:46.515 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.438375+0000", 2024-09-13T14:40:46.515 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.0310000000000001e-06 2024-09-13T14:40:46.515 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.515 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.515 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:40:46.515 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.438441+0000", 2024-09-13T14:40:46.515 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5136999999999997e-05 2024-09-13T14:40:46.515 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.516 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.516 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:40:46.516 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.438577+0000", 2024-09-13T14:40:46.516 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000136463 2024-09-13T14:40:46.516 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.516 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.516 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:40:46.516 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.438705+0000", 2024-09-13T14:40:46.516 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012761900000000001 2024-09-13T14:40:46.516 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.516 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.517 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:40:46.517 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.442980+0000", 2024-09-13T14:40:46.517 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0042754790000000004 2024-09-13T14:40:46.517 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.517 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:40:46.517 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.517 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.517 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.517 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:93318 5.6 5:6f7af590:::benchmark_data_smithi096_138841_object93317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-09-13T14:40:46.517 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:46.438625+0000", 2024-09-13T14:40:46.517 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0086099720000000005, 2024-09-13T14:40:46.517 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0088086150000000005, 2024-09-13T14:40:46.517 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:40:46.518 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:40:46.518 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:40:46.518 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:40:46.518 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:40:46.518 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:40:46.518 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 93318 2024-09-13T14:40:46.518 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.518 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:40:46.518 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.518 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:40:46.518 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.438625+0000", 2024-09-13T14:40:46.518 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.518 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.519 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.519 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:40:46.519 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.438625+0000", 2024-09-13T14:40:46.519 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.519 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.519 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.519 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:40:46.519 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.438626+0000", 2024-09-13T14:40:46.519 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5799999999999998e-07 2024-09-13T14:40:46.519 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.519 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.519 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:40:46.519 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.438644+0000", 2024-09-13T14:40:46.519 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7949999999999999e-05 2024-09-13T14:40:46.520 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.520 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.520 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:40:46.520 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.438654+0000", 2024-09-13T14:40:46.520 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0542e-05 2024-09-13T14:40:46.520 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.520 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.520 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:40:46.520 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.438661+0000", 2024-09-13T14:40:46.520 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5939999999999999e-06 2024-09-13T14:40:46.520 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.520 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.521 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:40:46.521 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.439646+0000", 2024-09-13T14:40:46.521 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00098500400000000004 2024-09-13T14:40:46.521 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.521 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.521 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:40:46.521 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.439759+0000", 2024-09-13T14:40:46.521 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011364799999999999 2024-09-13T14:40:46.521 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.521 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.521 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:40:46.521 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.439884+0000", 2024-09-13T14:40:46.521 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000124602 2024-09-13T14:40:46.521 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.522 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.522 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:40:46.522 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443001+0000", 2024-09-13T14:40:46.522 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0031173199999999998 2024-09-13T14:40:46.522 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.522 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:40:46.522 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.522 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.522 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.522 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:93320 5.b 5:d243b0e1:::benchmark_data_smithi096_138841_object93319:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-09-13T14:40:46.522 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:46.441418+0000", 2024-09-13T14:40:46.522 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0058169479999999997, 2024-09-13T14:40:46.522 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0060795379999999998, 2024-09-13T14:40:46.523 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:40:46.523 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:40:46.523 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:40:46.523 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:40:46.523 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:40:46.523 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:40:46.523 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 93320 2024-09-13T14:40:46.523 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.523 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:40:46.523 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.523 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:40:46.523 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441418+0000", 2024-09-13T14:40:46.523 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.524 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.524 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.524 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:40:46.524 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441418+0000", 2024-09-13T14:40:46.524 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.524 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.524 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.524 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:40:46.524 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441419+0000", 2024-09-13T14:40:46.524 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4300000000000001e-06 2024-09-13T14:40:46.524 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.524 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.524 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:40:46.525 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441440+0000", 2024-09-13T14:40:46.525 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1013e-05 2024-09-13T14:40:46.525 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.525 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.525 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:40:46.525 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441457+0000", 2024-09-13T14:40:46.525 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6631000000000001e-05 2024-09-13T14:40:46.525 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.525 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.525 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:40:46.525 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441464+0000", 2024-09-13T14:40:46.525 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.6490000000000008e-06 2024-09-13T14:40:46.525 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.525 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.526 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:40:46.526 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443147+0000", 2024-09-13T14:40:46.526 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001682085 2024-09-13T14:40:46.526 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.526 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.526 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:40:46.526 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443277+0000", 2024-09-13T14:40:46.526 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013022199999999999 2024-09-13T14:40:46.526 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.526 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.526 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:40:46.526 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443412+0000", 2024-09-13T14:40:46.526 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013499700000000001 2024-09-13T14:40:46.526 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.527 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:40:46.527 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.527 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.527 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.527 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:93321 5.a 5:5049038d:::benchmark_data_smithi096_138841_object93320:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-09-13T14:40:46.527 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:46.441483+0000", 2024-09-13T14:40:46.527 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0057517109999999996, 2024-09-13T14:40:46.527 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0060776299999999997, 2024-09-13T14:40:46.527 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:40:46.527 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:40:46.527 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:40:46.527 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:40:46.527 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:40:46.528 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:40:46.528 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 93321 2024-09-13T14:40:46.528 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.528 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:40:46.528 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.528 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:40:46.528 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441483+0000", 2024-09-13T14:40:46.528 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.528 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.528 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.528 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:40:46.528 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441483+0000", 2024-09-13T14:40:46.528 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.529 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.529 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.529 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:40:46.529 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441484+0000", 2024-09-13T14:40:46.529 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7899999999999998e-07 2024-09-13T14:40:46.529 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.529 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.529 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:40:46.529 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441503+0000", 2024-09-13T14:40:46.529 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8995999999999999e-05 2024-09-13T14:40:46.529 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.529 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.529 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:40:46.529 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441510+0000", 2024-09-13T14:40:46.530 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.046e-06 2024-09-13T14:40:46.530 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.530 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.530 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:40:46.530 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441515+0000", 2024-09-13T14:40:46.530 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0799999999999996e-06 2024-09-13T14:40:46.530 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.530 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.530 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:40:46.530 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441583+0000", 2024-09-13T14:40:46.530 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8345999999999999e-05 2024-09-13T14:40:46.530 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.530 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.530 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:40:46.531 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441701+0000", 2024-09-13T14:40:46.531 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011786900000000001 2024-09-13T14:40:46.531 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.531 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.531 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:40:46.531 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.441849+0000", 2024-09-13T14:40:46.531 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000147596 2024-09-13T14:40:46.531 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.531 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:40:46.531 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.531 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.531 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.531 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:93323 5.a 5.e5238eca (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-09-13T14:40:46.531 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:46.443872+0000", 2024-09-13T14:40:46.532 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.003363071, 2024-09-13T14:40:46.532 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0037413239999999999, 2024-09-13T14:40:46.532 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:40:46.532 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:40:46.532 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "queued for pg", 2024-09-13T14:40:46.532 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:40:46.532 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:40:46.532 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:40:46.532 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 93323 2024-09-13T14:40:46.532 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.532 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:40:46.532 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.532 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:40:46.532 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443872+0000", 2024-09-13T14:40:46.533 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.533 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.533 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.533 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:40:46.533 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443872+0000", 2024-09-13T14:40:46.533 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.533 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.533 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.533 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:40:46.533 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443873+0000", 2024-09-13T14:40:46.533 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.204e-06 2024-09-13T14:40:46.533 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.533 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.534 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:40:46.534 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443909+0000", 2024-09-13T14:40:46.534 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6321e-05 2024-09-13T14:40:46.534 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.534 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.534 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:40:46.534 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443920+0000", 2024-09-13T14:40:46.534 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0937999999999999e-05 2024-09-13T14:40:46.534 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.534 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.534 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:40:46.534 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443928+0000", 2024-09-13T14:40:46.534 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9549999999999992e-06 2024-09-13T14:40:46.534 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.535 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:40:46.535 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.535 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.535 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.535 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:93324 5.a 5.1142bc5a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-09-13T14:40:46.535 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:46.443941+0000", 2024-09-13T14:40:46.535 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0032933289999999998, 2024-09-13T14:40:46.535 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0037172160000000002, 2024-09-13T14:40:46.535 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:40:46.535 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:40:46.535 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "queued for pg", 2024-09-13T14:40:46.535 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:40:46.535 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:40:46.536 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:40:46.536 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 93324 2024-09-13T14:40:46.536 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.536 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:40:46.536 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.536 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:40:46.536 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443941+0000", 2024-09-13T14:40:46.536 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.536 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.536 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.536 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:40:46.536 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443941+0000", 2024-09-13T14:40:46.536 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.536 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.537 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.537 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:40:46.537 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443942+0000", 2024-09-13T14:40:46.537 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.1600000000000001e-07 2024-09-13T14:40:46.537 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.537 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.537 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:40:46.537 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443963+0000", 2024-09-13T14:40:46.537 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1421e-05 2024-09-13T14:40:46.537 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.537 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.537 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:40:46.537 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443970+0000", 2024-09-13T14:40:46.538 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9450000000000004e-06 2024-09-13T14:40:46.538 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.538 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.538 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:40:46.538 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.443975+0000", 2024-09-13T14:40:46.538 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2179999999999998e-06 2024-09-13T14:40:46.538 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.538 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:40:46.538 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.538 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.538 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.538 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:93322 5.4 e121/107 5:20e050cd:::benchmark_data_smithi096_138841_object93321:head v 121'5840, mlcod=121'5838)", 2024-09-13T14:40:46.538 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:46.444875+0000", 2024-09-13T14:40:46.538 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.002359318, 2024-09-13T14:40:46.539 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0028252500000000001, 2024-09-13T14:40:46.539 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:40:46.539 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:40:46.539 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:40:46.539 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:40:46.539 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.539 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:40:46.539 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.444875+0000", 2024-09-13T14:40:46.539 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.539 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.539 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.539 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:40:46.539 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.444875+0000", 2024-09-13T14:40:46.539 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.539 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.540 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.540 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:40:46.540 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.444876+0000", 2024-09-13T14:40:46.540 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0139999999999999e-06 2024-09-13T14:40:46.540 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.540 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.540 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:40:46.540 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.444916+0000", 2024-09-13T14:40:46.540 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9549999999999999e-05 2024-09-13T14:40:46.540 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.540 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.540 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:40:46.540 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.444927+0000", 2024-09-13T14:40:46.541 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1199000000000001e-05 2024-09-13T14:40:46.541 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.541 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.541 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:40:46.541 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.444934+0000", 2024-09-13T14:40:46.541 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9979999999999997e-06 2024-09-13T14:40:46.541 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.541 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.541 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:40:46.541 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.447666+0000", 2024-09-13T14:40:46.541 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027321340000000002 2024-09-13T14:40:46.541 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.541 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.541 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:40:46.542 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.447685+0000", 2024-09-13T14:40:46.542 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8448999999999999e-05 2024-09-13T14:40:46.542 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.542 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:40:46.542 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.542 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.542 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.542 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.35049.0:93318 5.6 e121/107)", 2024-09-13T14:40:46.542 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:46.444954+0000", 2024-09-13T14:40:46.542 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0022803749999999999, 2024-09-13T14:40:46.542 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.002787327, 2024-09-13T14:40:46.542 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:40:46.542 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:40:46.543 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "queued for pg", 2024-09-13T14:40:46.543 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:40:46.543 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.543 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:40:46.543 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.444954+0000", 2024-09-13T14:40:46.543 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.543 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.543 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.543 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:40:46.543 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.444954+0000", 2024-09-13T14:40:46.543 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.543 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.544 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.544 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:40:46.544 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.444955+0000", 2024-09-13T14:40:46.544 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.3000000000000002e-07 2024-09-13T14:40:46.544 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.544 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.544 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:40:46.544 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.444957+0000", 2024-09-13T14:40:46.544 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4169999999999999e-06 2024-09-13T14:40:46.544 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.544 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.544 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:40:46.544 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.444964+0000", 2024-09-13T14:40:46.545 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5729999999999996e-06 2024-09-13T14:40:46.545 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.545 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.545 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:40:46.545 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.444968+0000", 2024-09-13T14:40:46.545 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3050000000000003e-06 2024-09-13T14:40:46.545 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.545 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:40:46.545 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.545 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.545 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.545 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.35049.0:93318 5.6 e121/107)", 2024-09-13T14:40:46.545 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:46.445571+0000", 2024-09-13T14:40:46.545 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.001663763, 2024-09-13T14:40:46.546 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0022034580000000002, 2024-09-13T14:40:46.546 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:40:46.546 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:40:46.546 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "queued for pg", 2024-09-13T14:40:46.546 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:40:46.546 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.546 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:40:46.546 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.445571+0000", 2024-09-13T14:40:46.546 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.546 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.546 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.546 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:40:46.546 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.445571+0000", 2024-09-13T14:40:46.547 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.547 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.547 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.547 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:40:46.547 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.445572+0000", 2024-09-13T14:40:46.547 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0100000000000005e-07 2024-09-13T14:40:46.547 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.547 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.547 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:40:46.547 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.445576+0000", 2024-09-13T14:40:46.547 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0010000000000003e-06 2024-09-13T14:40:46.547 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.547 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.548 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:40:46.548 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.445586+0000", 2024-09-13T14:40:46.548 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.9739999999999997e-06 2024-09-13T14:40:46.548 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.548 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.548 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:40:46.548 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.445592+0000", 2024-09-13T14:40:46.548 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3929999999999996e-06 2024-09-13T14:40:46.548 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.548 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:40:46.548 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.548 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.549 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.549 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:93326 5.b 5:db9cd7e0:::benchmark_data_smithi096_138841_object93325:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-09-13T14:40:46.549 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:46.446259+0000", 2024-09-13T14:40:46.549 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.00097540699999999997, 2024-09-13T14:40:46.549 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001554868, 2024-09-13T14:40:46.549 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:40:46.549 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:40:46.549 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:40:46.549 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:40:46.549 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:40:46.549 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:40:46.549 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 93326 2024-09-13T14:40:46.549 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.550 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:40:46.550 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.550 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:40:46.550 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.446259+0000", 2024-09-13T14:40:46.550 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.550 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.550 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.550 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:40:46.550 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.446259+0000", 2024-09-13T14:40:46.550 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.550 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.550 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.550 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:40:46.551 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.446261+0000", 2024-09-13T14:40:46.551 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.234e-06 2024-09-13T14:40:46.551 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.551 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.551 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:40:46.551 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.446281+0000", 2024-09-13T14:40:46.551 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0681000000000001e-05 2024-09-13T14:40:46.551 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.551 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.551 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:40:46.551 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.446294+0000", 2024-09-13T14:40:46.551 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3078000000000001e-05 2024-09-13T14:40:46.551 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.551 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.552 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:40:46.552 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.446301+0000", 2024-09-13T14:40:46.552 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8979999999999999e-06 2024-09-13T14:40:46.552 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.552 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.552 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:40:46.552 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.447449+0000", 2024-09-13T14:40:46.552 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0011480679999999999 2024-09-13T14:40:46.552 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.552 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.552 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:40:46.552 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.447583+0000", 2024-09-13T14:40:46.553 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013407299999999999 2024-09-13T14:40:46.553 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.553 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.553 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:40:46.553 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.447713+0000", 2024-09-13T14:40:46.553 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012987599999999999 2024-09-13T14:40:46.553 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.553 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:40:46.553 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.553 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.553 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.553 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.35049.0:93321 5.a e121/107)", 2024-09-13T14:40:46.553 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:46.446333+0000", 2024-09-13T14:40:46.554 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.00090202399999999997, 2024-09-13T14:40:46.554 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0015327069999999999, 2024-09-13T14:40:46.554 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:40:46.554 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:40:46.554 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "queued for pg", 2024-09-13T14:40:46.554 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:40:46.554 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.554 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:40:46.554 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.446333+0000", 2024-09-13T14:40:46.554 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.554 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.554 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.554 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:40:46.555 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.446333+0000", 2024-09-13T14:40:46.555 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:40:46.555 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.555 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.555 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:40:46.555 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.446333+0000", 2024-09-13T14:40:46.555 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6499999999999999e-07 2024-09-13T14:40:46.555 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.555 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.555 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:40:46.555 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.446336+0000", 2024-09-13T14:40:46.555 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7350000000000001e-06 2024-09-13T14:40:46.555 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.555 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.556 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:40:46.556 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.446344+0000", 2024-09-13T14:40:46.556 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5269999999999998e-06 2024-09-13T14:40:46.556 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:40:46.556 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:40:46.556 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:40:46.556 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:46.446349+0000", 2024-09-13T14:40:46.556 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8570000000000003e-06 2024-09-13T14:40:46.556 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.556 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:40:46.556 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.556 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:40:46.557 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2024-09-13T14:40:46.557 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 12 2024-09-13T14:40:46.557 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:40:46.557 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:46.590 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:46.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:46.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:46.692 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:46.717 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 54 16 93656 93640 108.358 71.6875 0.0170988 0.00922462 2024-09-13T14:40:46.729 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:46.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:46.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:46.831 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:46.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:46.866 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:46.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:46.967 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:47.000 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:47.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:47.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:47.101 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:47.136 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:47.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:47.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:47.238 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:47.272 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:47.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:47.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:47.347 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:40:47.374 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:47.401 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:47.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:47.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:47.503 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:47.543 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:47.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:47.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:47.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:47 smithi096 ceph-mon[98901]: pgmap v287: 99 pgs: 99 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:40:47.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:47 smithi096 ceph-mon[102928]: pgmap v287: 99 pgs: 99 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:40:47.645 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:47.693 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:47.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:47.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:47.717 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 55 16 94720 94704 107.596 66.5 0.0312613 0.00928937 2024-09-13T14:40:47.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:47 smithi204 ceph-mon[87071]: pgmap v287: 99 pgs: 99 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:40:47.796 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:47.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:47.831 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:47.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:47.932 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:47.959 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:47.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:47.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:48.061 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:48.105 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:48.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:48.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:48.207 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:48.244 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:48.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:48.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:48.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:40:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:40:47] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.51.0" 2024-09-13T14:40:48.346 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:48.384 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:48.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:48.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:48.486 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:48.541 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:48.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:48.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:48.642 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:48.689 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:48.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:48.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:48.718 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 56 16 95900 95884 106.991 73.75 0.00535776 0.00934202 2024-09-13T14:40:48.790 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:48.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:48.836 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:48.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:48.937 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:48.965 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:48.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:48.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:49.067 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:49.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:49.108 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:49.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:49.209 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:49.236 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:49.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:49.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:49.338 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:49.380 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:49.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:49.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:49.482 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:49.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:49.512 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:49.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:49.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:49 smithi096 ceph-mon[98901]: pgmap v288: 99 pgs: 99 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T14:40:49.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:49 smithi096 ceph-mon[102928]: pgmap v288: 99 pgs: 99 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T14:40:49.578 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:40:49.613 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:49.686 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:49.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:49.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:49.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:49 smithi204 ceph-mon[87071]: pgmap v288: 99 pgs: 99 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T14:40:49.720 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 57 16 97033 97017 106.352 70.8125 0.00509656 0.00939816 2024-09-13T14:40:49.788 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:49.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:49.816 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:49.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:49.917 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:49.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:49.960 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:49.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:50.061 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:50.089 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:50.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:50.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:50.125 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:50.191 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:50.215 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:50.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:50.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:50.317 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:50.354 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:50.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:50.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:50.456 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:50.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:40:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:40:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:50.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:50.508 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:50.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:50.528 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:40:50.609 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:50.641 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:50.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:50.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:50.720 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 58 16 97945 97929 105.501 57 0.0176764 0.00947376 2024-09-13T14:40:50.743 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:50.790 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:50.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:50.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:50.891 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:50.931 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:50.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:50.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:50.937 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:51.033 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:51.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:51.081 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:51.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:51.182 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:51.226 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:51.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:51.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:51.299 INFO:teuthology.orchestra.run.smithi096.stderr:set pool 1 pgp_num to 11 2024-09-13T14:40:51.328 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:51.365 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:51.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:51.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:51.467 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:51 smithi204 ceph-mon[87071]: pgmap v289: 99 pgs: 99 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:40:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:51 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3102715320' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:40:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:51 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:40:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:51.506 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:51.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:51.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:51.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:51 smithi096 ceph-mon[98901]: pgmap v289: 99 pgs: 99 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:40:51.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:51 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3102715320' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:51 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:51 smithi096 ceph-mon[102928]: pgmap v289: 99 pgs: 99 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:51 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3102715320' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:51 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:51.608 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:51.643 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:51.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:51.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:51.720 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 59 16 98944 98928 104.771 62.4375 0.0061609 0.00953955 2024-09-13T14:40:51.745 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:51.788 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:51.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:51.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:51.890 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:51.931 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:51.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:51.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:52.033 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:52.079 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:52.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:52.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:52.154 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:52.181 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:52.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:52.223 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:52.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:52.324 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:52.360 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:52.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:52.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:52.462 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:52.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:52 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:40:52.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:52 smithi204 ceph-mon[87071]: osdmap e122: 8 total, 8 up, 7 in 2024-09-13T14:40:52.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:52.502 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:52.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:52.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:52 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:40:52.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:52 smithi096 ceph-mon[98901]: osdmap e122: 8 total, 8 up, 7 in 2024-09-13T14:40:52.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:52 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:40:52.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:52 smithi096 ceph-mon[102928]: osdmap e122: 8 total, 8 up, 7 in 2024-09-13T14:40:52.604 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:52.635 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:52.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:52.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:52.720 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:40:52.720697+0000 min lat: 0.000998702 max lat: 0.217899 avg lat: 0.00959401 2024-09-13T14:40:52.720 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:40:52.721 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 60 16 100052 100036 104.179 69.25 0.00428507 0.00959401 2024-09-13T14:40:52.737 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:52.770 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:52.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:52.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:52.872 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:52.899 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:52.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:52.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:53.001 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:53.030 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:53.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:53.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:53.132 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:53.168 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:53.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:53.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:53.270 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:53.315 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:53.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:53.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:53.417 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:53.450 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:53.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:53.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:53.552 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:53 smithi096 ceph-mon[98901]: pgmap v291: 99 pgs: 99 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:40:53.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:53 smithi096 ceph-mon[102928]: pgmap v291: 99 pgs: 99 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:40:53.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:53.596 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:53.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:53.697 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:53.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:53 smithi204 ceph-mon[87071]: pgmap v291: 99 pgs: 99 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:40:53.721 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 61 16 101179 101163 103.625 70.4375 0.00877968 0.00964526 2024-09-13T14:40:53.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:53.723 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:53.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:53.824 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:53.851 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:53.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:53.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:53.954 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:54.006 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:54.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:54.108 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:54.147 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:54.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:54.250 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:54.277 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:54.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:54.379 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:54.422 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:54.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:54.524 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:54.556 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:54.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:54.658 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:54.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.699 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:54.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:54.721 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 62 16 102280 102264 103.064 68.8125 0.00478548 0.00969811 2024-09-13T14:40:54.800 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:54.832 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:54.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:54.934 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:54.970 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:54.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:55.072 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:55.118 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:55.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:55.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:55.220 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:55.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:55.249 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:55.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:55.350 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:55.393 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:55.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:55.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:55.494 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:55.523 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:55.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:55.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:55.553 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:55 smithi096 ceph-mon[98901]: pgmap v292: 99 pgs: 99 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:40:55.553 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:55 smithi096 ceph-mon[102928]: pgmap v292: 99 pgs: 99 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:40:55.625 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:55.654 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:55.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:55.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:55.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:55 smithi204 ceph-mon[87071]: pgmap v292: 99 pgs: 99 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:40:55.721 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 63 16 103417 103401 102.556 71.0625 0.0154321 0.00974578 2024-09-13T14:40:55.756 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:55.761 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:40:55.761 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:40:55.762 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:40:55.762 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:40:55.762 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:40:55.787 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:55.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:55.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:55.889 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:55.931 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:55.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:55.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:56.033 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:56.066 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:56.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:56.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:56.167 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:56.201 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:56.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:56.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:56.302 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:56.335 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:56.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:56.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:56.437 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:56.479 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:56.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:56.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:56.581 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:56.613 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:56.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:56.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:56.715 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:56.721 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 64 16 104555 104539 102.065 71.125 0.0208895 0.00979276 2024-09-13T14:40:56.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:56.766 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:56.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:56.867 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:56.890 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:56.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:56.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:56.992 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:57.022 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:57.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:57.124 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:57.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.154 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:57.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:57.256 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:57.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.287 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:57.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:57.389 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:57.423 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:57.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:57.525 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:57.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.563 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:57.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:57.665 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:57.675 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:57 smithi096 ceph-mon[98901]: pgmap v293: 99 pgs: 99 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T14:40:57.675 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:57 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2274916377' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:57.676 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:57 smithi096 ceph-mon[102928]: pgmap v293: 99 pgs: 99 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T14:40:57.676 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:57 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2274916377' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:57.676 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:40:57.676 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:40:57.676 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:40:57.676 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:40:57.677 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:40:57.677 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:40:57.677 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:40:57.677 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:40:57.677 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:40:57.677 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:40:57.677 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:40:57.677 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:40:57.677 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:40:57.677 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:40:57.677 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:40:57.678 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:40:57.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.700 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:57.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:57.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:57 smithi204 ceph-mon[87071]: pgmap v293: 99 pgs: 99 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T14:40:57.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:57 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2274916377' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:57.721 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 65 16 105605 105589 101.504 65.625 0.0195812 0.00984723 2024-09-13T14:40:57.802 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:57.837 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:57.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:57.939 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:57.973 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:57.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:58.075 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:58.111 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:58.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:58.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:58.213 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:58.242 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:58.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:58.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:58.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:40:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:40:57] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.51.0" 2024-09-13T14:40:58.331 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:40:58.332 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T14:40:58.344 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:58.368 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:58.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:58.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:58.470 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:58.513 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:58.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:58.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:58.615 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:58.669 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:58.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:58.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:58.721 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 66 16 106663 106647 100.968 66.125 0.0201781 0.00989913 2024-09-13T14:40:58.772 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:58.804 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:58.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:58.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:58.906 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:58.934 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:58.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:58.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:59.036 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:59.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:59.073 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:59.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:59.175 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:59.215 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:59.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:59.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:59.317 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:59.356 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:40:59.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:59.371 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:40:59.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:59.473 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:59.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:59.501 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:59.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:40:59 smithi096 ceph-mon[98901]: pgmap v294: 99 pgs: 99 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:40:59.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:40:59 smithi096 ceph-mon[102928]: pgmap v294: 99 pgs: 99 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:40:59.602 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:59.629 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:59.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:59.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:59.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:40:59 smithi204 ceph-mon[87071]: pgmap v294: 99 pgs: 99 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:40:59.721 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 67 15 107722 107707 100.45 66.25 0.017637 0.00994995 2024-09-13T14:40:59.732 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:59.763 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:59.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:59.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:59.865 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:59.890 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:40:59.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:59.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:59.991 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:00.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:00.025 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:00.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:00.126 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:00.155 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:00.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:00.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:00.257 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:00.287 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:00.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:00.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:00.389 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:00.416 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:00.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:00.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:00.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:41:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:41:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:00.518 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:00.585 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:00.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:00.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:00.605 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:00.623 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:00.687 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:00.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:00.720 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:00.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:00.721 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 68 16 108717 108701 99.8863 62.125 0.0121472 0.0100068 2024-09-13T14:41:00.821 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:00.851 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:00.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:00.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:00.913 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:41:00.952 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:01.011 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:01.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:01.113 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:01.141 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:01.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:01.243 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:01.297 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:01.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:01.363 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:41:01.399 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:01.446 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:01.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:01.548 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:01.578 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:01.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:01.680 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:01.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:01 smithi204 ceph-mon[87071]: pgmap v295: 99 pgs: 99 active+clean; 6.4 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:41:01.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:01 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3728164220' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:01.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:01 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:01.721 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:01.722 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 69 16 109632 109616 99.2675 57.1875 0.0102064 0.0100689 2024-09-13T14:41:01.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:01.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:01 smithi096 ceph-mon[98901]: pgmap v295: 99 pgs: 99 active+clean; 6.4 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:41:01.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:01 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3728164220' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:01.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:01 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:01.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:01 smithi096 ceph-mon[102928]: pgmap v295: 99 pgs: 99 active+clean; 6.4 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:41:01.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:01 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3728164220' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:01.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:01 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:01.823 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:01.856 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:01.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:01.958 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:01.981 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:01.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:02.083 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:02.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:02.125 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:02.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:02.226 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:02.258 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:02.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:02.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:02.361 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:02.392 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:02.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:02.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:02.493 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:02.528 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:02.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:02.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:02.629 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:02.662 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:02.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:02.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:02.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:02 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:02.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:02 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:41:02.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:02 smithi204 ceph-mon[87071]: osdmap e123: 8 total, 8 up, 7 in 2024-09-13T14:41:02.721 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 70 16 110647 110631 98.7556 63.4375 0.00244501 0.010119 2024-09-13T14:41:02.764 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:02.800 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:02.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:02.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:02.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:02 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:02.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:02 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:41:02.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:02 smithi096 ceph-mon[98901]: osdmap e123: 8 total, 8 up, 7 in 2024-09-13T14:41:02.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:02 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:02.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:02 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:41:02.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:02 smithi096 ceph-mon[102928]: osdmap e123: 8 total, 8 up, 7 in 2024-09-13T14:41:02.902 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:02.949 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:02.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:02.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:03.051 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:03.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:03.091 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:03.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:03.192 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:03.213 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:03.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:03.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:03.315 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:03.348 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:03.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:03.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:03.450 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:03.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:03.494 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:03.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:03.595 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:03.623 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:03.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:03.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:03.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:03 smithi204 ceph-mon[87071]: pgmap v297: 99 pgs: 99 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:41:03.721 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 71 16 111634 111618 98.2335 61.6875 0.0152735 0.0101757 2024-09-13T14:41:03.725 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:03.762 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:03.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:03.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:03.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:03 smithi096 ceph-mon[98901]: pgmap v297: 99 pgs: 99 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:41:03.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:03 smithi096 ceph-mon[102928]: pgmap v297: 99 pgs: 99 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:41:03.865 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:03.912 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:03.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:03.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:04.014 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:04.037 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:04.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:04.139 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:04.165 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:04.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:04.267 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:04.315 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:04.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:04.417 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:04.452 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:04.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:04.554 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:04.582 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:04.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:04.684 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:04.712 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:04.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:04.722 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 72 16 112712 112696 97.8049 67.375 0.0136819 0.0102195 2024-09-13T14:41:04.814 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:04.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.849 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:04.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:04.950 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:04.988 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:04.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:05.090 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:05.131 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:05.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:05.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:05.233 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:05.270 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:05.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:05.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:05.372 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:05.402 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:05.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:05.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:05.504 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:05.542 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:05.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:05.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:05.644 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:05.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:05.701 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:05.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:05.709 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:05 smithi096 ceph-mon[98901]: pgmap v298: 99 pgs: 99 active+clean; 6.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-13T14:41:05.710 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:05 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:41:05.710 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:05 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:41:05.710 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:05 smithi096 ceph-mon[102928]: pgmap v298: 99 pgs: 99 active+clean; 6.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-13T14:41:05.710 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:05 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:41:05.710 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:05 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:41:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:05 smithi204 ceph-mon[87071]: pgmap v298: 99 pgs: 99 active+clean; 6.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-13T14:41:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:05 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:41:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:05 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:41:05.724 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 73 16 113735 113719 97.3381 63.9375 0.0188686 0.0102683 2024-09-13T14:41:05.801 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:05.850 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:05.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:05.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:05.952 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:05.975 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:05.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:05.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:06.077 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:06.123 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:06.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:06.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:06.225 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:06.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:06.259 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:06.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:06.360 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:06.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:06.419 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:06.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:06.520 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:06.565 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:06.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:06.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:06.667 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:06.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:06.695 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:06.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:06.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:06.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:41:06.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:41:06.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:41:06.724 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 74 16 114610 114594 96.7617 54.6875 0.0204978 0.0103296 2024-09-13T14:41:06.759 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:06.759 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:41:06.759 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:41:06.759 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:41:06.760 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:06.760 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:41:06.760 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:41:06.760 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:41:06.797 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:06.820 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:06.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:06.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:06.839 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:06.922 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:06.950 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:06.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:06.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:07.053 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:07.094 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:07.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:07.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:07.196 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:07.224 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:07.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:07.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:07.326 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:07.375 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:07.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:07.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:07.477 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:07.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:07.518 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:07.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:07.619 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:07.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:07.657 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:07.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:07.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:07 smithi204 ceph-mon[87071]: pgmap v299: 99 pgs: 99 active+clean; 6.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:41:07.724 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 75 16 115433 115417 96.1574 51.4375 0.0144831 0.0103936 2024-09-13T14:41:07.726 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:07 smithi096 ceph-mon[98901]: pgmap v299: 99 pgs: 99 active+clean; 6.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:41:07.726 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:07 smithi096 ceph-mon[102928]: pgmap v299: 99 pgs: 99 active+clean; 6.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:41:07.758 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:07.810 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:07.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:07.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:07.912 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:07.950 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:07.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:07.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:08.027 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:41:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:41:07] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-13T14:41:08.053 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:08.075 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:41:08.075 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:41:08.078 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:08.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:08.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:08.180 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:08.218 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:08.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:08.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:08.320 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:08.355 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:08.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:08.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:08.457 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:08.493 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:08.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:08.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:08.505 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T14:41:08.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:08.621 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:08.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:08.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:08.716 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:08 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2077798002' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:41:08.717 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:08 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:41:08.717 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:08 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2077798002' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:41:08.717 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:08 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:41:08.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:08 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2077798002' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:41:08.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:08 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:41:08.723 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:08.724 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 76 16 116410 116394 95.6957 61.0625 0.0318032 0.0104448 2024-09-13T14:41:08.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:08.765 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:08.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:08.806 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:41:08.867 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:08.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:08.897 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:08.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:08.997 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:09.030 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:09.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:09.132 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:09.163 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:09.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:09.265 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:09.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.288 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:09.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:09.389 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:09.425 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:09.527 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:09.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.567 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:09.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:09.668 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:09.705 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:09.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:09.712 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:09 smithi096 ceph-mon[102928]: pgmap v300: 99 pgs: 99 active+clean; 6.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:41:09.712 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:09 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:41:09.712 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:09 smithi096 ceph-mon[102928]: osdmap e124: 8 total, 8 up, 7 in 2024-09-13T14:41:09.712 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:09 smithi096 ceph-mon[98901]: pgmap v300: 99 pgs: 99 active+clean; 6.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:41:09.712 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:09 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:41:09.712 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:09 smithi096 ceph-mon[98901]: osdmap e124: 8 total, 8 up, 7 in 2024-09-13T14:41:09.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:09 smithi204 ceph-mon[87071]: pgmap v300: 99 pgs: 99 active+clean; 6.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:41:09.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:09 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:41:09.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:09 smithi204 ceph-mon[87071]: osdmap e124: 8 total, 8 up, 7 in 2024-09-13T14:41:09.724 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 77 16 117407 117391 95.2621 62.3125 0.0085859 0.0104926 2024-09-13T14:41:09.752 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T14:41:09.808 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:09.840 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:09.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:09.942 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:09.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.985 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:09.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:10.086 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:10.139 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:10.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:10.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:10.241 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:10.270 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:10.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:10.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:10.371 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:10.392 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:41:10.396 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:10.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:10.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:10.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:41:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:41:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:10.498 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:10.557 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:10.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:10.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:10.659 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:10.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:10.687 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:10.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:10.725 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 78 16 118365 118349 94.8073 59.875 0.013646 0.0105429 2024-09-13T14:41:10.788 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:10.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:10.838 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:10.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:10.939 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:10.986 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:10.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:10.996 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:10.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:11.097 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:11.115 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:11.155 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:11.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:11.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:11.257 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:11.287 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:11.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:11.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:11.389 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:11.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:11.422 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:11.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:11.486 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:11.522 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:11.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:11.553 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:11.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:11.655 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:11.685 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:11.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:11.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:11 smithi204 ceph-mon[87071]: pgmap v302: 99 pgs: 99 active+clean; 7.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:41:11.725 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 79 16 119347 119331 94.3841 61.375 0.016718 0.0105904 2024-09-13T14:41:11.756 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:11 smithi096 ceph-mon[98901]: pgmap v302: 99 pgs: 99 active+clean; 7.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:41:11.756 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:11 smithi096 ceph-mon[102928]: pgmap v302: 99 pgs: 99 active+clean; 7.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:41:11.788 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:11.823 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:11.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:11.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:11.925 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:11.956 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:11.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:11.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:12.058 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:12.094 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:12.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:12.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:12.195 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:12.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:12.224 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:12.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:12.325 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:12.358 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:12.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:12.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:12.460 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:12.500 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:12.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:12.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:12.602 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:12.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:12.645 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:12.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:12.726 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:41:12.725966+0000 min lat: 0.000998702 max lat: 0.217899 avg lat: 0.0106286 2024-09-13T14:41:12.726 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:41:12.726 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 80 16 120423 120407 94.0449 67.25 0.0110207 0.0106286 2024-09-13T14:41:12.746 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:12.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:12.778 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:12.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:12.879 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:12.915 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:12.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:12.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:13.017 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:13.051 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:13.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:13.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:13.154 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:13.177 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:13.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:13.279 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:13.320 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:13.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:13.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:13.422 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:13.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:13.460 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:13.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:13.561 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:13.595 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:13.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:13.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:13.697 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:13.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:13 smithi204 ceph-mon[87071]: pgmap v303: 99 pgs: 99 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 61 MiB/s wr, 973 op/s 2024-09-13T14:41:13.725 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 81 16 121457 121441 93.6817 64.625 0.00806638 0.0106691 2024-09-13T14:41:13.728 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:13.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:13.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:13.754 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:13 smithi096 ceph-mon[98901]: pgmap v303: 99 pgs: 99 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 61 MiB/s wr, 973 op/s 2024-09-13T14:41:13.754 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:13 smithi096 ceph-mon[102928]: pgmap v303: 99 pgs: 99 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 61 MiB/s wr, 973 op/s 2024-09-13T14:41:13.830 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:13.856 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:13.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:13.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:13.958 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:13.986 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:13.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:13.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:14.088 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:14.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:14.128 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:14.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:14.229 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:14.262 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:14.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:14.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:14.363 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:14.409 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:14.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:14.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:14.424 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:41:14.424 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:41:14.460 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:41:14.460 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:41:14.492 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:41:14.492 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:41:14.511 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:14.529 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:41:14.529 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:41:14.539 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:14.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:14.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:14.561 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:41:14.561 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:41:14.590 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:41:14.591 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:41:14.624 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:41:14.624 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:41:14.624 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:14 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2723583537' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:14.625 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:14 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2723583537' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:14.641 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:14.661 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:41:14.661 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:41:14.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:14.691 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:14.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:14.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:14 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2723583537' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:14.726 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 82 16 122420 122404 93.2732 60.1875 0.0107582 0.0107163 2024-09-13T14:41:14.792 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:14.836 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:14.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:14.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:14.938 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:14.974 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:14.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:14.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:15.076 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:15.109 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:15.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:15.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:15.211 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:15.243 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:15.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:15.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:15.345 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:15.383 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:15.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:15.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:15.484 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:15.539 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:15.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:15.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:15.641 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:15.681 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:15.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:15.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:15.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:15 smithi204 ceph-mon[87071]: pgmap v304: 99 pgs: 99 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 61 MiB/s wr, 971 op/s 2024-09-13T14:41:15.726 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 83 16 123468 123452 92.9386 65.5 0.00858583 0.010755 2024-09-13T14:41:15.783 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:15.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:15 smithi096 ceph-mon[98901]: pgmap v304: 99 pgs: 99 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 61 MiB/s wr, 971 op/s 2024-09-13T14:41:15.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:15 smithi096 ceph-mon[102928]: pgmap v304: 99 pgs: 99 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 61 MiB/s wr, 971 op/s 2024-09-13T14:41:15.826 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:15.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:15.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:15.928 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:15.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:15.960 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:15.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:16.061 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:16.097 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:16.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:16.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:16.199 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:16.229 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:16.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:16.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:16.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:16.370 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:16.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:16.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:16.472 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:16.505 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:16.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:16.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:16.606 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:16.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:16.634 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:16.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:16.726 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 84 16 124499 124483 92.5993 64.4375 0.0232862 0.010794 2024-09-13T14:41:16.736 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:16.763 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:16.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:16.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:16.865 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:16.910 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:16.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:16.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:17.012 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:17.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.037 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:17.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:17.138 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:17.175 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:17.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:17.278 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:17.310 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:17.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:17.412 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:17.420 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:41:17.420 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:41:17.420 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:41:17.420 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:124993 5.a e124/107 5:541398cf:::benchmark_data_smithi096_138841_object124992:head v 124'8012, mlcod=124'8010)", 2024-09-13T14:41:17.420 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:17.245835+0000", 2024-09-13T14:41:17.420 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0050681959999999996, 2024-09-13T14:41:17.420 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0050960909999999996, 2024-09-13T14:41:17.420 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:41:17.421 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:41:17.421 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:41:17.421 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:41:17.421 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:41:17.421 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:41:17.421 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:17.245835+0000", 2024-09-13T14:41:17.421 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:41:17.421 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:41:17.421 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:41:17.421 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:41:17.421 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:17.245835+0000", 2024-09-13T14:41:17.421 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:41:17.421 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:41:17.421 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:41:17.422 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:41:17.422 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:17.245836+0000", 2024-09-13T14:41:17.422 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0160000000000001e-06 2024-09-13T14:41:17.422 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:41:17.422 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:41:17.422 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:41:17.422 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:17.246589+0000", 2024-09-13T14:41:17.422 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00075361799999999995 2024-09-13T14:41:17.422 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:41:17.422 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:41:17.422 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:41:17.422 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:17.246616+0000", 2024-09-13T14:41:17.422 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6605999999999999e-05 2024-09-13T14:41:17.422 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:41:17.423 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:41:17.423 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:41:17.423 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:17.246624+0000", 2024-09-13T14:41:17.423 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9799999999999998e-06 2024-09-13T14:41:17.423 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:41:17.423 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:41:17.423 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:41:17.423 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:17.246728+0000", 2024-09-13T14:41:17.423 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010375200000000001 2024-09-13T14:41:17.423 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:41:17.423 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:41:17.423 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:41:17.423 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:17.246746+0000", 2024-09-13T14:41:17.424 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8581e-05 2024-09-13T14:41:17.424 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:41:17.424 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:41:17.424 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:41:17.424 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:41:17.424 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2024-09-13T14:41:17.424 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 1 2024-09-13T14:41:17.424 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:41:17.449 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:17.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:17.551 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:17.579 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:17.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:17.681 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:17.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:17 smithi204 ceph-mon[87071]: pgmap v305: 99 pgs: 99 active+clean; 7.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 61 MiB/s wr, 970 op/s 2024-09-13T14:41:17.726 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:17.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:17.728 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 85 16 125436 125420 92.1979 58.5625 0.0163269 0.0108416 2024-09-13T14:41:17.816 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:17 smithi096 ceph-mon[98901]: pgmap v305: 99 pgs: 99 active+clean; 7.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 61 MiB/s wr, 970 op/s 2024-09-13T14:41:17.816 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:17 smithi096 ceph-mon[102928]: pgmap v305: 99 pgs: 99 active+clean; 7.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 61 MiB/s wr, 970 op/s 2024-09-13T14:41:17.829 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:17.868 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:17.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:17.924 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:41:17.970 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:17.998 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:17.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:18.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:41:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:41:17] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-13T14:41:18.100 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:18.168 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:18.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:18.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:18.270 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:18.302 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:18.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:18.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:18.405 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:18.448 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:18.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:18.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:18.471 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:41:18.550 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:18.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:18.588 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:18.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:18.689 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:18.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:18 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2199676910' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:41:18.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:18.725 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:18.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:18.727 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 86 16 126412 126396 91.8351 61 0.0117647 0.0108844 2024-09-13T14:41:18.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:18 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2199676910' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:41:18.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:18 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2199676910' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:41:18.826 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:18.863 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:18.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:18.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:18.965 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:18.988 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:18.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:18.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:19.090 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:19.128 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:19.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:19.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:19.223 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:41:19.230 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:19.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:19.310 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:19.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:19.411 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:19.439 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:19.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:19.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:19.541 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:19.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:19.581 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:19.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:19.682 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:19.725 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:19 smithi204 ceph-mon[87071]: pgmap v306: 99 pgs: 99 active+clean; 7.5 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 61 MiB/s wr, 983 op/s 2024-09-13T14:41:19.725 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:19 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:19.725 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:19 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2199676910' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:41:19.725 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:19 smithi204 ceph-mon[87071]: osdmap e125: 8 total, 8 up, 7 in 2024-09-13T14:41:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:19.726 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:19.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:19.727 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 87 16 127314 127298 91.4275 56.375 0.00700214 0.0109324 2024-09-13T14:41:19.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:19 smithi096 ceph-mon[98901]: pgmap v306: 99 pgs: 99 active+clean; 7.5 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 61 MiB/s wr, 983 op/s 2024-09-13T14:41:19.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:19 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:19.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:19 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2199676910' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:41:19.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:19 smithi096 ceph-mon[98901]: osdmap e125: 8 total, 8 up, 7 in 2024-09-13T14:41:19.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:19 smithi096 ceph-mon[102928]: pgmap v306: 99 pgs: 99 active+clean; 7.5 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 61 MiB/s wr, 983 op/s 2024-09-13T14:41:19.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:19 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:19.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:19 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2199676910' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:41:19.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:19 smithi096 ceph-mon[102928]: osdmap e125: 8 total, 8 up, 7 in 2024-09-13T14:41:19.827 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:19.864 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:19.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:19.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:19.966 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:20.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.004 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:20.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:20.105 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:20.139 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:20.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:20.241 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:20.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.277 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:20.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:20.378 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:20.418 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:20.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:20.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:41:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:41:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:20.520 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:20.554 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:20.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:20.656 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:20.680 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:20.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:20.727 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 88 16 128267 128251 91.0654 59.5625 0.0159329 0.0109761 2024-09-13T14:41:20.782 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:20.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.814 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:20.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:20.915 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:20.947 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:20.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:21.049 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:21.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:21.083 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:21.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:21.144 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:21.184 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:21.213 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:21.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:21.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:21.315 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:21.342 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:21.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:21.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:21.444 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:21.466 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T14:41:21.474 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:21.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:21.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:21.577 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:21.604 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:21.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:21.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:21.706 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:21.727 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 89 16 129225 129209 90.715 59.875 0.010596 0.0110183 2024-09-13T14:41:21.741 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:21.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:21.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:21.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:21 smithi096 ceph-mon[98901]: pgmap v308: 99 pgs: 99 active+clean; 7.7 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 62 MiB/s wr, 991 op/s 2024-09-13T14:41:21.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:21.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:21 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/332956395' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:41:21.812 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:21 smithi096 ceph-mon[102928]: pgmap v308: 99 pgs: 99 active+clean; 7.7 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 62 MiB/s wr, 991 op/s 2024-09-13T14:41:21.812 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:21.812 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:21 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/332956395' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:41:21.844 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:21.883 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:21.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:21.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:21.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:21 smithi204 ceph-mon[87071]: pgmap v308: 99 pgs: 99 active+clean; 7.7 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 62 MiB/s wr, 991 op/s 2024-09-13T14:41:21.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:21.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:21 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/332956395' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:41:21.985 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:22.017 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:22.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:22.119 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:22.138 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:22.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.165 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:22.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:22.265 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:22.309 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:22.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:22.411 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:22.445 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:22.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:22.547 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:22.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:22 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/332956395' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:41:22.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:22 smithi096 ceph-mon[98901]: osdmap e126: 8 total, 8 up, 7 in 2024-09-13T14:41:22.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:22 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/332956395' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:41:22.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:22 smithi096 ceph-mon[102928]: osdmap e126: 8 total, 8 up, 7 in 2024-09-13T14:41:22.578 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:22.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:22.681 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:22.692 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T14:41:22.706 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:22.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:22.727 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Total time run: 90.0069 2024-09-13T14:41:22.728 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Total writes made: 130193 2024-09-13T14:41:22.728 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Write size: 65536 2024-09-13T14:41:22.728 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Object size: 65536 2024-09-13T14:41:22.728 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Bandwidth (MB/sec): 90.4049 2024-09-13T14:41:22.728 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev Bandwidth: 39.1152 2024-09-13T14:41:22.728 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max bandwidth (MB/sec): 227.5 2024-09-13T14:41:22.728 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min bandwidth (MB/sec): 51.4375 2024-09-13T14:41:22.728 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Average IOPS: 1446 2024-09-13T14:41:22.728 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev IOPS: 625.844 2024-09-13T14:41:22.728 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max IOPS: 3640 2024-09-13T14:41:22.728 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min IOPS: 823 2024-09-13T14:41:22.728 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Average Latency(s): 0.0110573 2024-09-13T14:41:22.729 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev Latency(s): 0.00864713 2024-09-13T14:41:22.729 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max latency(s): 0.217899 2024-09-13T14:41:22.729 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min latency(s): 0.000998702 2024-09-13T14:41:22.732 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:41:22.808 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:22.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.843 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:22.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:22.944 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:22.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:22 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/332956395' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:41:22.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:22 smithi204 ceph-mon[87071]: osdmap e126: 8 total, 8 up, 7 in 2024-09-13T14:41:22.971 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:22.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:23.073 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:23.103 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:23.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:23.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:23.116 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:41:23.205 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:23.267 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:23.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:23.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:23.369 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:23.398 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:23.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:23.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:23.500 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:23.531 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:23.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:23.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:23.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:23 smithi096 ceph-mon[98901]: pgmap v310: 99 pgs: 99 active+clean; 7.8 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:41:23.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:23 smithi096 ceph-mon[102928]: pgmap v310: 99 pgs: 99 active+clean; 7.8 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:41:23.633 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:23.661 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:23.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:23.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:23.762 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:23.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:23.791 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:23.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:23.892 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:23.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:23.916 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:23.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:23.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:23 smithi204 ceph-mon[87071]: pgmap v310: 99 pgs: 99 active+clean; 7.8 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:41:24.017 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:24.052 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:24.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:24.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:24.153 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:24.196 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:24.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:24.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:24.298 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:24.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:24.333 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:24.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:24.434 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:24.470 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:24.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:24.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:24.504 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:24.571 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:24.611 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:24.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:24.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:24.714 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:24.748 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:24.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:24.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:24.850 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:24.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:24.895 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:24.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:24.996 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:25.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.032 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:25.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:25.133 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:25.160 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:25.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:25.262 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:25.287 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:24 smithi096 ceph-mon[98901]: pgmap v311: 99 pgs: 99 active+clean; 7.8 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 64 MiB/s wr, 1.04k op/s 2024-09-13T14:41:25.287 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:24 smithi096 ceph-mon[102928]: pgmap v311: 99 pgs: 99 active+clean; 7.8 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 64 MiB/s wr, 1.04k op/s 2024-09-13T14:41:25.294 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:25.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:25.395 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:25.436 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:25.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:25.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:24 smithi204 ceph-mon[87071]: pgmap v311: 99 pgs: 99 active+clean; 7.8 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 64 MiB/s wr, 1.04k op/s 2024-09-13T14:41:25.538 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:25.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.559 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:25.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:25.661 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:25.691 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:25.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:25.793 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:25.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.829 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:25.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:25.930 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:25.966 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:25.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:26.069 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:26.103 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:26.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:26.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:26.205 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:26.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:26.237 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:26.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:26.338 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:26.370 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:26.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:26.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:26.471 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:26.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:26.503 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:26.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:26.604 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:26.631 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:26.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:26.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:26.714 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:41:26.714 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:41:26.714 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:41:26.714 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:41:26.714 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:41:26.732 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:26.757 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:26.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:26.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:26.858 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:26.889 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:26.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:26.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:26.991 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:27.029 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:27.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:27.131 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:27.161 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:27.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:27.263 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:27.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.290 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:27.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:27.305 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:27 smithi096 ceph-mon[98901]: pgmap v312: 99 pgs: 99 active+clean; 7.9 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 127 B/s rd, 54 MiB/s wr, 907 op/s 2024-09-13T14:41:27.305 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:27 smithi096 ceph-mon[102928]: pgmap v312: 99 pgs: 99 active+clean; 7.9 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 127 B/s rd, 54 MiB/s wr, 907 op/s 2024-09-13T14:41:27.390 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:27.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.420 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:27.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:27.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:27 smithi204 ceph-mon[87071]: pgmap v312: 99 pgs: 99 active+clean; 7.9 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 127 B/s rd, 54 MiB/s wr, 907 op/s 2024-09-13T14:41:27.522 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:27.549 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:27.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:27.651 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:27.679 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:27.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:27.781 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:27.822 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:27.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:27.924 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:27.936 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:41:27.936 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:41:27.954 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:27.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:27.975 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:41:27.975 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:41:28.017 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:28 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3200844250' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:28.018 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:41:28.018 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:41:28.018 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:41:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:41:27] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-13T14:41:28.019 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:28 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3200844250' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:28.056 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:28.057 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:41:28.057 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:41:28.093 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:41:28.094 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:41:28.094 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:28.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:28.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:28.129 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:41:28.129 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:41:28.164 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:41:28.165 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:41:28.196 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:28.198 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:41:28.199 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:41:28.222 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:28.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:28.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:28.324 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:28.361 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:28.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:28.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:28.462 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:41:28.463 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:28.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:28 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3200844250' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:28.487 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:28.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:28.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:28.588 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:28.617 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:28.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:28.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:28.719 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:28.751 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:28.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:28.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:28.853 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:28.890 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:28.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:28.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:28.992 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:29.026 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:29.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:29.128 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:29.159 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:29.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:29.261 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:29.297 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:29.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:29.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:29 smithi096 ceph-mon[98901]: pgmap v313: 99 pgs: 99 active+clean; 7.8 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 111 B/s rd, 44 MiB/s wr, 962 op/s 2024-09-13T14:41:29.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:29 smithi096 ceph-mon[102928]: pgmap v313: 99 pgs: 99 active+clean; 7.8 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 111 B/s rd, 44 MiB/s wr, 962 op/s 2024-09-13T14:41:29.399 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:29.436 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:29.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:29.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:29 smithi204 ceph-mon[87071]: pgmap v313: 99 pgs: 99 active+clean; 7.8 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 111 B/s rd, 44 MiB/s wr, 962 op/s 2024-09-13T14:41:29.537 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:29.558 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:29.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:29.660 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:29.689 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:29.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:29.742 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:29.790 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:29.840 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:29.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:29.942 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:29.972 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:29.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:30.074 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:30.098 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:30.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:30.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:30.203 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:30.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:30.246 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:30.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:30.347 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:30.388 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:30.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:30.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:41:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:41:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:30.489 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:30.530 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:30.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:30.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:30.632 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:30.661 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:30.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:30.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:30.763 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:30.803 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:30.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:30.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:30.905 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:30.937 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:30.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:30.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:31.039 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:31.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:31 smithi096 ceph-mon[98901]: pgmap v314: 99 pgs: 99 active+clean; 7.6 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 31 MiB/s wr, 989 op/s 2024-09-13T14:41:31.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:31 smithi096 ceph-mon[102928]: pgmap v314: 99 pgs: 99 active+clean; 7.6 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 31 MiB/s wr, 989 op/s 2024-09-13T14:41:31.073 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:31.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:31.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:31.103 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:41:31.175 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:31.206 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:31.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:31.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:31.302 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:31.308 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:31.378 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:31.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:31.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:31 smithi204 ceph-mon[87071]: pgmap v314: 99 pgs: 99 active+clean; 7.6 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 31 MiB/s wr, 989 op/s 2024-09-13T14:41:31.480 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:31.524 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:31.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:31.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:31.626 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:31.661 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:31.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:31.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:31.763 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:31.789 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:31.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:31.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:31.891 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:31.944 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:31.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:31.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:32.039 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T14:41:32.045 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:32.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:32.086 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:32.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:32.188 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:32.227 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:32.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:32.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:32.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:32 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2140573969' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:41:32.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:32 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2140573969' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:41:32.329 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:32.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:32.374 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:32.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:32.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:32 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2140573969' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:41:32.474 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:32.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:32.513 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:32.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:32.614 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:32.642 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:32.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:32.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:32.744 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:32.772 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:32.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:32.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:32.874 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:32.916 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:32.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:32.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:33.018 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:33.053 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:33.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:33.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:33.155 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:33.192 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:33.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:33.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:33.279 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:33 smithi096 ceph-mon[98901]: pgmap v315: 99 pgs: 99 active+clean; 7.5 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 100 B/s rd, 30 MiB/s wr, 1.16k op/s 2024-09-13T14:41:33.279 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:33 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2140573969' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:41:33.279 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:33 smithi096 ceph-mon[98901]: osdmap e127: 8 total, 8 up, 7 in 2024-09-13T14:41:33.279 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:33 smithi096 ceph-mon[102928]: pgmap v315: 99 pgs: 99 active+clean; 7.5 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 100 B/s rd, 30 MiB/s wr, 1.16k op/s 2024-09-13T14:41:33.279 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:33 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2140573969' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:41:33.279 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:33 smithi096 ceph-mon[102928]: osdmap e127: 8 total, 8 up, 7 in 2024-09-13T14:41:33.294 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:33.318 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T14:41:33.333 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:33.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:33.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:33.436 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:33.461 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:33.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:33.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:33 smithi204 ceph-mon[87071]: pgmap v315: 99 pgs: 99 active+clean; 7.5 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 100 B/s rd, 30 MiB/s wr, 1.16k op/s 2024-09-13T14:41:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:33 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2140573969' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:41:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:33 smithi204 ceph-mon[87071]: osdmap e127: 8 total, 8 up, 7 in 2024-09-13T14:41:33.563 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:33.600 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:33.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:33.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:33.702 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:33.755 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:33.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:33.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:33.856 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:33.885 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:33.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:33.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:33.987 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:34.037 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:34.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:34.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:34.149 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:34.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:34.185 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:34.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:34.286 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:34.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:34.312 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:34.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:34.414 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:34.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:34.453 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:34.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:34.554 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:34.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:34.593 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:34.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:34.606 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:41:34.607 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:41:34.683 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:34.694 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:34.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:34.745 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:34.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:34.846 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:34.885 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:34.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:34.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:34.987 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:35.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:35.026 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:35.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:35.051 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:41:35.127 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:35.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:35.159 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:35.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:35.260 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:35.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:35.287 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:35.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:35.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:35 smithi096 ceph-mon[98901]: pgmap v317: 99 pgs: 99 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 11 MiB/s wr, 1.01k op/s 2024-09-13T14:41:35.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:35 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1891157333' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:35.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:35 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:35.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:35 smithi096 ceph-mon[102928]: pgmap v317: 99 pgs: 99 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 11 MiB/s wr, 1.01k op/s 2024-09-13T14:41:35.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:35 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1891157333' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:35.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:35 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:35.388 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:35.401 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:41:35.422 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:35.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:35.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:35.435 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:35 smithi204 ceph-mon[87071]: pgmap v317: 99 pgs: 99 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 11 MiB/s wr, 1.01k op/s 2024-09-13T14:41:35.436 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:35 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1891157333' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:35.436 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:35 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:35.524 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:35.560 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:35.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:35.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:35.679 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:35.718 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:35.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:35.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:35.784 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:41:35.820 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:35.861 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:35.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:35.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:35.963 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:35.995 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:35.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:35.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:36.097 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:36.130 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:36.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:36.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:36.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:36 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:41:36.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:36 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:41:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:36 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:41:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:36 smithi204 ceph-mon[87071]: osdmap e128: 8 total, 8 up, 7 in 2024-09-13T14:41:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:41:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:36.231 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:36.258 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:36.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:36.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:36.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:36 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:41:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:36 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:41:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:36 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:41:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:36 smithi096 ceph-mon[98901]: osdmap e128: 8 total, 8 up, 7 in 2024-09-13T14:41:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:41:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:36 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:41:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:36 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:41:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:36 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:41:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:36 smithi096 ceph-mon[102928]: osdmap e128: 8 total, 8 up, 7 in 2024-09-13T14:41:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:41:36.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:36.359 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:36.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:36.396 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:36.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:36.498 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:36.509 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:36.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:36.536 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:36.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:36.637 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:36.671 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:36.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:36.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:36.773 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:36.806 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:36.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:36.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:36.908 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:36.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:36.942 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:36.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:37.043 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:37.081 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:37.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:37.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:37.183 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:37.229 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:37.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:37.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:37.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:37 smithi096 ceph-mon[98901]: pgmap v319: 99 pgs: 99 active+clean; 7.3 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 955 op/s 2024-09-13T14:41:37.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:37 smithi096 ceph-mon[102928]: pgmap v319: 99 pgs: 99 active+clean; 7.3 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 955 op/s 2024-09-13T14:41:37.331 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:37.362 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:37.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:37.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:37.433 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:37 smithi204 ceph-mon[87071]: pgmap v319: 99 pgs: 99 active+clean; 7.3 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 955 op/s 2024-09-13T14:41:37.464 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:37.493 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:37.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:37.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:37.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:37.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:37.630 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:37.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:37.731 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:37.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:37.759 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:37.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:37.860 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:37.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:37.903 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:37.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:37.914 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:41:37.914 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:41:38.004 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:38.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:38.041 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:38.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:38.142 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:38.182 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:38.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:38.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:38.284 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:38.313 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:41:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:41:37] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.51.0" 2024-09-13T14:41:38.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:38.323 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:38.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:38.424 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:38.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:38.457 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:38.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:38.558 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:38.592 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:41:38.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:38.597 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:38.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:38.698 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:38.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:38.738 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:38.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:38.839 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:38.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:38.873 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:38.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:38.974 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:38.987 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:41:39.009 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:39.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:39.111 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:39.142 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:39.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:39.244 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:39.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.281 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:39.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:39.317 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:41:39.317 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:41:39.318 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:39 smithi096 ceph-mon[98901]: pgmap v320: 99 pgs: 99 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 937 op/s 2024-09-13T14:41:39.318 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:39 smithi096 ceph-mon[102928]: pgmap v320: 99 pgs: 99 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 937 op/s 2024-09-13T14:41:39.355 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:41:39.356 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:41:39.382 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:39.390 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:41:39.391 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:41:39.396 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:39 smithi204 ceph-mon[87071]: pgmap v320: 99 pgs: 99 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 937 op/s 2024-09-13T14:41:39.419 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:39.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:39.423 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:41:39.423 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:41:39.465 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:41:39.466 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:41:39.504 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:41:39.504 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:41:39.520 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:39.537 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:41:39.537 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:41:39.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.556 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:39.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:39.573 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:41:39.574 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:41:39.656 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:39.689 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:41:39.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.692 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:39.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:39.795 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:39.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.845 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:39.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:39.946 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:39.975 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:39.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:40.077 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:40.111 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:40.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:40.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:40.155 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:40 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1718923435' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:40.156 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:41:39 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=cleanup t=2024-09-13T14:41:39.859410089Z level=info msg="Completed cleanup jobs" duration=11.919626ms 2024-09-13T14:41:40.213 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:40.245 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:40 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1718923435' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:40.246 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:40 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1718923435' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:40.263 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:40.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:40.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:40.367 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:40.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:40.406 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:40.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:40.430 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:41:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:41:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:40.506 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:40.544 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:40.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:40.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:40.646 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:40.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:40.677 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:40.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:40.778 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:40.814 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:40.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:40.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:40.870 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:41:40.871 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:41:40.871 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T14:41:40.871 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T14:41:40.871 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:41:40.916 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:40.961 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:40.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:40.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:41.063 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:41.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:41.100 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:41.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:41.193 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:41.201 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:41.226 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:41.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:41.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:41.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:41 smithi096 ceph-mon[98901]: pgmap v321: 99 pgs: 99 active+clean; 7.0 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 996 op/s 2024-09-13T14:41:41.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:41 smithi096 ceph-mon[102928]: pgmap v321: 99 pgs: 99 active+clean; 7.0 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 996 op/s 2024-09-13T14:41:41.328 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:41.356 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:41.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:41.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:41.457 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:41.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:41 smithi204 ceph-mon[87071]: pgmap v321: 99 pgs: 99 active+clean; 7.0 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 996 op/s 2024-09-13T14:41:41.496 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:41.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:41.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:41.552 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:41:41.598 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:41.640 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:41.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:41.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:41.742 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:41.774 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:41.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:41.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:41.875 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:41.912 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:41.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:41.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:41.931 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:41:42.015 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:42.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:42.047 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:42.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:42.101 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T14:41:42.148 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:42.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:42.182 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:42.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:42.283 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:42.319 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:42.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:42.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:42.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:42 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/839836933' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:41:42.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:42 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/839836933' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:41:42.358 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:42 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/839836933' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:41:42.421 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:42.457 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:42.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:42.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:42.559 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:42.591 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:42.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:42.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:42.652 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:41:42.692 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:42.745 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:42.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:42.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:42.847 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:42.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:42.891 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:42.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:42.992 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:43.031 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:43.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:43.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:43.133 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:43.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:43.171 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:43.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:43.272 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:43.307 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:43.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:43.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:43.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:43 smithi096 ceph-mon[98901]: pgmap v322: 99 pgs: 99 active+clean; 6.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 946 op/s 2024-09-13T14:41:43.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:43 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/839836933' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:41:43.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:43 smithi096 ceph-mon[98901]: osdmap e129: 8 total, 8 up, 7 in 2024-09-13T14:41:43.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:43 smithi096 ceph-mon[102928]: pgmap v322: 99 pgs: 99 active+clean; 6.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 946 op/s 2024-09-13T14:41:43.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:43 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/839836933' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:41:43.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:43 smithi096 ceph-mon[102928]: osdmap e129: 8 total, 8 up, 7 in 2024-09-13T14:41:43.409 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:43.418 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:43 smithi204 ceph-mon[87071]: pgmap v322: 99 pgs: 99 active+clean; 6.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 946 op/s 2024-09-13T14:41:43.418 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:43 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/839836933' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:41:43.418 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:43 smithi204 ceph-mon[87071]: osdmap e129: 8 total, 8 up, 7 in 2024-09-13T14:41:43.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:43.443 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:43.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:43.544 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:43.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:43.580 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:43.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:43.681 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:43.708 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T14:41:43.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:43.753 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:43.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:43.831 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:43.854 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:43.906 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:43.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:43.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:44.008 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:44.059 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:44.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:44.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:44.101 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:41:44.101 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T14:41:44.101 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T14:41:44.102 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:41:44.102 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.102 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:55392 5.e 5:71539784:::benchmark_data_smithi096_138841_object55391:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e114)", 2024-09-13T14:41:44.102 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:17.053092+0000", 2024-09-13T14:41:44.102 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 87.044736799999995, 2024-09-13T14:41:44.102 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.058145627999999998, 2024-09-13T14:41:44.102 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.102 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.102 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.102 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.102 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.102 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 55392 2024-09-13T14:41:44.102 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.103 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.103 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.103 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.103 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.053092+0000", 2024-09-13T14:41:44.103 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.103 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.103 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.103 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.103 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.053092+0000", 2024-09-13T14:41:44.103 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.103 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.103 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.103 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.104 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.053091+0000", 2024-09-13T14:41:44.104 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:41:44.104 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.104 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.104 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.104 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.053120+0000", 2024-09-13T14:41:44.104 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9258e-05 2024-09-13T14:41:44.104 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.104 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.104 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.104 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.053127+0000", 2024-09-13T14:41:44.104 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0299999999999996e-06 2024-09-13T14:41:44.104 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.105 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.105 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.105 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.053132+0000", 2024-09-13T14:41:44.105 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0609999999999997e-06 2024-09-13T14:41:44.105 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.105 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.105 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.105 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.055733+0000", 2024-09-13T14:41:44.105 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0026017129999999999 2024-09-13T14:41:44.105 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.105 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.106 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.106 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.055829+0000", 2024-09-13T14:41:44.106 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5450000000000003e-05 2024-09-13T14:41:44.106 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.106 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.106 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:41:44.106 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.055967+0000", 2024-09-13T14:41:44.106 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013813000000000001 2024-09-13T14:41:44.106 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.107 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.107 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.107 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.057893+0000", 2024-09-13T14:41:44.107 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0019261529999999999 2024-09-13T14:41:44.107 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.107 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.107 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.107 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.058159+0000", 2024-09-13T14:41:44.107 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00026650699999999997 2024-09-13T14:41:44.107 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.107 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.107 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.107 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.111179+0000", 2024-09-13T14:41:44.108 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.053019854999999998 2024-09-13T14:41:44.108 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.108 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.108 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.108 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.111213+0000", 2024-09-13T14:41:44.108 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3748999999999998e-05 2024-09-13T14:41:44.108 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.109 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.109 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.109 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.111237+0000", 2024-09-13T14:41:44.109 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4300000000000001e-05 2024-09-13T14:41:44.109 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.109 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.109 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.109 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.109 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.109 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:55400 5.e 5:79e01c46:::benchmark_data_smithi096_138841_object55399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e114)", 2024-09-13T14:41:44.109 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:17.058536+0000", 2024-09-13T14:41:44.109 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 87.039292228999997, 2024-09-13T14:41:44.109 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.054706210999999998, 2024-09-13T14:41:44.110 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.110 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.110 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.110 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.110 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.110 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 55400 2024-09-13T14:41:44.110 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.110 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.110 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.110 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.110 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.058536+0000", 2024-09-13T14:41:44.110 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.110 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.110 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.110 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.111 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.058536+0000", 2024-09-13T14:41:44.111 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.111 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.111 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.111 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.111 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.058535+0000", 2024-09-13T14:41:44.111 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:41:44.111 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.111 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.111 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.111 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.058577+0000", 2024-09-13T14:41:44.111 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2011000000000001e-05 2024-09-13T14:41:44.111 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.112 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.112 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.112 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.058589+0000", 2024-09-13T14:41:44.112 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1722000000000001e-05 2024-09-13T14:41:44.112 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.112 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.112 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.112 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.058594+0000", 2024-09-13T14:41:44.112 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5500000000000002e-06 2024-09-13T14:41:44.112 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.112 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.112 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.112 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.058640+0000", 2024-09-13T14:41:44.112 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5445000000000003e-05 2024-09-13T14:41:44.113 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.113 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.113 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.113 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.058769+0000", 2024-09-13T14:41:44.113 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000129018 2024-09-13T14:41:44.113 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.113 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.113 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:41:44.113 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.058881+0000", 2024-09-13T14:41:44.113 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011270800000000001 2024-09-13T14:41:44.113 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.113 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.114 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.114 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.060680+0000", 2024-09-13T14:41:44.114 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0017986969999999999 2024-09-13T14:41:44.114 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.114 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.114 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.114 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.061309+0000", 2024-09-13T14:41:44.114 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00062850299999999998 2024-09-13T14:41:44.114 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.114 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.114 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.114 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.113172+0000", 2024-09-13T14:41:44.114 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.051863268999999997 2024-09-13T14:41:44.114 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.115 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.115 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.115 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.113204+0000", 2024-09-13T14:41:44.115 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2314000000000003e-05 2024-09-13T14:41:44.115 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.115 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.115 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.115 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.113243+0000", 2024-09-13T14:41:44.115 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8389000000000001e-05 2024-09-13T14:41:44.115 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.115 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.115 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.116 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.116 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.116 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:55883 5.e 5:74898c3f:::benchmark_data_smithi096_138841_object55882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e114)", 2024-09-13T14:41:44.116 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:17.470349+0000", 2024-09-13T14:41:44.116 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 86.627479746999995, 2024-09-13T14:41:44.116 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.061721614000000001, 2024-09-13T14:41:44.116 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.116 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.116 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.117 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.117 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.117 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 55883 2024-09-13T14:41:44.117 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.117 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.117 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.117 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.118 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.470349+0000", 2024-09-13T14:41:44.118 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.118 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.118 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.118 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.118 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.470349+0000", 2024-09-13T14:41:44.118 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.118 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.118 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.118 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.118 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.470348+0000", 2024-09-13T14:41:44.118 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:41:44.119 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.119 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.119 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.119 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.470399+0000", 2024-09-13T14:41:44.119 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1310999999999997e-05 2024-09-13T14:41:44.119 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.119 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.119 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.119 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.470410+0000", 2024-09-13T14:41:44.119 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0845e-05 2024-09-13T14:41:44.119 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.120 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.120 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.120 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.470416+0000", 2024-09-13T14:41:44.120 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5899999999999996e-06 2024-09-13T14:41:44.120 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.120 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.120 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.120 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.470457+0000", 2024-09-13T14:41:44.120 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1032000000000003e-05 2024-09-13T14:41:44.120 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.120 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.120 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.121 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.470577+0000", 2024-09-13T14:41:44.121 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119338 2024-09-13T14:41:44.121 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.121 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.121 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:41:44.121 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.470709+0000", 2024-09-13T14:41:44.121 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000132001 2024-09-13T14:41:44.121 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.121 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.121 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.121 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.472333+0000", 2024-09-13T14:41:44.121 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001624084 2024-09-13T14:41:44.121 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.121 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.122 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.122 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.473634+0000", 2024-09-13T14:41:44.122 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001300935 2024-09-13T14:41:44.122 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.122 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.122 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.122 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.531992+0000", 2024-09-13T14:41:44.122 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.058358570999999998 2024-09-13T14:41:44.122 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.122 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.122 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.122 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.532037+0000", 2024-09-13T14:41:44.123 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4623000000000003e-05 2024-09-13T14:41:44.123 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.123 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.123 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.123 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.532070+0000", 2024-09-13T14:41:44.123 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3531000000000001e-05 2024-09-13T14:41:44.123 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.123 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.123 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.123 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.123 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.123 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:56124 5.e 5:7d1f6436:::benchmark_data_smithi096_138841_object56123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e114)", 2024-09-13T14:41:44.123 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:17.651958+0000", 2024-09-13T14:41:44.124 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 86.445870444999997, 2024-09-13T14:41:44.124 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.058017676999999997, 2024-09-13T14:41:44.124 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.124 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.124 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.124 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.124 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.124 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 56124 2024-09-13T14:41:44.124 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.124 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.124 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.124 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.124 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.651958+0000", 2024-09-13T14:41:44.125 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.125 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.125 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.125 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.125 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.651958+0000", 2024-09-13T14:41:44.125 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.125 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.125 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.125 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.125 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.651956+0000", 2024-09-13T14:41:44.125 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:41:44.125 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.125 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.126 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.126 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.652008+0000", 2024-09-13T14:41:44.126 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1879000000000003e-05 2024-09-13T14:41:44.126 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.126 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.126 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.126 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.652023+0000", 2024-09-13T14:41:44.126 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5276000000000001e-05 2024-09-13T14:41:44.126 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.126 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.126 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.126 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.652029+0000", 2024-09-13T14:41:44.126 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.716e-06 2024-09-13T14:41:44.126 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.127 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.127 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.127 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.652080+0000", 2024-09-13T14:41:44.127 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0958000000000002e-05 2024-09-13T14:41:44.127 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.127 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.127 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.127 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.652172+0000", 2024-09-13T14:41:44.127 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1572999999999999e-05 2024-09-13T14:41:44.127 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.127 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.127 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:41:44.127 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.652268+0000", 2024-09-13T14:41:44.127 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6776999999999999e-05 2024-09-13T14:41:44.128 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.128 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.128 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.128 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.654731+0000", 2024-09-13T14:41:44.128 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0024627529999999998 2024-09-13T14:41:44.128 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.128 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.128 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.128 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.655222+0000", 2024-09-13T14:41:44.128 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00049116399999999999 2024-09-13T14:41:44.128 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.128 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.128 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.128 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.709901+0000", 2024-09-13T14:41:44.129 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.054678597000000002 2024-09-13T14:41:44.129 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.129 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.129 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.129 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.709946+0000", 2024-09-13T14:41:44.129 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5306999999999998e-05 2024-09-13T14:41:44.129 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.129 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.129 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.129 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:17.709976+0000", 2024-09-13T14:41:44.129 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9541999999999999e-05 2024-09-13T14:41:44.129 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.129 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.129 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.130 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.130 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.130 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:57388 5.e 5:7b726f59:::benchmark_data_smithi096_138841_object57387:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-09-13T14:41:44.130 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:18.443187+0000", 2024-09-13T14:41:44.130 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 85.654641928999993, 2024-09-13T14:41:44.130 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.041237922000000003, 2024-09-13T14:41:44.130 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.130 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.130 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.130 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.130 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.130 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 57388 2024-09-13T14:41:44.130 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.130 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.131 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.131 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.131 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:18.443187+0000", 2024-09-13T14:41:44.131 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.131 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.131 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.131 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.131 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:18.443187+0000", 2024-09-13T14:41:44.131 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.131 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.131 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.131 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.131 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:18.443185+0000", 2024-09-13T14:41:44.132 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:41:44.132 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.132 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.132 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.132 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:18.443240+0000", 2024-09-13T14:41:44.132 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4534999999999999e-05 2024-09-13T14:41:44.132 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.132 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.132 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.132 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:18.443260+0000", 2024-09-13T14:41:44.132 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0483999999999999e-05 2024-09-13T14:41:44.132 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.132 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.133 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.133 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:18.443270+0000", 2024-09-13T14:41:44.133 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0073e-05 2024-09-13T14:41:44.133 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.133 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.133 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.133 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:18.443376+0000", 2024-09-13T14:41:44.133 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000106142 2024-09-13T14:41:44.133 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.133 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.133 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.133 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:18.443643+0000", 2024-09-13T14:41:44.133 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000266443 2024-09-13T14:41:44.133 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.134 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.134 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:41:44.134 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:18.443781+0000", 2024-09-13T14:41:44.134 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013793 2024-09-13T14:41:44.134 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.134 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.134 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.134 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:18.445269+0000", 2024-09-13T14:41:44.134 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0014880659999999999 2024-09-13T14:41:44.134 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.134 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.134 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.134 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:18.445534+0000", 2024-09-13T14:41:44.134 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00026491500000000001 2024-09-13T14:41:44.135 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.135 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.135 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.135 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:18.484337+0000", 2024-09-13T14:41:44.135 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.038803758000000001 2024-09-13T14:41:44.135 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.135 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.135 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.135 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:18.484385+0000", 2024-09-13T14:41:44.135 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7404999999999998e-05 2024-09-13T14:41:44.135 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.135 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.135 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.136 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:18.484425+0000", 2024-09-13T14:41:44.136 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9823999999999997e-05 2024-09-13T14:41:44.136 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.136 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.136 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.136 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.136 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.136 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:67777 5.e 5:72aa4110:::benchmark_data_smithi096_138841_object67776:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:41:44.136 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:25.204952+0000", 2024-09-13T14:41:44.136 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 78.892877053999996, 2024-09-13T14:41:44.136 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.111105538, 2024-09-13T14:41:44.136 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.136 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.137 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.137 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.137 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.137 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 67777 2024-09-13T14:41:44.137 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.137 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.137 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.137 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.137 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.204952+0000", 2024-09-13T14:41:44.137 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.137 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.137 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.137 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.137 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.204952+0000", 2024-09-13T14:41:44.138 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.138 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.138 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.138 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.138 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.204950+0000", 2024-09-13T14:41:44.138 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:41:44.138 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.138 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.138 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.138 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.205010+0000", 2024-09-13T14:41:44.138 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.016e-05 2024-09-13T14:41:44.138 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.139 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.139 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.139 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.205022+0000", 2024-09-13T14:41:44.139 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1518999999999999e-05 2024-09-13T14:41:44.139 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.139 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.139 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.139 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.205028+0000", 2024-09-13T14:41:44.139 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8669999999999999e-06 2024-09-13T14:41:44.139 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.140 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.140 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.140 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.205081+0000", 2024-09-13T14:41:44.140 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3263999999999997e-05 2024-09-13T14:41:44.140 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.140 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.140 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.140 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.205193+0000", 2024-09-13T14:41:44.140 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000112124 2024-09-13T14:41:44.140 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.140 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.140 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:41:44.141 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.205303+0000", 2024-09-13T14:41:44.141 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010935200000000001 2024-09-13T14:41:44.141 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.141 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.141 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.141 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.207502+0000", 2024-09-13T14:41:44.141 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0021989879999999998 2024-09-13T14:41:44.141 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.141 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.141 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.141 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.208144+0000", 2024-09-13T14:41:44.141 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00064259699999999996 2024-09-13T14:41:44.141 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.141 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.142 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.142 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.315970+0000", 2024-09-13T14:41:44.142 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.107825882 2024-09-13T14:41:44.142 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.142 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.142 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.142 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.316020+0000", 2024-09-13T14:41:44.142 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0407e-05 2024-09-13T14:41:44.142 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.142 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.142 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.142 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.316057+0000", 2024-09-13T14:41:44.142 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6652000000000001e-05 2024-09-13T14:41:44.143 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.143 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.143 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.143 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.143 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.143 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:68978 5.e 5:7866ad26:::benchmark_data_smithi096_138841_object68977:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:41:44.143 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:26.332734+0000", 2024-09-13T14:41:44.143 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 77.765094485000006, 2024-09-13T14:41:44.143 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.050245748999999999, 2024-09-13T14:41:44.143 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.143 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.143 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.143 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.144 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.144 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 68978 2024-09-13T14:41:44.144 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.144 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.144 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.144 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.144 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:26.332734+0000", 2024-09-13T14:41:44.144 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.144 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.144 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.144 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.144 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:26.332734+0000", 2024-09-13T14:41:44.144 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.145 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.145 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.145 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.145 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:26.332733+0000", 2024-09-13T14:41:44.145 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:41:44.145 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.145 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.145 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.145 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:26.332772+0000", 2024-09-13T14:41:44.145 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8643000000000003e-05 2024-09-13T14:41:44.145 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.145 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.145 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.146 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:26.332780+0000", 2024-09-13T14:41:44.146 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0989999999999999e-06 2024-09-13T14:41:44.146 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.146 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.146 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.146 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:26.332784+0000", 2024-09-13T14:41:44.146 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1860000000000002e-06 2024-09-13T14:41:44.146 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.146 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.146 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.146 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:26.332827+0000", 2024-09-13T14:41:44.146 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2228999999999998e-05 2024-09-13T14:41:44.147 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.147 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.147 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.147 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:26.332931+0000", 2024-09-13T14:41:44.147 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010451900000000001 2024-09-13T14:41:44.147 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.147 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.147 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:41:44.147 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:26.333047+0000", 2024-09-13T14:41:44.147 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000115424 2024-09-13T14:41:44.147 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.147 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.147 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.148 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:26.335520+0000", 2024-09-13T14:41:44.148 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0024732119999999998 2024-09-13T14:41:44.148 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.148 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.148 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.148 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:26.336021+0000", 2024-09-13T14:41:44.148 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000500998 2024-09-13T14:41:44.148 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.148 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.148 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.148 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:26.382887+0000", 2024-09-13T14:41:44.148 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.046866288999999998 2024-09-13T14:41:44.148 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.149 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.149 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.149 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:26.382943+0000", 2024-09-13T14:41:44.149 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5736999999999998e-05 2024-09-13T14:41:44.149 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.149 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.149 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.149 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:26.382980+0000", 2024-09-13T14:41:44.149 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6999000000000003e-05 2024-09-13T14:41:44.149 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.149 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.149 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.149 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.150 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.150 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:70370 5.1 5:885e31b1:::benchmark_data_smithi096_138841_object70369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:41:44.150 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:27.339886+0000", 2024-09-13T14:41:44.150 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 76.757942923000002, 2024-09-13T14:41:44.150 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.073231750999999998, 2024-09-13T14:41:44.150 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.150 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.150 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.150 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.150 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.150 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 70370 2024-09-13T14:41:44.150 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.150 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.151 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.151 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.151 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.339886+0000", 2024-09-13T14:41:44.151 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.151 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.151 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.151 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.151 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.339886+0000", 2024-09-13T14:41:44.151 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.151 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.151 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.151 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.152 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.339884+0000", 2024-09-13T14:41:44.152 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:41:44.152 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.152 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.152 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.152 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.339953+0000", 2024-09-13T14:41:44.152 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9506999999999997e-05 2024-09-13T14:41:44.152 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.152 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.152 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.152 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.339964+0000", 2024-09-13T14:41:44.152 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0508e-05 2024-09-13T14:41:44.152 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.153 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.153 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.153 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.339969+0000", 2024-09-13T14:41:44.153 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3419999999999998e-06 2024-09-13T14:41:44.153 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.153 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.153 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.153 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.340002+0000", 2024-09-13T14:41:44.153 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2716000000000003e-05 2024-09-13T14:41:44.153 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.153 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.153 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.154 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.340083+0000", 2024-09-13T14:41:44.154 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1192999999999993e-05 2024-09-13T14:41:44.154 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.154 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.154 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:41:44.154 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.340167+0000", 2024-09-13T14:41:44.154 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3932999999999998e-05 2024-09-13T14:41:44.154 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.154 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.154 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.154 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.342358+0000", 2024-09-13T14:41:44.154 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0021906439999999998 2024-09-13T14:41:44.154 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.155 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.155 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.155 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.342704+0000", 2024-09-13T14:41:44.155 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00034595600000000002 2024-09-13T14:41:44.155 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.155 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.155 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.155 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.412955+0000", 2024-09-13T14:41:44.155 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.070251172000000001 2024-09-13T14:41:44.155 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.155 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.155 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.155 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.413064+0000", 2024-09-13T14:41:44.156 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000108689 2024-09-13T14:41:44.156 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.156 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.156 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.156 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.413117+0000", 2024-09-13T14:41:44.156 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3754000000000001e-05 2024-09-13T14:41:44.156 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.156 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.156 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.156 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.156 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.156 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:70373 5.e 5:72b7c280:::benchmark_data_smithi096_138841_object70372:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:41:44.156 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:27.343761+0000", 2024-09-13T14:41:44.157 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 76.754067280000001, 2024-09-13T14:41:44.157 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.075602238000000002, 2024-09-13T14:41:44.157 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.157 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.157 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.157 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.157 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.157 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 70373 2024-09-13T14:41:44.157 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.157 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.157 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.157 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.157 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.343761+0000", 2024-09-13T14:41:44.158 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.158 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.158 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.158 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.158 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.343761+0000", 2024-09-13T14:41:44.158 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.158 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.158 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.158 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.158 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.343760+0000", 2024-09-13T14:41:44.158 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:41:44.158 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.158 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.158 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.159 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.343816+0000", 2024-09-13T14:41:44.159 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5699000000000003e-05 2024-09-13T14:41:44.159 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.159 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.159 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.159 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.343829+0000", 2024-09-13T14:41:44.159 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3205e-05 2024-09-13T14:41:44.159 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.159 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.159 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.159 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.343835+0000", 2024-09-13T14:41:44.159 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8130000000000001e-06 2024-09-13T14:41:44.159 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.160 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.160 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.160 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.343875+0000", 2024-09-13T14:41:44.160 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0191999999999997e-05 2024-09-13T14:41:44.160 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.160 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.160 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.160 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.343966+0000", 2024-09-13T14:41:44.160 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1296000000000001e-05 2024-09-13T14:41:44.160 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.160 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.160 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:41:44.160 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.344069+0000", 2024-09-13T14:41:44.161 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:44.162 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000102336 2024-09-13T14:41:44.162 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.162 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.163 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.163 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.411879+0000", 2024-09-13T14:41:44.163 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.067810513000000003 2024-09-13T14:41:44.163 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.163 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.163 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.163 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.419275+0000", 2024-09-13T14:41:44.163 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0073961410000000002 2024-09-13T14:41:44.163 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.163 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.163 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.163 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.419285+0000", 2024-09-13T14:41:44.164 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.3950000000000007e-06 2024-09-13T14:41:44.164 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.164 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.164 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.164 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.419331+0000", 2024-09-13T14:41:44.164 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5748000000000003e-05 2024-09-13T14:41:44.164 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.164 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.164 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.164 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.419364+0000", 2024-09-13T14:41:44.164 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2978999999999997e-05 2024-09-13T14:41:44.164 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.165 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.165 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.165 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.165 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.165 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:70374 5.1 5:8c425893:::benchmark_data_smithi096_138841_object70373:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:41:44.165 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:27.343844+0000", 2024-09-13T14:41:44.165 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 76.753984238000001, 2024-09-13T14:41:44.165 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.070330395000000004, 2024-09-13T14:41:44.165 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.165 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.165 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.165 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.166 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.166 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 70374 2024-09-13T14:41:44.166 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.166 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.166 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.166 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.166 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.343844+0000", 2024-09-13T14:41:44.166 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.166 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.166 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.166 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.166 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.343844+0000", 2024-09-13T14:41:44.166 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.167 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.167 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.167 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.167 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.343844+0000", 2024-09-13T14:41:44.167 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:41:44.167 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.167 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.167 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.167 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.343886+0000", 2024-09-13T14:41:44.167 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.206e-05 2024-09-13T14:41:44.167 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.168 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.168 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.168 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.343894+0000", 2024-09-13T14:41:44.168 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.875e-06 2024-09-13T14:41:44.168 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.168 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.168 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.168 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.343897+0000", 2024-09-13T14:41:44.168 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8059999999999999e-06 2024-09-13T14:41:44.168 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.168 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.168 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.168 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.343939+0000", 2024-09-13T14:41:44.169 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2778999999999998e-05 2024-09-13T14:41:44.169 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.169 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.169 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.169 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.344063+0000", 2024-09-13T14:41:44.169 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012361300000000001 2024-09-13T14:41:44.169 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.169 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.169 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:41:44.169 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.408373+0000", 2024-09-13T14:41:44.169 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.06431009 2024-09-13T14:41:44.169 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.169 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.170 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.170 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.414065+0000", 2024-09-13T14:41:44.170 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0056921799999999998 2024-09-13T14:41:44.170 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.170 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.170 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.170 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.414071+0000", 2024-09-13T14:41:44.170 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1319999999999999e-06 2024-09-13T14:41:44.170 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.170 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.170 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.170 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.414096+0000", 2024-09-13T14:41:44.171 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4202999999999998e-05 2024-09-13T14:41:44.171 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.171 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.171 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.171 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.414144+0000", 2024-09-13T14:41:44.171 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8389e-05 2024-09-13T14:41:44.171 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.171 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.171 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.171 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.414175+0000", 2024-09-13T14:41:44.171 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0753999999999997e-05 2024-09-13T14:41:44.171 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.171 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.172 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.172 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.172 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.172 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.35049.0:70370 5.1 e119/107 ondisk, result = 0)", 2024-09-13T14:41:44.172 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:27.344452+0000", 2024-09-13T14:41:44.172 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 76.753376689999996, 2024-09-13T14:41:44.172 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.068671963000000003, 2024-09-13T14:41:44.172 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.172 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:41:44.172 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.172 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.172 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.172 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.344452+0000", 2024-09-13T14:41:44.173 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.173 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.173 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.173 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.173 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.344452+0000", 2024-09-13T14:41:44.173 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.173 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.173 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.173 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.173 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.344451+0000", 2024-09-13T14:41:44.173 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:41:44.173 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.173 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.174 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.174 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.344455+0000", 2024-09-13T14:41:44.174 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4800000000000001e-06 2024-09-13T14:41:44.174 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.174 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.174 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.174 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.407785+0000", 2024-09-13T14:41:44.174 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.063330653000000001 2024-09-13T14:41:44.174 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.174 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.174 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.174 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.407817+0000", 2024-09-13T14:41:44.175 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1576999999999999e-05 2024-09-13T14:41:44.175 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.175 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.175 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.175 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.412943+0000", 2024-09-13T14:41:44.175 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0051259499999999998 2024-09-13T14:41:44.175 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.175 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.175 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.175 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.412949+0000", 2024-09-13T14:41:44.175 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.489e-06 2024-09-13T14:41:44.175 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.176 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.176 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.176 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.413124+0000", 2024-09-13T14:41:44.176 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00017441399999999999 2024-09-13T14:41:44.176 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.176 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.176 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.176 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.176 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.176 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.35049.0:70373 5.e e119/107 ondisk, result = 0)", 2024-09-13T14:41:44.176 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:27.346622+0000", 2024-09-13T14:41:44.176 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 76.751206440999994, 2024-09-13T14:41:44.177 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.065259687999999996, 2024-09-13T14:41:44.177 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.177 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:41:44.177 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.177 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.177 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.177 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.346622+0000", 2024-09-13T14:41:44.177 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.177 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.177 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.177 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.177 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.346622+0000", 2024-09-13T14:41:44.178 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.178 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.178 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.178 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.178 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.346622+0000", 2024-09-13T14:41:44.178 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:41:44.178 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.178 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.178 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.178 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.346625+0000", 2024-09-13T14:41:44.178 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.636e-06 2024-09-13T14:41:44.178 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.179 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.179 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.179 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.408136+0000", 2024-09-13T14:41:44.179 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.061510312999999997 2024-09-13T14:41:44.179 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.179 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.179 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.179 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.408145+0000", 2024-09-13T14:41:44.179 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6120000000000005e-06 2024-09-13T14:41:44.179 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.179 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.179 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.180 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.411867+0000", 2024-09-13T14:41:44.180 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0037219309999999999 2024-09-13T14:41:44.180 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.180 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.180 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.180 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.411873+0000", 2024-09-13T14:41:44.180 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3790000000000003e-06 2024-09-13T14:41:44.180 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.180 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.180 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.180 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.411882+0000", 2024-09-13T14:41:44.180 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3720000000000005e-06 2024-09-13T14:41:44.180 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.181 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.181 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.181 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.181 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.181 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.35049.0:70384 5.9 e119/107 5:995f2648:::benchmark_data_smithi096_138841_object70383:head v 119'4363, mlcod=119'4361)", 2024-09-13T14:41:44.181 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:27.377500+0000", 2024-09-13T14:41:44.181 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 76.720329065000001, 2024-09-13T14:41:44.181 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.045108890999999998, 2024-09-13T14:41:44.181 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.181 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.181 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.181 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.182 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.182 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.377500+0000", 2024-09-13T14:41:44.182 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.182 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.182 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.182 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.182 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.377500+0000", 2024-09-13T14:41:44.182 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.182 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.182 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.182 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.183 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.377499+0000", 2024-09-13T14:41:44.183 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:41:44.183 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.183 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.183 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.183 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.377521+0000", 2024-09-13T14:41:44.183 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2458000000000001e-05 2024-09-13T14:41:44.183 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.183 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.183 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.183 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.407870+0000", 2024-09-13T14:41:44.183 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.030348884999999999 2024-09-13T14:41:44.183 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.184 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.184 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.184 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.407893+0000", 2024-09-13T14:41:44.184 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.3629999999999999e-05 2024-09-13T14:41:44.184 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.184 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.184 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.184 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.411953+0000", 2024-09-13T14:41:44.184 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0040592060000000001 2024-09-13T14:41:44.184 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.184 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.184 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.185 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.411969+0000", 2024-09-13T14:41:44.185 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6668999999999999e-05 2024-09-13T14:41:44.185 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.185 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.185 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.185 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.422566+0000", 2024-09-13T14:41:44.185 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010596747 2024-09-13T14:41:44.185 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.185 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.185 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.185 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:27.422608+0000", 2024-09-13T14:41:44.185 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2292000000000001e-05 2024-09-13T14:41:44.185 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.186 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.186 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.186 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.186 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.186 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:92309 5.e 5:719d9584:::benchmark_data_smithi096_138841_object92308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-09-13T14:41:44.186 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:45.530743+0000", 2024-09-13T14:41:44.186 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 58.567085110999997, 2024-09-13T14:41:44.186 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.041816484000000001, 2024-09-13T14:41:44.186 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.187 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.187 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.187 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.187 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.187 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 92309 2024-09-13T14:41:44.187 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.187 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.187 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.187 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.187 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:45.530743+0000", 2024-09-13T14:41:44.187 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.187 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.187 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.187 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.188 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:45.530743+0000", 2024-09-13T14:41:44.188 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.188 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.188 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.188 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.188 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:45.530743+0000", 2024-09-13T14:41:44.188 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:41:44.188 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.188 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.188 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.188 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:45.530784+0000", 2024-09-13T14:41:44.188 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0995999999999998e-05 2024-09-13T14:41:44.188 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.189 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.189 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.189 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:45.530809+0000", 2024-09-13T14:41:44.189 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5236e-05 2024-09-13T14:41:44.189 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.189 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.189 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.189 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:45.530825+0000", 2024-09-13T14:41:44.189 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5625e-05 2024-09-13T14:41:44.189 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.189 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.189 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.189 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:45.536398+0000", 2024-09-13T14:41:44.189 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0055728949999999996 2024-09-13T14:41:44.190 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.190 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.190 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.190 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:45.536518+0000", 2024-09-13T14:41:44.190 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000120776 2024-09-13T14:41:44.190 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.190 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.190 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:41:44.190 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:45.536656+0000", 2024-09-13T14:41:44.190 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013782099999999999 2024-09-13T14:41:44.190 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.190 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.190 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.191 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:45.538777+0000", 2024-09-13T14:41:44.191 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0021211810000000002 2024-09-13T14:41:44.191 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.191 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.191 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.191 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:45.538970+0000", 2024-09-13T14:41:44.191 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00019297199999999999 2024-09-13T14:41:44.191 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.191 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.191 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.191 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:45.572497+0000", 2024-09-13T14:41:44.191 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.033526223000000001 2024-09-13T14:41:44.191 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.191 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.192 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.192 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:45.572526+0000", 2024-09-13T14:41:44.192 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9439999999999999e-05 2024-09-13T14:41:44.192 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.192 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.192 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.192 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:45.572560+0000", 2024-09-13T14:41:44.192 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3898e-05 2024-09-13T14:41:44.192 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.192 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.192 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.192 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.192 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.193 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:97097 5.1 5:8054e3e4:::benchmark_data_smithi096_138841_object97096:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-09-13T14:41:44.193 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:49.784263+0000", 2024-09-13T14:41:44.193 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 54.313565928999999, 2024-09-13T14:41:44.193 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.044799142, 2024-09-13T14:41:44.193 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.193 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.193 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.193 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.193 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.193 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 97097 2024-09-13T14:41:44.193 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.193 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.194 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.194 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.194 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:49.784263+0000", 2024-09-13T14:41:44.194 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.194 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.194 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.194 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.194 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:49.784263+0000", 2024-09-13T14:41:44.194 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.194 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.194 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.194 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.194 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:49.784254+0000", 2024-09-13T14:41:44.194 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999909 2024-09-13T14:41:44.195 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.195 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.195 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.195 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:49.784352+0000", 2024-09-13T14:41:44.195 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.8393000000000005e-05 2024-09-13T14:41:44.195 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.195 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.195 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.195 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:49.784364+0000", 2024-09-13T14:41:44.195 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1549000000000001e-05 2024-09-13T14:41:44.195 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.195 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.195 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.196 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:49.784369+0000", 2024-09-13T14:41:44.196 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3229999999999997e-06 2024-09-13T14:41:44.196 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.196 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.196 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.196 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:49.784400+0000", 2024-09-13T14:41:44.196 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1448000000000001e-05 2024-09-13T14:41:44.196 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.196 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.196 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.196 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:49.784496+0000", 2024-09-13T14:41:44.196 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5636000000000004e-05 2024-09-13T14:41:44.196 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.197 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.197 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:41:44.197 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:49.784639+0000", 2024-09-13T14:41:44.197 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000143122 2024-09-13T14:41:44.197 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.197 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.197 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.197 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:49.789914+0000", 2024-09-13T14:41:44.197 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0052754159999999998 2024-09-13T14:41:44.197 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.197 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.197 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.197 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:49.790664+0000", 2024-09-13T14:41:44.198 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00074987899999999995 2024-09-13T14:41:44.198 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.198 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.198 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.198 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:49.828998+0000", 2024-09-13T14:41:44.198 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.038333500999999999 2024-09-13T14:41:44.198 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.198 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.198 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.198 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:49.829033+0000", 2024-09-13T14:41:44.198 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4832000000000001e-05 2024-09-13T14:41:44.198 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.198 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.199 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.199 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:49.829062+0000", 2024-09-13T14:41:44.199 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9094e-05 2024-09-13T14:41:44.199 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.199 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.199 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.199 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.199 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.199 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:126465 5.e 5:7b2c7700:::benchmark_data_smithi096_138841_object126464:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2024-09-13T14:41:44.200 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:41:18.780177+0000", 2024-09-13T14:41:44.200 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 25.317651585, 2024-09-13T14:41:44.200 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.055092919999999997, 2024-09-13T14:41:44.200 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.200 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.200 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.200 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.200 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.200 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 126465 2024-09-13T14:41:44.200 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.200 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.200 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.201 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.201 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.780177+0000", 2024-09-13T14:41:44.201 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.201 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.201 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.201 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.201 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.780177+0000", 2024-09-13T14:41:44.201 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.201 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.201 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.201 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.201 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.780175+0000", 2024-09-13T14:41:44.201 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:41:44.202 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.202 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.202 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.202 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.780221+0000", 2024-09-13T14:41:44.202 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5881000000000003e-05 2024-09-13T14:41:44.202 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.202 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.202 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.202 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.780234+0000", 2024-09-13T14:41:44.202 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2738000000000001e-05 2024-09-13T14:41:44.202 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.202 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.202 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.202 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.780240+0000", 2024-09-13T14:41:44.203 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7690000000000002e-06 2024-09-13T14:41:44.203 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.203 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.203 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.203 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.780275+0000", 2024-09-13T14:41:44.203 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4955e-05 2024-09-13T14:41:44.203 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.203 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.203 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.203 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.780371+0000", 2024-09-13T14:41:44.203 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6714999999999998e-05 2024-09-13T14:41:44.203 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.203 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.204 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:41:44.204 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.780471+0000", 2024-09-13T14:41:44.204 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9489999999999998e-05 2024-09-13T14:41:44.204 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.204 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.204 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.204 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.786245+0000", 2024-09-13T14:41:44.204 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0057740930000000001 2024-09-13T14:41:44.204 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.204 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.204 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.204 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.786256+0000", 2024-09-13T14:41:44.204 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0658e-05 2024-09-13T14:41:44.205 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.205 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.205 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.205 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.835187+0000", 2024-09-13T14:41:44.205 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.048931715000000001 2024-09-13T14:41:44.205 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.205 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.205 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.205 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.835230+0000", 2024-09-13T14:41:44.205 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3198999999999998e-05 2024-09-13T14:41:44.205 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.205 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.205 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.205 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.835270+0000", 2024-09-13T14:41:44.206 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9402e-05 2024-09-13T14:41:44.206 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.206 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.206 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.206 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.206 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.206 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:126466 5.e 5:70cd2751:::benchmark_data_smithi096_138841_object126465:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2024-09-13T14:41:44.206 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:41:18.780471+0000", 2024-09-13T14:41:44.206 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 25.317357417, 2024-09-13T14:41:44.206 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.058593701999999998, 2024-09-13T14:41:44.206 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.206 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.207 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.207 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.207 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.207 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 126466 2024-09-13T14:41:44.207 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.207 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.207 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.207 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.207 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.780471+0000", 2024-09-13T14:41:44.207 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.207 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.207 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.207 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.208 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.780471+0000", 2024-09-13T14:41:44.208 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.208 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.208 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.208 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.208 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.780470+0000", 2024-09-13T14:41:44.208 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:41:44.208 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.208 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.208 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.208 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.780554+0000", 2024-09-13T14:41:44.208 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3960000000000003e-05 2024-09-13T14:41:44.209 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.209 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.209 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.209 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.780565+0000", 2024-09-13T14:41:44.209 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1277000000000001e-05 2024-09-13T14:41:44.209 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.209 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.209 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.209 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.780589+0000", 2024-09-13T14:41:44.209 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.3204e-05 2024-09-13T14:41:44.209 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.210 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.210 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.210 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.783728+0000", 2024-09-13T14:41:44.210 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003139298 2024-09-13T14:41:44.210 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.210 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.210 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.210 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.783834+0000", 2024-09-13T14:41:44.210 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000105765 2024-09-13T14:41:44.210 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.210 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.210 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:41:44.210 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.783933+0000", 2024-09-13T14:41:44.210 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9098999999999994e-05 2024-09-13T14:41:44.211 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.211 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.211 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.211 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.786385+0000", 2024-09-13T14:41:44.211 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002452381 2024-09-13T14:41:44.211 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.211 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.211 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.211 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.786699+0000", 2024-09-13T14:41:44.211 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00031342700000000002 2024-09-13T14:41:44.211 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.211 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.211 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.212 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.838971+0000", 2024-09-13T14:41:44.212 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.052272725999999999 2024-09-13T14:41:44.212 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.212 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.212 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.212 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.839026+0000", 2024-09-13T14:41:44.212 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4475000000000003e-05 2024-09-13T14:41:44.212 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.212 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.212 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.212 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:18.839065+0000", 2024-09-13T14:41:44.212 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8989999999999998e-05 2024-09-13T14:41:44.213 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.213 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.213 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.213 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.213 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.213 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:149131 5.e 5:7da2ca27:::benchmark_data_smithi096_138841_object18935:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-09-13T14:41:44.213 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:41:41.819193+0000", 2024-09-13T14:41:44.213 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 2.2786352170000002, 2024-09-13T14:41:44.213 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.061065304000000001, 2024-09-13T14:41:44.213 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.213 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.213 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.213 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.213 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.214 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 149131 2024-09-13T14:41:44.214 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.214 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.214 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.214 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.214 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.819193+0000", 2024-09-13T14:41:44.214 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.214 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.214 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.214 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.214 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.819193+0000", 2024-09-13T14:41:44.214 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.215 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.215 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.215 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.215 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.819193+0000", 2024-09-13T14:41:44.215 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:41:44.215 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.215 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.215 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.215 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.819195+0000", 2024-09-13T14:41:44.215 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.092e-06 2024-09-13T14:41:44.215 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.215 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.216 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.216 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.819201+0000", 2024-09-13T14:41:44.216 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.524e-06 2024-09-13T14:41:44.216 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.216 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.216 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.216 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.819204+0000", 2024-09-13T14:41:44.216 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4750000000000002e-06 2024-09-13T14:41:44.216 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.216 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.216 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.216 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.823501+0000", 2024-09-13T14:41:44.216 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0042965240000000003 2024-09-13T14:41:44.217 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.217 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.217 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.217 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.823616+0000", 2024-09-13T14:41:44.217 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011554500000000001 2024-09-13T14:41:44.217 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.217 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.217 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:41:44.217 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.823727+0000", 2024-09-13T14:41:44.217 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000110358 2024-09-13T14:41:44.217 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.217 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.217 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.218 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.832761+0000", 2024-09-13T14:41:44.218 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0090342809999999999 2024-09-13T14:41:44.218 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.218 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.218 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.218 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.832802+0000", 2024-09-13T14:41:44.218 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1498000000000001e-05 2024-09-13T14:41:44.218 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.218 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.218 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.218 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.880117+0000", 2024-09-13T14:41:44.218 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.047315090999999997 2024-09-13T14:41:44.218 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.218 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.219 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.219 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.880215+0000", 2024-09-13T14:41:44.219 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7244000000000006e-05 2024-09-13T14:41:44.219 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.219 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.219 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.219 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.880259+0000", 2024-09-13T14:41:44.219 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3971999999999998e-05 2024-09-13T14:41:44.219 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.219 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.219 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.219 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.219 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.219 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:149133 5.e 5:75a573e8:::benchmark_data_smithi096_138841_object18937:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-09-13T14:41:44.220 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:41:41.819450+0000", 2024-09-13T14:41:44.220 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 2.2783785710000002, 2024-09-13T14:41:44.220 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.060952517999999997, 2024-09-13T14:41:44.220 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.220 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.220 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.220 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.220 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.220 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 149133 2024-09-13T14:41:44.220 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.220 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.220 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.220 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.221 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.819450+0000", 2024-09-13T14:41:44.221 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.221 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.221 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.221 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.221 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.819450+0000", 2024-09-13T14:41:44.221 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.221 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.221 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.221 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.221 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.819449+0000", 2024-09-13T14:41:44.221 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:41:44.221 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.222 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.222 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.222 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.819454+0000", 2024-09-13T14:41:44.222 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3680000000000004e-06 2024-09-13T14:41:44.222 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.222 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.222 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.222 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.819462+0000", 2024-09-13T14:41:44.222 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7430000000000008e-06 2024-09-13T14:41:44.222 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.222 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.222 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.222 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.819467+0000", 2024-09-13T14:41:44.223 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8040000000000001e-06 2024-09-13T14:41:44.223 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.223 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.223 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.223 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.826969+0000", 2024-09-13T14:41:44.223 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0075013349999999996 2024-09-13T14:41:44.223 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.223 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.223 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.223 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.827044+0000", 2024-09-13T14:41:44.223 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5680999999999996e-05 2024-09-13T14:41:44.223 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.223 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.224 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:41:44.224 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.827108+0000", 2024-09-13T14:41:44.224 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4002000000000001e-05 2024-09-13T14:41:44.224 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.224 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.224 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.224 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.832768+0000", 2024-09-13T14:41:44.224 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0056593499999999996 2024-09-13T14:41:44.224 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.224 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.224 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.225 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.835844+0000", 2024-09-13T14:41:44.225 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0030761199999999999 2024-09-13T14:41:44.225 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.225 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.225 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.225 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.880292+0000", 2024-09-13T14:41:44.225 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.044448204999999998 2024-09-13T14:41:44.225 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.225 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.225 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.225 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.880366+0000", 2024-09-13T14:41:44.225 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.3644000000000002e-05 2024-09-13T14:41:44.225 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.226 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.226 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.226 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.880403+0000", 2024-09-13T14:41:44.226 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7004e-05 2024-09-13T14:41:44.226 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.226 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.226 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.226 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.226 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.226 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:149135 5.e 5:72fbba5e:::benchmark_data_smithi096_138841_object18939:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-09-13T14:41:44.226 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:41:41.824199+0000", 2024-09-13T14:41:44.226 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 2.2736297159999999, 2024-09-13T14:41:44.227 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.063356735999999997, 2024-09-13T14:41:44.227 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:44.227 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:44.227 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:44.227 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:44.227 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:44.227 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 149135 2024-09-13T14:41:44.227 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.227 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:44.227 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.227 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:44.227 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.824199+0000", 2024-09-13T14:41:44.228 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.228 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.228 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.228 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:44.228 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.824199+0000", 2024-09-13T14:41:44.228 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:44.228 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.228 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.228 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:44.228 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.824196+0000", 2024-09-13T14:41:44.228 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999976 2024-09-13T14:41:44.228 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.228 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.228 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:44.229 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.824204+0000", 2024-09-13T14:41:44.229 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9019999999999999e-06 2024-09-13T14:41:44.229 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.229 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.229 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:44.229 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.824217+0000", 2024-09-13T14:41:44.229 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2870999999999999e-05 2024-09-13T14:41:44.229 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.229 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.229 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:44.229 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.824223+0000", 2024-09-13T14:41:44.229 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.711e-06 2024-09-13T14:41:44.230 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.230 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.230 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:44.230 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.829693+0000", 2024-09-13T14:41:44.230 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0054695960000000002 2024-09-13T14:41:44.230 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.230 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.230 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:44.230 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.829792+0000", 2024-09-13T14:41:44.230 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9921999999999996e-05 2024-09-13T14:41:44.230 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.230 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.231 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:41:44.231 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.829858+0000", 2024-09-13T14:41:44.231 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5345000000000005e-05 2024-09-13T14:41:44.231 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.231 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.231 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:44.231 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.832772+0000", 2024-09-13T14:41:44.231 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029140949999999998 2024-09-13T14:41:44.231 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.231 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.231 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.231 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.835878+0000", 2024-09-13T14:41:44.231 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0031065580000000001 2024-09-13T14:41:44.231 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.232 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.232 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:44.232 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.887449+0000", 2024-09-13T14:41:44.232 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.051570181 2024-09-13T14:41:44.232 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.232 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.232 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:41:44.232 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.887518+0000", 2024-09-13T14:41:44.232 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9068999999999999e-05 2024-09-13T14:41:44.232 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:44.232 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:44.232 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:41:44.232 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:41.887556+0000", 2024-09-13T14:41:44.233 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7867999999999997e-05 2024-09-13T14:41:44.233 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.233 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.233 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.233 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:44.233 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:44.233 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:41:44.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:44.273 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:44.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:44.375 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:44.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:44.412 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:44.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:44.513 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:44.545 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:44.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:44.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:44.647 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:44.686 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:44.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:44.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:44.788 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:44.816 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:44.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:44.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:44.918 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:44.957 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:44.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:44.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:45.017 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:41:45.058 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:45.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:45.097 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:45.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:45.114 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:41:45.198 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:45.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:45.231 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:45.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:45.318 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:45 smithi096 ceph-mon[98901]: pgmap v324: 99 pgs: 99 active+clean; 6.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 954 op/s 2024-09-13T14:41:45.318 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:45 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2609969589' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:41:45.319 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:45 smithi096 ceph-mon[102928]: pgmap v324: 99 pgs: 99 active+clean; 6.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 954 op/s 2024-09-13T14:41:45.319 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:45 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2609969589' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:41:45.332 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:45.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:45.374 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:45.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:45.392 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:41:45.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:45 smithi204 ceph-mon[87071]: pgmap v324: 99 pgs: 99 active+clean; 6.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 954 op/s 2024-09-13T14:41:45.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:45 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2609969589' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:41:45.475 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:45.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:45.504 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:45.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:45.606 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:45.641 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:45.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:45.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:45.743 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:45.772 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:45.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:45.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:45.874 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:45.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:45.909 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:45.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:46.010 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:46.038 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:46.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:46.140 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:46.178 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:46.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:46.280 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:46.310 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:46.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:46 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:46 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2609969589' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:41:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:46 smithi096 ceph-mon[98901]: osdmap e130: 8 total, 8 up, 7 in 2024-09-13T14:41:46.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:46 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:46.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:46 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2609969589' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:41:46.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:46 smithi096 ceph-mon[102928]: osdmap e130: 8 total, 8 up, 7 in 2024-09-13T14:41:46.412 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:46.437 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:46.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:46.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:46 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:46.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:46 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2609969589' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:41:46.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:46 smithi204 ceph-mon[87071]: osdmap e130: 8 total, 8 up, 7 in 2024-09-13T14:41:46.540 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:46.572 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:46.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:46.674 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:46.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.712 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:46.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:46.813 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:46.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.848 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:46.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:46.949 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:46.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.985 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:46.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:47.086 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:47.119 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:47.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:47.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:47.221 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:47.259 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:47.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:47.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:47.361 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:47.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:47.395 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:47.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:47.426 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:47 smithi204 ceph-mon[87071]: pgmap v326: 99 pgs: 99 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 909 op/s 2024-09-13T14:41:47.496 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:47.529 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:47.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:47.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:47.571 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:41:47.572 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:41:47.572 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.572 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:154900 5.7 5:ef0ca01e:::benchmark_data_smithi096_138841_object24704:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:41:47.572 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:41:47.552723+0000", 2024-09-13T14:41:47.572 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.017683397, 2024-09-13T14:41:47.572 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.017707276000000001, 2024-09-13T14:41:47.572 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:47.572 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:41:47.572 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:47.572 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:47.572 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:47.572 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 154900 2024-09-13T14:41:47.573 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.573 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:47.573 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.573 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:47.573 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.552723+0000", 2024-09-13T14:41:47.573 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:47.573 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.573 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.573 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:47.573 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.552723+0000", 2024-09-13T14:41:47.573 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:47.573 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.574 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.574 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:47.574 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.552722+0000", 2024-09-13T14:41:47.574 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:41:47.574 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.574 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.574 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:47.574 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.552727+0000", 2024-09-13T14:41:47.574 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4149999999999998e-06 2024-09-13T14:41:47.574 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.574 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.574 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:47.574 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.552738+0000", 2024-09-13T14:41:47.574 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0294e-05 2024-09-13T14:41:47.575 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.575 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.575 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:47.575 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.552743+0000", 2024-09-13T14:41:47.575 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5130000000000004e-06 2024-09-13T14:41:47.575 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.575 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.575 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:47.575 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.552790+0000", 2024-09-13T14:41:47.575 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6536999999999998e-05 2024-09-13T14:41:47.575 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.575 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.576 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:47.576 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.552940+0000", 2024-09-13T14:41:47.576 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015057199999999999 2024-09-13T14:41:47.576 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.576 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.576 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:41:47.576 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.553488+0000", 2024-09-13T14:41:47.576 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00054799299999999998 2024-09-13T14:41:47.576 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.576 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.576 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:47.576 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.557067+0000", 2024-09-13T14:41:47.577 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035786120000000001 2024-09-13T14:41:47.577 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.577 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.577 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:47.577 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.560022+0000", 2024-09-13T14:41:47.577 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029556809999999999 2024-09-13T14:41:47.577 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:47.577 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:47.577 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:47.577 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.577 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.578 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:154914 5.7 5:e617408c:::benchmark_data_smithi096_138841_object24718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:41:47.578 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:41:47.566183+0000", 2024-09-13T14:41:47.578 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0042232499999999996, 2024-09-13T14:41:47.578 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0042996570000000001, 2024-09-13T14:41:47.578 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:47.578 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:41:47.578 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:47.578 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:47.578 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:47.578 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 154914 2024-09-13T14:41:47.578 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.578 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:47.578 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.578 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:47.579 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566183+0000", 2024-09-13T14:41:47.579 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:47.579 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.579 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.579 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:47.579 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566183+0000", 2024-09-13T14:41:47.579 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:47.579 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.579 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.579 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:47.579 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566181+0000", 2024-09-13T14:41:47.579 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:41:47.579 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.580 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.580 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:47.580 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566189+0000", 2024-09-13T14:41:47.580 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7540000000000006e-06 2024-09-13T14:41:47.580 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.580 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.580 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:47.580 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566216+0000", 2024-09-13T14:41:47.580 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7267000000000002e-05 2024-09-13T14:41:47.580 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.580 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.580 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:47.580 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566223+0000", 2024-09-13T14:41:47.581 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9929999999999998e-06 2024-09-13T14:41:47.581 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.581 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.581 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:47.581 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566273+0000", 2024-09-13T14:41:47.581 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9670999999999999e-05 2024-09-13T14:41:47.581 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.581 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.581 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:47.581 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566413+0000", 2024-09-13T14:41:47.581 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013962499999999999 2024-09-13T14:41:47.581 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.581 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.581 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:41:47.582 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566558+0000", 2024-09-13T14:41:47.582 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014523800000000001 2024-09-13T14:41:47.582 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.582 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.582 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:41:47.582 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.569805+0000", 2024-09-13T14:41:47.582 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003246947 2024-09-13T14:41:47.582 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:47.582 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:47.582 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:47.582 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.582 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.582 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:154915 5.f 5:f8debdf1:::benchmark_data_smithi096_138841_object24719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:41:47.583 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:41:47.566506+0000", 2024-09-13T14:41:47.583 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0038997540000000001, 2024-09-13T14:41:47.583 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0040182059999999999, 2024-09-13T14:41:47.583 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:41:47.583 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:41:47.583 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:41:47.583 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:41:47.583 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:41:47.583 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 154915 2024-09-13T14:41:47.583 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.583 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:41:47.583 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.583 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:41:47.583 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566506+0000", 2024-09-13T14:41:47.584 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:47.584 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.584 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.584 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:41:47.584 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566506+0000", 2024-09-13T14:41:47.584 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:41:47.584 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.584 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.584 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:41:47.584 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566505+0000", 2024-09-13T14:41:47.584 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:41:47.584 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.585 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.585 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:41:47.585 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566511+0000", 2024-09-13T14:41:47.585 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7570000000000001e-06 2024-09-13T14:41:47.585 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.585 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.585 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:41:47.585 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566523+0000", 2024-09-13T14:41:47.585 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1711e-05 2024-09-13T14:41:47.585 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.585 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.585 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:41:47.585 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566530+0000", 2024-09-13T14:41:47.586 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6479999999999997e-06 2024-09-13T14:41:47.586 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.586 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.586 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:41:47.586 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.566804+0000", 2024-09-13T14:41:47.586 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00027430500000000002 2024-09-13T14:41:47.586 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.586 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.586 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:41:47.586 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.567131+0000", 2024-09-13T14:41:47.586 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00032688999999999998 2024-09-13T14:41:47.586 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:41:47.586 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:41:47.587 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:41:47.587 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.567307+0000", 2024-09-13T14:41:47.587 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00017637800000000001 2024-09-13T14:41:47.587 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:47.587 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:41:47.587 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:47.587 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:41:47.587 INFO:teuthology.orchestra.run.smithi204.stdout: ], 2024-09-13T14:41:47.587 INFO:teuthology.orchestra.run.smithi204.stdout: "num_ops": 3 2024-09-13T14:41:47.587 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:41:47.588 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:47 smithi096 ceph-mon[98901]: pgmap v326: 99 pgs: 99 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 909 op/s 2024-09-13T14:41:47.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:47 smithi096 ceph-mon[102928]: pgmap v326: 99 pgs: 99 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 909 op/s 2024-09-13T14:41:47.632 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:47.660 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:47.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:47.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:47.762 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:47.800 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:47.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:47.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:47.902 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:47.936 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:47.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:47.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:48.038 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:48.058 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:48.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:48.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:48.082 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:41:48.160 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:48.188 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:48.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:48.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:48.290 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:48.308 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:41:48.321 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:48.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:48.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:48.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:41:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:41:47] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.51.0" 2024-09-13T14:41:48.424 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:48.464 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:48.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:48.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:48.566 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:48.611 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:48.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:48.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:48.696 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:41:48.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:48.754 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:48.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:48.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:48.856 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:48.880 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:48.881 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:48.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:48.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:48.983 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:49.018 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:49.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:49.120 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:49.143 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:49.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:49.245 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:49.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.289 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:49.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:49.390 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:49.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.421 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:49.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:49 smithi204 ceph-mon[87071]: pgmap v327: 99 pgs: 99 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 916 op/s 2024-09-13T14:41:49.523 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:49.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.562 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:49.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:49.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:49 smithi096 ceph-mon[98901]: pgmap v327: 99 pgs: 99 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 916 op/s 2024-09-13T14:41:49.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:49 smithi096 ceph-mon[102928]: pgmap v327: 99 pgs: 99 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 916 op/s 2024-09-13T14:41:49.662 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:49.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.699 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:49.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:49.800 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:49.831 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:49.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:49.933 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:49.967 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:49.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:50.069 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:50.106 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:50.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:50.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:50.208 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:50.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:50.244 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:50.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:50.345 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:50.373 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:50.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:50.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:50.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:41:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:41:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:50.476 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:50.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:50.516 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:50.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:50.617 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:50.654 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:50.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:50.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:50.756 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:50.789 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:50.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:50.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:50.893 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:50.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:50.931 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:50.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:51.032 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:51.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:51.079 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:51.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:51.180 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:51.187 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:51 smithi204 ceph-mon[87071]: pgmap v328: 99 pgs: 99 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:41:51.187 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:41:51.188 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:51.214 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:51.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:51.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:51.223 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:51.316 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:51.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:51.363 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:51.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:51.464 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:51.471 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:51 smithi096 ceph-mon[102928]: pgmap v328: 99 pgs: 99 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:41:51.471 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:41:51.471 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:51.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:51 smithi096 ceph-mon[98901]: pgmap v328: 99 pgs: 99 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:41:51.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:41:51.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:51.494 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:51.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:51.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:51.596 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:51.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:51.633 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:51.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:51.645 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:41:51.645 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:41:51.646 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T14:41:51.646 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T14:41:51.646 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:41:51.734 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:51.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:51.769 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:51.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:51.870 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:51.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:51.907 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:51.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:52.008 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:52.047 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:52.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:52.149 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:52.177 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:52.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:52.203 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:41:52.204 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:41:52.204 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:41:52.204 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:41:52.204 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:41:52.204 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:41:52.205 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:41:52.205 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:41:52.205 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:41:52.205 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:41:52.238 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:41:52.238 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:41:52.266 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:41:52.266 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:41:52.279 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:52.294 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:41:52.294 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:41:52.309 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:52.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:52.315 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:52 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/336527728' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:52.316 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:52 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/336527728' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:52.411 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:52.442 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:52.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:52.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:52 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/336527728' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:52.544 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:52.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.586 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:52.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:52.687 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:52.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.728 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:52.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:52.829 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:52.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.864 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:52.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:52.965 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:52.990 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:53.092 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:53.116 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:41:53.132 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:53.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:53.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:53.235 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:53.267 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:53.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:53.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:53.369 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:53.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:53.428 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:53.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:53.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:53 smithi204 ceph-mon[87071]: pgmap v329: 99 pgs: 99 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:41:53.495 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:41:53.529 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:53.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:53.570 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:53.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:53.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:53 smithi096 ceph-mon[98901]: pgmap v329: 99 pgs: 99 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:41:53.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:53 smithi096 ceph-mon[102928]: pgmap v329: 99 pgs: 99 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:41:53.671 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:53.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:53.715 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:53.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:53.816 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:53.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:53.854 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:53.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:53.955 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:54.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:54.002 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:54.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:54.104 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:54.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:54.133 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:54.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:54.192 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T14:41:54.234 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:54.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:54.268 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:54.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:54.369 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:54.397 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:54.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:54.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:54.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:54 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3956563005' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:41:54.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:54 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:41:54.499 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:54.527 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:54.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:54.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:54.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:54 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3956563005' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:41:54.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:54 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:41:54.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:54 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3956563005' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:41:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:54 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:41:54.628 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:54.664 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:54.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:54.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:54.766 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:54.806 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:54.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:54.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:54.908 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:54.932 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:54.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:54.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:55.034 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:55.066 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:55.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:55.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:55.168 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:55.209 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:55.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:55.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:55.311 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:55.342 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:55 smithi204 ceph-mon[87071]: pgmap v330: 99 pgs: 99 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 977 op/s 2024-09-13T14:41:55.342 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:55 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:41:55.342 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:55 smithi204 ceph-mon[87071]: osdmap e131: 8 total, 8 up, 7 in 2024-09-13T14:41:55.353 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:55.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:55.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:55.415 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T14:41:55.452 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:55 smithi096 ceph-mon[98901]: pgmap v330: 99 pgs: 99 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 977 op/s 2024-09-13T14:41:55.452 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:55 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:41:55.452 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:55 smithi096 ceph-mon[98901]: osdmap e131: 8 total, 8 up, 7 in 2024-09-13T14:41:55.452 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:55 smithi096 ceph-mon[102928]: pgmap v330: 99 pgs: 99 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 977 op/s 2024-09-13T14:41:55.453 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:55 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:41:55.453 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:55 smithi096 ceph-mon[102928]: osdmap e131: 8 total, 8 up, 7 in 2024-09-13T14:41:55.457 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:55.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:55.509 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:55.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:55.610 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:55.630 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:41:55.630 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:41:55.639 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:55.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:55.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:55.679 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:41:55.741 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:55.767 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:55.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:55.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:55.868 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:55.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:55.910 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:55.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:56.011 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:56.058 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:56.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:56.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:56.119 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:56.160 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:56.175 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:56.184 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:56.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:56.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:56.286 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:56.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:56.331 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:56.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:56.432 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:56.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:56.457 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:56.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:56.558 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:56.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:56.601 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:56.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:56.702 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:56.723 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:41:56.745 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:56.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:56.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:56.847 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:56.890 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:56.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:56.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:56.992 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:57.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.032 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:57.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:57.126 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:41:57.133 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:57.168 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:57.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:57.200 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:41:57.270 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:57.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.308 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:57.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:57.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:57 smithi096 ceph-mon[98901]: pgmap v332: 99 pgs: 99 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 976 op/s 2024-09-13T14:41:57.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:57 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2030155226' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:57.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:57 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:57.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:57 smithi096 ceph-mon[102928]: pgmap v332: 99 pgs: 99 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 976 op/s 2024-09-13T14:41:57.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:57 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2030155226' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:57.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:57 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:57.409 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:57.434 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:57.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:57 smithi204 ceph-mon[87071]: pgmap v332: 99 pgs: 99 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 976 op/s 2024-09-13T14:41:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:57 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2030155226' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:57 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:57.536 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:57.577 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:57.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:57.679 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:57.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.724 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:57.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:57.825 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:57.859 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:57.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:57.960 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:57.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.994 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:57.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:58.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:41:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:41:57] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.51.0" 2024-09-13T14:41:58.095 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:58.130 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:58.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:58.232 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:58.263 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:58.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:58.365 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:58.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.394 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:58.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:58.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:58 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:58.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:58 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:41:58.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:58 smithi204 ceph-mon[87071]: osdmap e132: 8 total, 8 up, 7 in 2024-09-13T14:41:58.492 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:58 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:58.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:58 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:41:58.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:58 smithi096 ceph-mon[98901]: osdmap e132: 8 total, 8 up, 7 in 2024-09-13T14:41:58.493 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:58 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:58.493 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:58 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:41:58.493 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:58 smithi096 ceph-mon[102928]: osdmap e132: 8 total, 8 up, 7 in 2024-09-13T14:41:58.495 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:58.538 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:58.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:58.640 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:58.685 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:58.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:58.787 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:58.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.817 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:58.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:58.918 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:58.954 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:58.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:59.056 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:59.086 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:59.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:59.188 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:59.225 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:59.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:59.253 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:41:59 smithi204 ceph-mon[87071]: pgmap v334: 99 pgs: 99 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 902 op/s 2024-09-13T14:41:59.275 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:41:59.275 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:41:59.326 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:59.356 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:59.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:59.458 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:59.499 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:41:59.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:59.535 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:41:59 smithi096 ceph-mon[98901]: pgmap v334: 99 pgs: 99 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 902 op/s 2024-09-13T14:41:59.536 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:41:59 smithi096 ceph-mon[102928]: pgmap v334: 99 pgs: 99 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 902 op/s 2024-09-13T14:41:59.601 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:59.625 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:59.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:59.728 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:59.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.761 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:59.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:59.862 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:59.892 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:41:59.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:59.971 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:41:59.994 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:00.047 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:00.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:00.149 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:00.182 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:00.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:00.233 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T14:42:00.284 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:00.322 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:00.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:00.350 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:42:00.406 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:42:00.425 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:00.452 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:00.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:00.483 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:00 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3130813173' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:42:00.483 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:42:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:42:00.216Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:00.499 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:00 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3130813173' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:42:00.499 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:00 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3130813173' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:42:00.554 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:00.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.613 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:00.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:00.714 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:00.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.761 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:00.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:00.862 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:00.887 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:00.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:00.989 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:01.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.023 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:01.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:01.124 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:01.158 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:01.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:01.260 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:01.283 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:01.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:01.385 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:01.418 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:01.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:01.520 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:01.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:01 smithi204 ceph-mon[87071]: pgmap v335: 99 pgs: 99 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 997 op/s 2024-09-13T14:42:01.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:01 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3130813173' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:42:01.523 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:01 smithi204 ceph-mon[87071]: osdmap e133: 8 total, 8 up, 7 in 2024-09-13T14:42:01.555 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:01.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:01 smithi096 ceph-mon[98901]: pgmap v335: 99 pgs: 99 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 997 op/s 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:01 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3130813173' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:01 smithi096 ceph-mon[98901]: osdmap e133: 8 total, 8 up, 7 in 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:01 smithi096 ceph-mon[102928]: pgmap v335: 99 pgs: 99 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 997 op/s 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:01 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3130813173' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:01 smithi096 ceph-mon[102928]: osdmap e133: 8 total, 8 up, 7 in 2024-09-13T14:42:01.657 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:01.710 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:01.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:01.812 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:01.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.856 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:01.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:01.957 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:02.005 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:02.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:02.093 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T14:42:02.108 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:02.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.192 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:02.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:02.257 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:42:02.258 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:42:02.293 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:02.338 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:02.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:02.440 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:02.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.477 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:02.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:02.578 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:02.622 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:02.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:02.724 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:02.728 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:02.728 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:42:02.729 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:42:02.747 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:02.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:02.777 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:42:02.777 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:42:02.808 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:42:02.808 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:42:02.846 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:42:02.846 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:42:02.849 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:02.884 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:02.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:02.887 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:42:02.888 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:42:02.923 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:42:02.923 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:42:02.957 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:42:02.957 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:42:02.958 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:42:02.985 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:42:02.985 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:42:02.986 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:03.029 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:03.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:03.131 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:03.169 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:03.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:03.240 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:03 smithi096 ceph-mon[98901]: pgmap v337: 99 pgs: 99 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 905 op/s 2024-09-13T14:42:03.240 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:03 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2814306306' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:03.270 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:03.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.319 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:03.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:03.352 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:42:03.420 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:03.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.461 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:03.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:03.522 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:03 smithi204 ceph-mon[87071]: pgmap v337: 99 pgs: 99 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 905 op/s 2024-09-13T14:42:03.522 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:03 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2814306306' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:03.562 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:03.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:03 smithi096 ceph-mon[102928]: pgmap v337: 99 pgs: 99 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 905 op/s 2024-09-13T14:42:03.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:03 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2814306306' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:03.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.597 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:03.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:03.698 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:03.730 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:03.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:03.831 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:03.865 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:03.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:03.968 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:03.996 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:03.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:04.098 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:04.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.133 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:04.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:04.144 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:04.234 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:04.273 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:04.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:04.375 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:04.408 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:04.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:04.510 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:04.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.539 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:04.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:04.640 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:04.676 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:04.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:04.778 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:04.809 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:04.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:04.910 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:04.950 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:04.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:05.052 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:05.084 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:05.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:05.186 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:05.226 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:05.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:05.263 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:42:05.263 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T14:42:05.263 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T14:42:05.263 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [] 2024-09-13T14:42:05.263 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:42:05.328 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:05.360 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:05.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:05.462 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:05.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:05 smithi204 ceph-mon[87071]: pgmap v338: 99 pgs: 99 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:42:05.488 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:05 smithi096 ceph-mon[98901]: pgmap v338: 99 pgs: 99 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:42:05.489 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:05 smithi096 ceph-mon[102928]: pgmap v338: 99 pgs: 99 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:42:05.490 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:05.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:05.592 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:05.621 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:05.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:05.723 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:05.759 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:05.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:05.861 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:05.893 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:05.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:05.995 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:06.028 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:06.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:06.130 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:06.162 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:06.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:06.202 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:42:06.264 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:06.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.336 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:06.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:06.406 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:06.406 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:42:06.406 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:06.406 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:42:06.437 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:06.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:06.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:42:06.474 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:06.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:06.576 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:06.792 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:42:06.836 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:06.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:06.938 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:06.981 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:06.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:07.083 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:07.147 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:07.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:07.249 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:07.274 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:42:07.298 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:07 smithi096 ceph-mon[98901]: pgmap v339: 99 pgs: 99 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 875 op/s 2024-09-13T14:42:07.298 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:07 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3030301632' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:42:07.298 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:07 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:42:07.298 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:07 smithi096 ceph-mon[102928]: pgmap v339: 99 pgs: 99 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 875 op/s 2024-09-13T14:42:07.298 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:07 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3030301632' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:42:07.298 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:07 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:42:07.301 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:07.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:07.403 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:07.447 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:07.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:07.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:07 smithi204 ceph-mon[87071]: pgmap v339: 99 pgs: 99 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 875 op/s 2024-09-13T14:42:07.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:07 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3030301632' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:42:07.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:07 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:42:07.550 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:07.575 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:07.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:07.676 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:07.709 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:07.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:07.810 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:07.836 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:07.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:07.938 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:07.971 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:07.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:08.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:42:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:42:07] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-13T14:42:08.073 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:08.097 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:08.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:08.198 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:08.249 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:08.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:08.350 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:08.364 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:08 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:08.364 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:08 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:42:08.364 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:08 smithi204 ceph-mon[87071]: osdmap e134: 8 total, 8 up, 7 in 2024-09-13T14:42:08.385 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:08.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:08.487 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:08.529 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:08.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:08.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:08 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:08.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:08 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:42:08.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:08 smithi096 ceph-mon[98901]: osdmap e134: 8 total, 8 up, 7 in 2024-09-13T14:42:08.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:08 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:08.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:08 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:42:08.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:08 smithi096 ceph-mon[102928]: osdmap e134: 8 total, 8 up, 7 in 2024-09-13T14:42:08.631 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:08.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.689 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:08.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:08.790 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:08.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.824 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:08.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:08.925 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:08.954 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:08.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:09.058 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:09.082 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:42:09.086 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:42:09.086 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:42:09.086 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:42:09.086 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.35049.0:175587 5.d e134/107 5:b0af944e:::benchmark_data_smithi096_138841_object45391:head v 134'11051, mlcod=134'11049)", 2024-09-13T14:42:09.086 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:42:09.082362+0000", 2024-09-13T14:42:09.087 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0025770160000000001, 2024-09-13T14:42:09.087 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0026087229999999999, 2024-09-13T14:42:09.087 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:42:09.087 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:42:09.087 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:42:09.087 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:42:09.087 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:42:09.087 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:42:09.082362+0000", 2024-09-13T14:42:09.087 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:42:09.087 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:42:09.087 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:42:09.087 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:42:09.087 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:42:09.082362+0000", 2024-09-13T14:42:09.087 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:42:09.088 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:42:09.088 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:42:09.088 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:42:09.088 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:42:09.082361+0000", 2024-09-13T14:42:09.088 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:42:09.088 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:42:09.088 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:42:09.088 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:42:09.088 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:42:09.082367+0000", 2024-09-13T14:42:09.088 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9780000000000002e-06 2024-09-13T14:42:09.088 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:42:09.088 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:42:09.088 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:42:09.089 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:42:09.082373+0000", 2024-09-13T14:42:09.089 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.925e-06 2024-09-13T14:42:09.089 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:42:09.089 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:42:09.089 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:42:09.089 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:42:09.082377+0000", 2024-09-13T14:42:09.089 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.478e-06 2024-09-13T14:42:09.089 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:42:09.089 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:42:09.089 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:42:09.089 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:42:09.082436+0000", 2024-09-13T14:42:09.089 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9049000000000003e-05 2024-09-13T14:42:09.089 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:42:09.089 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:42:09.090 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:42:09.090 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:42:09.082458+0000", 2024-09-13T14:42:09.090 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2018000000000002e-05 2024-09-13T14:42:09.090 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:42:09.090 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:42:09.090 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:42:09.090 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:42:09.090 INFO:teuthology.orchestra.run.smithi204.stdout: ], 2024-09-13T14:42:09.090 INFO:teuthology.orchestra.run.smithi204.stdout: "num_ops": 1 2024-09-13T14:42:09.090 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:42:09.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.091 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:09.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:09.192 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:09.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.225 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:09.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:09.305 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T14:42:09.326 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:09.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.385 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:09.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:09.486 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:09.519 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:09.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:09.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:09 smithi096 ceph-mon[98901]: pgmap v341: 99 pgs: 99 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 926 op/s 2024-09-13T14:42:09.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:09 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2928529562' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:42:09.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:09 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:42:09.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:09 smithi096 ceph-mon[102928]: pgmap v341: 99 pgs: 99 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 926 op/s 2024-09-13T14:42:09.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:09 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2928529562' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:42:09.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:09 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:42:09.621 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:09.666 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:09.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:09.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:09 smithi204 ceph-mon[87071]: pgmap v341: 99 pgs: 99 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 926 op/s 2024-09-13T14:42:09.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:09 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2928529562' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:42:09.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:09 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:42:09.768 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:09.828 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:09.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:09.867 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:42:09.930 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:09.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.983 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:09.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:10.085 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:10.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.152 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:10.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:10.215 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:42:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:42:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:10.253 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:10.288 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:10.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:10.390 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:10.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.432 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:10.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:10.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:10 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:42:10.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:10 smithi204 ceph-mon[87071]: osdmap e135: 8 total, 8 up, 7 in 2024-09-13T14:42:10.479 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:42:10.531 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T14:42:10.531 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:42:10.533 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:10.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.569 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:10.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:10.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:10 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:42:10.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:10 smithi096 ceph-mon[98901]: osdmap e135: 8 total, 8 up, 7 in 2024-09-13T14:42:10.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:10 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:42:10.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:10 smithi096 ceph-mon[102928]: osdmap e135: 8 total, 8 up, 7 in 2024-09-13T14:42:10.596 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:10.670 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:10.698 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:10.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:10.800 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:10.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.830 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:10.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:10.931 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:10.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.969 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:10.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:10.973 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:42:11.070 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:11.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.116 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:11.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:11.217 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:11.242 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:11.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:11.344 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:11.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.386 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:11.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:11.487 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:11.527 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:11 smithi204 ceph-mon[87071]: pgmap v343: 99 pgs: 99 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 991 op/s 2024-09-13T14:42:11.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.555 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:11.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:11.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:11 smithi096 ceph-mon[102928]: pgmap v343: 99 pgs: 99 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 991 op/s 2024-09-13T14:42:11.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:11 smithi096 ceph-mon[98901]: pgmap v343: 99 pgs: 99 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 991 op/s 2024-09-13T14:42:11.657 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:11.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.699 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:11.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:11.800 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:11.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.840 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:11.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:11.941 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:11.985 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:11.991 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:11.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:12.037 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:12.093 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:12.149 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:12.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:12.251 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:12.300 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:12.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:12.402 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:12.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.445 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:12.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:12.546 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:12.580 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:12.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:12.682 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:12.716 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:12.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:12.818 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:12.856 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:12.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:12.958 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:12.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.993 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:12.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:13.025 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:42:13.025 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:42:13.025 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T14:42:13.025 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T14:42:13.025 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:42:13.094 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:13.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.132 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:13.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:13.233 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:13.276 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:13.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:13.378 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:13.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.407 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:13.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:13.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:13 smithi096 ceph-mon[98901]: pgmap v344: 99 pgs: 99 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 861 op/s 2024-09-13T14:42:13.477 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:13 smithi096 ceph-mon[102928]: pgmap v344: 99 pgs: 99 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 861 op/s 2024-09-13T14:42:13.508 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:13.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.561 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:13.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:13.662 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:13.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.695 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:13.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:13.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:13 smithi204 ceph-mon[87071]: pgmap v344: 99 pgs: 99 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 861 op/s 2024-09-13T14:42:13.796 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:13.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.833 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:13.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:13.903 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:42:13.904 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:42:13.934 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:13.937 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:42:13.937 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:42:13.973 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:13.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:13.976 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:42:13.977 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:42:14.009 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:42:14.009 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:42:14.039 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:42:14.040 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:42:14.072 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:42:14.073 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:42:14.075 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:14.101 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:42:14.101 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:42:14.111 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:42:14.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.127 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:14.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:14.133 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:42:14.133 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:42:14.228 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:14.258 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:14.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:14.360 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:14.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.392 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:14.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:14.493 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:14.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.528 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:14.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:14.629 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:14.645 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:42:14.663 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:14.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:14.697 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:14 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4138344393' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:14.697 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:14 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4138344393' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:14.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:14 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4138344393' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:14.765 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:14.803 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:14.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:14.904 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:14.937 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:14.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:15.039 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:15.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.077 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:15.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:15.178 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:15.205 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:15.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:15.306 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:15.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.346 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:15.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:15.448 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:15.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.475 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:15.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:15.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:15 smithi096 ceph-mon[98901]: pgmap v345: 99 pgs: 99 active+clean; 5.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 987 op/s 2024-09-13T14:42:15.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:15 smithi096 ceph-mon[102928]: pgmap v345: 99 pgs: 99 active+clean; 5.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 987 op/s 2024-09-13T14:42:15.576 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:15.614 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:15.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:15.717 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:15.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:15 smithi204 ceph-mon[87071]: pgmap v345: 99 pgs: 99 active+clean; 5.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 987 op/s 2024-09-13T14:42:15.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.751 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:15.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:15.852 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:15.878 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:15.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:15.980 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:16.020 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:16.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:16.122 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:16.144 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:16.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:16.246 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:16.280 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:16.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:16.381 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:16.421 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:16.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:16.523 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:16.569 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:16.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:16.670 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:16.699 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:16.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:16.802 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:16.831 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:16.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:16.933 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:16.970 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:16.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:17.072 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:17.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.107 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:17.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:17.208 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:17.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.253 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:17.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:17.354 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:17.389 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:17.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:17.461 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:17 smithi096 ceph-mon[98901]: pgmap v346: 99 pgs: 99 active+clean; 5.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 817 op/s 2024-09-13T14:42:17.461 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:17 smithi096 ceph-mon[102928]: pgmap v346: 99 pgs: 99 active+clean; 5.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 817 op/s 2024-09-13T14:42:17.491 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:17.519 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:17.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:17.583 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:17 smithi204 ceph-mon[87071]: pgmap v346: 99 pgs: 99 active+clean; 5.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 817 op/s 2024-09-13T14:42:17.621 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:17.658 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:17.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:17.728 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:42:17.728 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:42:17.760 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:17.790 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:17.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:17.892 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:17.921 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:17.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:18.023 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:18.064 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:18.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:18.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:42:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:42:17] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-13T14:42:18.166 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:18.198 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:18.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:18.300 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:18.329 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:18.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:18.431 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:18.461 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:18.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:18.563 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:18.603 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:18.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:18.653 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:42:18.705 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:18.739 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:18.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:18.741 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:18 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:18.741 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:18 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:18.741 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:18 smithi096 ceph-mon[98901]: pgmap v347: 99 pgs: 99 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 938 op/s 2024-09-13T14:42:18.741 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:18 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:18.742 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:18 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:18.742 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:18 smithi096 ceph-mon[102928]: pgmap v347: 99 pgs: 99 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 938 op/s 2024-09-13T14:42:18.840 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:18.865 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:18.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:18.967 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:18.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:18 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:18.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:18 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:18.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:18 smithi204 ceph-mon[87071]: pgmap v347: 99 pgs: 99 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 938 op/s 2024-09-13T14:42:19.030 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:19.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:19.132 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:19.157 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:19.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:19.259 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:19.288 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:19.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:19.390 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:19.443 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:19.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:19.500 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:42:19.546 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:19.546 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:42:19.570 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:19.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:19.642 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:42:19.644 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:42:19.672 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:19.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.742 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:19.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:19.790 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:19 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2398951845' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:19.790 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:19 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:19.790 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:19 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2398951845' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:19.790 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:19 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:19.843 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:19.882 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:19.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:19.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:19 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2398951845' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:19.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:19 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:19.984 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:20.009 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:20.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:20.111 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:20.156 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:20.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:20.258 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:20.289 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:20.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:20.391 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:20.416 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:20.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:20.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:42:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:42:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:20.512 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":348,"stamp":"2024-09-13T14:42:19.540864+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5010167781,"num_objects":76405,"num_object_clones":0,"num_object_copies":229215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76405,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":184918,"num_write_kb":8344976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":4203971,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119643,"ondisk_log_size":119643,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16196256,"kb_used_data":14815080,"kb_used_omap":34,"kb_used_meta":1381021,"kb_avail":640019808,"statfs":{"total":671965249536,"available":655380283392,"internally_reserved":0,"allocated":15170641920,"data_stored":15165093348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35126,"internal_metadata":1414166218},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-627310592,"num_objects":-9572,"num_object_clones":0,"num_object_copies":-28716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9572,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.241074"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":134,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.311398+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:42:09.311398+0000","last_peered":"2024-09-13T14:42:09.311398+0000","last_clean":"2024-09-13T14:42:09.311398+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:42:09.311398+0000","last_undegraded":"2024-09-13T14:42:09.311398+0000","last_fullsized":"2024-09-13T14:42:09.311398+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":101,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.272845+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:42:07.272845+0000","last_peered":"2024-09-13T14:42:07.272845+0000","last_clean":"2024-09-13T14:42:07.272845+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:42:07.272845+0000","last_undegraded":"2024-09-13T14:42:07.272845+0000","last_fullsized":"2024-09-13T14:42:07.272845+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":99,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.264779+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:42:07.264779+0000","last_peered":"2024-09-13T14:42:07.264779+0000","last_clean":"2024-09-13T14:42:07.264779+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:42:07.264779+0000","last_undegraded":"2024-09-13T14:42:07.264779+0000","last_fullsized":"2024-09-13T14:42:07.264779+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":127,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.332365+0000","last_change":"2024-09-13T14:38:13.977954+0000","last_active":"2024-09-13T14:42:09.332365+0000","last_peered":"2024-09-13T14:42:09.332365+0000","last_clean":"2024-09-13T14:42:09.332365+0000","last_became_active":"2024-09-13T14:38:13.977619+0000","last_became_peered":"2024-09-13T14:38:13.977619+0000","last_unstale":"2024-09-13T14:42:09.332365+0000","last_undegraded":"2024-09-13T14:42:09.332365+0000","last_fullsized":"2024-09-13T14:42:09.332365+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:58:30.259967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":128,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.338795+0000","last_change":"2024-09-13T14:38:13.978232+0000","last_active":"2024-09-13T14:42:09.338795+0000","last_peered":"2024-09-13T14:42:09.338795+0000","last_clean":"2024-09-13T14:42:09.338795+0000","last_became_active":"2024-09-13T14:38:13.977921+0000","last_became_peered":"2024-09-13T14:38:13.977921+0000","last_unstale":"2024-09-13T14:42:09.338795+0000","last_undegraded":"2024-09-13T14:42:09.338795+0000","last_fullsized":"2024-09-13T14:42:09.338795+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:10.127626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":92,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.273366+0000","last_change":"2024-09-13T14:39:42.765301+0000","last_active":"2024-09-13T14:42:07.273366+0000","last_peered":"2024-09-13T14:42:07.273366+0000","last_clean":"2024-09-13T14:42:07.273366+0000","last_became_active":"2024-09-13T14:39:42.764977+0000","last_became_peered":"2024-09-13T14:39:42.764977+0000","last_unstale":"2024-09-13T14:42:07.273366+0000","last_undegraded":"2024-09-13T14:42:07.273366+0000","last_fullsized":"2024-09-13T14:42:07.273366+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:25.223972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":114,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.329904+0000","last_change":"2024-09-13T14:37:40.925691+0000","last_active":"2024-09-13T14:42:09.329904+0000","last_peered":"2024-09-13T14:42:09.329904+0000","last_clean":"2024-09-13T14:42:09.329904+0000","last_became_active":"2024-09-13T14:37:40.925360+0000","last_became_peered":"2024-09-13T14:37:40.925360+0000","last_unstale":"2024-09-13T14:42:09.329904+0000","last_undegraded":"2024-09-13T14:42:09.329904+0000","last_fullsized":"2024-09-13T14:42:09.329904+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:07.337967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":89,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.329428+0000","last_change":"2024-09-13T14:39:42.781872+0000","last_active":"2024-09-13T14:42:09.329428+0000","last_peered":"2024-09-13T14:42:09.329428+0000","last_clean":"2024-09-13T14:42:09.329428+0000","last_became_active":"2024-09-13T14:39:42.770254+0000","last_became_peered":"2024-09-13T14:39:42.770254+0000","last_unstale":"2024-09-13T14:42:09.329428+0000","last_undegraded":"2024-09-13T14:42:09.329428+0000","last_fullsized":"2024-09-13T14:42:09.329428+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:31.211081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.9","version":"135'11327","reported_seq":11363,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:16.615726+0000","last_change":"2024-09-13T14:39:42.764535+0000","last_active":"2024-09-13T14:42:16.615726+0000","last_peered":"2024-09-13T14:42:16.615726+0000","last_clean":"2024-09-13T14:42:16.615726+0000","last_became_active":"2024-09-13T14:39:42.764101+0000","last_became_peered":"2024-09-13T14:39:42.764101+0000","last_unstale":"2024-09-13T14:42:16.615726+0000","last_undegraded":"2024-09-13T14:42:16.615726+0000","last_fullsized":"2024-09-13T14:42:16.615726+0000","mapping_epoch":107,"log_start":"114'3400","ondisk_log_start":"114'3400","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":7927,"log_dups_size":0,"ondisk_log_size":7927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:22.498969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317521920,"num_objects":4845,"num_object_clones":0,"num_object_copies":14535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11327,"num_write_kb":517504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":119,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.320877+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:42:09.320877+0000","last_peered":"2024-09-13T14:42:09.320877+0000","last_clean":"2024-09-13T14:42:09.320877+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:42:09.320877+0000","last_undegraded":"2024-09-13T14:42:09.320877+0000","last_fullsized":"2024-09-13T14:42:09.320877+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":92,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.264470+0000","last_change":"2024-09-13T14:39:42.764876+0000","last_active":"2024-09-13T14:42:07.264470+0000","last_peered":"2024-09-13T14:42:07.264470+0000","last_clean":"2024-09-13T14:42:07.264470+0000","last_became_active":"2024-09-13T14:39:42.764082+0000","last_became_peered":"2024-09-13T14:39:42.764082+0000","last_unstale":"2024-09-13T14:42:07.264470+0000","last_undegraded":"2024-09-13T14:42:07.264470+0000","last_fullsized":"2024-09-13T14:42:07.264470+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:10:07.458952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"135'11378","reported_seq":11446,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.048070+0000","last_change":"2024-09-13T14:39:42.767715+0000","last_active":"2024-09-13T14:42:19.048070+0000","last_peered":"2024-09-13T14:42:19.048070+0000","last_clean":"2024-09-13T14:42:19.048070+0000","last_became_active":"2024-09-13T14:39:42.766678+0000","last_became_peered":"2024-09-13T14:39:42.766678+0000","last_unstale":"2024-09-13T14:42:19.048070+0000","last_undegraded":"2024-09-13T14:42:19.048070+0000","last_fullsized":"2024-09-13T14:42:19.048070+0000","mapping_epoch":107,"log_start":"114'3200","ondisk_log_start":"114'3200","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":8178,"log_dups_size":0,"ondisk_log_size":8178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:30:38.645181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302907392,"num_objects":4622,"num_object_clones":0,"num_object_copies":13866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11378,"num_write_kb":512000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":101,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.273191+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:42:07.273191+0000","last_peered":"2024-09-13T14:42:07.273191+0000","last_clean":"2024-09-13T14:42:07.273191+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:42:07.273191+0000","last_undegraded":"2024-09-13T14:42:07.273191+0000","last_fullsized":"2024-09-13T14:42:07.273191+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":126,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.274648+0000","last_change":"2024-09-13T14:38:13.979677+0000","last_active":"2024-09-13T14:42:07.274648+0000","last_peered":"2024-09-13T14:42:07.274648+0000","last_clean":"2024-09-13T14:42:07.274648+0000","last_became_active":"2024-09-13T14:38:13.979323+0000","last_became_peered":"2024-09-13T14:38:13.979323+0000","last_unstale":"2024-09-13T14:42:07.274648+0000","last_undegraded":"2024-09-13T14:42:07.274648+0000","last_fullsized":"2024-09-13T14:42:07.274648+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:29:03.491254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"135'11522","reported_seq":11588,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.057286+0000","last_change":"2024-09-13T14:39:42.766942+0000","last_active":"2024-09-13T14:42:19.057286+0000","last_peered":"2024-09-13T14:42:19.057286+0000","last_clean":"2024-09-13T14:42:19.057286+0000","last_became_active":"2024-09-13T14:39:42.766647+0000","last_became_peered":"2024-09-13T14:39:42.766647+0000","last_unstale":"2024-09-13T14:42:19.057286+0000","last_undegraded":"2024-09-13T14:42:19.057286+0000","last_fullsized":"2024-09-13T14:42:19.057286+0000","mapping_epoch":107,"log_start":"114'3400","ondisk_log_start":"114'3400","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":8122,"log_dups_size":0,"ondisk_log_size":8122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:16.267901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299237376,"num_objects":4566,"num_object_clones":0,"num_object_copies":13698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11522,"num_write_kb":514816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":97,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.272966+0000","last_change":"2024-09-13T14:39:42.774547+0000","last_active":"2024-09-13T14:42:07.272966+0000","last_peered":"2024-09-13T14:42:07.272966+0000","last_clean":"2024-09-13T14:42:07.272966+0000","last_became_active":"2024-09-13T14:39:42.766006+0000","last_became_peered":"2024-09-13T14:39:42.766006+0000","last_unstale":"2024-09-13T14:42:07.272966+0000","last_undegraded":"2024-09-13T14:42:07.272966+0000","last_fullsized":"2024-09-13T14:42:07.272966+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:06:29.632163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":118,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.313393+0000","last_change":"2024-09-13T14:39:42.765763+0000","last_active":"2024-09-13T14:42:09.313393+0000","last_peered":"2024-09-13T14:42:09.313393+0000","last_clean":"2024-09-13T14:42:09.313393+0000","last_became_active":"2024-09-13T14:39:42.765441+0000","last_became_peered":"2024-09-13T14:39:42.765441+0000","last_unstale":"2024-09-13T14:42:09.313393+0000","last_undegraded":"2024-09-13T14:42:09.313393+0000","last_fullsized":"2024-09-13T14:42:09.313393+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:06.483638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.a","version":"135'11910","reported_seq":11978,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:18.986882+0000","last_change":"2024-09-13T14:39:42.763571+0000","last_active":"2024-09-13T14:42:18.986882+0000","last_peered":"2024-09-13T14:42:18.986882+0000","last_clean":"2024-09-13T14:42:18.986882+0000","last_became_active":"2024-09-13T14:39:42.763270+0000","last_became_peered":"2024-09-13T14:39:42.763270+0000","last_unstale":"2024-09-13T14:42:18.986882+0000","last_undegraded":"2024-09-13T14:42:18.986882+0000","last_fullsized":"2024-09-13T14:42:18.986882+0000","mapping_epoch":107,"log_start":"115'3800","ondisk_log_start":"115'3800","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":8110,"log_dups_size":0,"ondisk_log_size":8110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:38:09.281466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313131008,"num_objects":4778,"num_object_clones":0,"num_object_copies":14334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11910,"num_write_kb":534016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.265053+0000","last_change":"2024-09-13T14:39:42.765558+0000","last_active":"2024-09-13T14:42:07.265053+0000","last_peered":"2024-09-13T14:42:07.265053+0000","last_clean":"2024-09-13T14:42:07.265053+0000","last_became_active":"2024-09-13T14:39:42.765246+0000","last_became_peered":"2024-09-13T14:39:42.765246+0000","last_unstale":"2024-09-13T14:42:07.265053+0000","last_undegraded":"2024-09-13T14:42:07.265053+0000","last_fullsized":"2024-09-13T14:42:07.265053+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:08.640610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":128,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.330040+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:42:09.330040+0000","last_peered":"2024-09-13T14:42:09.330040+0000","last_clean":"2024-09-13T14:42:09.330040+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:42:09.330040+0000","last_undegraded":"2024-09-13T14:42:09.330040+0000","last_fullsized":"2024-09-13T14:42:09.330040+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":262,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.273217+0000","last_change":"2024-09-13T14:40:32.590204+0000","last_active":"2024-09-13T14:42:07.273217+0000","last_peered":"2024-09-13T14:42:07.273217+0000","last_clean":"2024-09-13T14:42:07.273217+0000","last_became_active":"2024-09-13T14:40:22.949830+0000","last_became_peered":"2024-09-13T14:40:22.949830+0000","last_unstale":"2024-09-13T14:42:07.273217+0000","last_undegraded":"2024-09-13T14:42:07.273217+0000","last_fullsized":"2024-09-13T14:42:07.273217+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:32.472827+0000","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.d","version":"135'11633","reported_seq":11702,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.027750+0000","last_change":"2024-09-13T14:39:42.759038+0000","last_active":"2024-09-13T14:42:19.027750+0000","last_peered":"2024-09-13T14:42:19.027750+0000","last_clean":"2024-09-13T14:42:19.027750+0000","last_became_active":"2024-09-13T14:39:42.758753+0000","last_became_peered":"2024-09-13T14:39:42.758753+0000","last_unstale":"2024-09-13T14:42:19.027750+0000","last_undegraded":"2024-09-13T14:42:19.027750+0000","last_fullsized":"2024-09-13T14:42:19.027750+0000","mapping_epoch":107,"log_start":"114'3500","ondisk_log_start":"114'3500","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":8133,"log_dups_size":0,"ondisk_log_size":8133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:14:07.557396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309723160,"num_objects":4727,"num_object_clones":0,"num_object_copies":14181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4727,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11633,"num_write_kb":523457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":131,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.311172+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:42:09.311172+0000","last_peered":"2024-09-13T14:42:09.311172+0000","last_clean":"2024-09-13T14:42:09.311172+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:42:09.311172+0000","last_undegraded":"2024-09-13T14:42:09.311172+0000","last_fullsized":"2024-09-13T14:42:09.311172+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":109,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.276495+0000","last_change":"2024-09-13T14:39:42.775584+0000","last_active":"2024-09-13T14:42:07.276495+0000","last_peered":"2024-09-13T14:42:07.276495+0000","last_clean":"2024-09-13T14:42:07.276495+0000","last_became_active":"2024-09-13T14:39:42.775254+0000","last_became_peered":"2024-09-13T14:39:42.775254+0000","last_unstale":"2024-09-13T14:42:07.276495+0000","last_undegraded":"2024-09-13T14:42:07.276495+0000","last_fullsized":"2024-09-13T14:42:07.276495+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:44.607098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":43,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.264356+0000","last_change":"2024-09-13T14:40:27.868402+0000","last_active":"2024-09-13T14:42:07.264356+0000","last_peered":"2024-09-13T14:42:07.264356+0000","last_clean":"2024-09-13T14:42:07.264356+0000","last_became_active":"2024-09-13T14:40:22.966301+0000","last_became_peered":"2024-09-13T14:40:22.966301+0000","last_unstale":"2024-09-13T14:42:07.264356+0000","last_undegraded":"2024-09-13T14:42:07.264356+0000","last_fullsized":"2024-09-13T14:42:07.264356+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:18.660186+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.c","version":"135'11478","reported_seq":11546,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:17.024873+0000","last_change":"2024-09-13T14:39:42.772481+0000","last_active":"2024-09-13T14:42:17.024873+0000","last_peered":"2024-09-13T14:42:17.024873+0000","last_clean":"2024-09-13T14:42:17.024873+0000","last_became_active":"2024-09-13T14:39:42.772211+0000","last_became_peered":"2024-09-13T14:39:42.772211+0000","last_unstale":"2024-09-13T14:42:17.024873+0000","last_undegraded":"2024-09-13T14:42:17.024873+0000","last_fullsized":"2024-09-13T14:42:17.024873+0000","mapping_epoch":107,"log_start":"120'4600","ondisk_log_start":"120'4600","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6878,"log_dups_size":0,"ondisk_log_size":6878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:30:16.860807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314441728,"num_objects":4798,"num_object_clones":0,"num_object_copies":14394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11478,"num_write_kb":520832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":91,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.274831+0000","last_change":"2024-09-13T14:39:42.776751+0000","last_active":"2024-09-13T14:42:07.274831+0000","last_peered":"2024-09-13T14:42:07.274831+0000","last_clean":"2024-09-13T14:42:07.274831+0000","last_became_active":"2024-09-13T14:39:42.774855+0000","last_became_peered":"2024-09-13T14:39:42.774855+0000","last_unstale":"2024-09-13T14:42:07.274831+0000","last_undegraded":"2024-09-13T14:42:07.274831+0000","last_fullsized":"2024-09-13T14:42:07.274831+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:11.022013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":116,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.275330+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:42:07.275330+0000","last_peered":"2024-09-13T14:42:07.275330+0000","last_clean":"2024-09-13T14:42:07.275330+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:42:07.275330+0000","last_undegraded":"2024-09-13T14:42:07.275330+0000","last_fullsized":"2024-09-13T14:42:07.275330+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"135'11428","reported_seq":11464,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:16.684359+0000","last_change":"2024-09-13T14:39:42.764380+0000","last_active":"2024-09-13T14:42:16.684359+0000","last_peered":"2024-09-13T14:42:16.684359+0000","last_clean":"2024-09-13T14:42:16.684359+0000","last_became_active":"2024-09-13T14:39:42.764127+0000","last_became_peered":"2024-09-13T14:39:42.764127+0000","last_unstale":"2024-09-13T14:42:16.684359+0000","last_undegraded":"2024-09-13T14:42:16.684359+0000","last_fullsized":"2024-09-13T14:42:16.684359+0000","mapping_epoch":107,"log_start":"119'4100","ondisk_log_start":"119'4100","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":7328,"log_dups_size":0,"ondisk_log_size":7328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:37:32.713989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324403200,"num_objects":4950,"num_object_clones":0,"num_object_copies":14850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11428,"num_write_kb":524096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":114,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.329361+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:42:09.329361+0000","last_peered":"2024-09-13T14:42:09.329361+0000","last_clean":"2024-09-13T14:42:09.329361+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:42:09.329361+0000","last_undegraded":"2024-09-13T14:42:09.329361+0000","last_fullsized":"2024-09-13T14:42:09.329361+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":109,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.275204+0000","last_change":"2024-09-13T14:39:42.776187+0000","last_active":"2024-09-13T14:42:07.275204+0000","last_peered":"2024-09-13T14:42:07.275204+0000","last_clean":"2024-09-13T14:42:07.275204+0000","last_became_active":"2024-09-13T14:39:42.775751+0000","last_became_peered":"2024-09-13T14:39:42.775751+0000","last_unstale":"2024-09-13T14:42:07.275204+0000","last_undegraded":"2024-09-13T14:42:07.275204+0000","last_fullsized":"2024-09-13T14:42:07.275204+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:35.155717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":36,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.264553+0000","last_change":"2024-09-13T14:40:25.861254+0000","last_active":"2024-09-13T14:42:07.264553+0000","last_peered":"2024-09-13T14:42:07.264553+0000","last_clean":"2024-09-13T14:42:07.264553+0000","last_became_active":"2024-09-13T14:40:22.957614+0000","last_became_peered":"2024-09-13T14:40:22.957614+0000","last_unstale":"2024-09-13T14:42:07.264553+0000","last_undegraded":"2024-09-13T14:42:07.264553+0000","last_fullsized":"2024-09-13T14:42:07.264553+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:53.388190+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.e","version":"135'11713","reported_seq":11749,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:16.590739+0000","last_change":"2024-09-13T14:39:42.772007+0000","last_active":"2024-09-13T14:42:16.590739+0000","last_peered":"2024-09-13T14:42:16.590739+0000","last_clean":"2024-09-13T14:42:16.590739+0000","last_became_active":"2024-09-13T14:39:42.771672+0000","last_became_peered":"2024-09-13T14:39:42.771672+0000","last_unstale":"2024-09-13T14:42:16.590739+0000","last_undegraded":"2024-09-13T14:42:16.590739+0000","last_fullsized":"2024-09-13T14:42:16.590739+0000","mapping_epoch":107,"log_start":"120'4800","ondisk_log_start":"120'4800","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6913,"log_dups_size":0,"ondisk_log_size":6913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:26:12.134635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320798720,"num_objects":4895,"num_object_clones":0,"num_object_copies":14685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11713,"num_write_kb":531456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":135,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.311453+0000","last_change":"2024-09-13T14:38:13.981576+0000","last_active":"2024-09-13T14:42:09.311453+0000","last_peered":"2024-09-13T14:42:09.311453+0000","last_clean":"2024-09-13T14:42:09.311453+0000","last_became_active":"2024-09-13T14:38:13.981262+0000","last_became_peered":"2024-09-13T14:38:13.981262+0000","last_unstale":"2024-09-13T14:42:09.311453+0000","last_undegraded":"2024-09-13T14:42:09.311453+0000","last_fullsized":"2024-09-13T14:42:09.311453+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:20.284250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":116,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312986+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:42:09.312986+0000","last_peered":"2024-09-13T14:42:09.312986+0000","last_clean":"2024-09-13T14:42:09.312986+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:42:09.312986+0000","last_undegraded":"2024-09-13T14:42:09.312986+0000","last_fullsized":"2024-09-13T14:42:09.312986+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.4","version":"135'11384","reported_seq":11420,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:16.658866+0000","last_change":"2024-09-13T14:39:42.765514+0000","last_active":"2024-09-13T14:42:16.658866+0000","last_peered":"2024-09-13T14:42:16.658866+0000","last_clean":"2024-09-13T14:42:16.658866+0000","last_became_active":"2024-09-13T14:39:42.765209+0000","last_became_peered":"2024-09-13T14:39:42.765209+0000","last_unstale":"2024-09-13T14:42:16.658866+0000","last_undegraded":"2024-09-13T14:42:16.658866+0000","last_fullsized":"2024-09-13T14:42:16.658866+0000","mapping_epoch":107,"log_start":"114'3400","ondisk_log_start":"114'3400","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":7984,"log_dups_size":0,"ondisk_log_size":7984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:59.785672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316276736,"num_objects":4826,"num_object_clones":0,"num_object_copies":14478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11384,"num_write_kb":518720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":253,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.272692+0000","last_change":"2024-09-13T14:40:22.639864+0000","last_active":"2024-09-13T14:42:07.272692+0000","last_peered":"2024-09-13T14:42:07.272692+0000","last_clean":"2024-09-13T14:42:07.272692+0000","last_became_active":"2024-09-13T14:40:20.926503+0000","last_became_peered":"2024-09-13T14:40:20.926503+0000","last_unstale":"2024-09-13T14:42:07.272692+0000","last_undegraded":"2024-09-13T14:42:07.272692+0000","last_fullsized":"2024-09-13T14:42:07.272692+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:22.639760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:22.639760+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:08.523563+0000","scrub_duration":0.0051120610000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":662,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.265064+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:42:07.265064+0000","last_peered":"2024-09-13T14:42:07.265064+0000","last_clean":"2024-09-13T14:42:07.265064+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:42:07.265064+0000","last_undegraded":"2024-09-13T14:42:07.265064+0000","last_fullsized":"2024-09-13T14:42:07.265064+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":92,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.311167+0000","last_change":"2024-09-13T14:39:42.768341+0000","last_active":"2024-09-13T14:42:09.311167+0000","last_peered":"2024-09-13T14:42:09.311167+0000","last_clean":"2024-09-13T14:42:09.311167+0000","last_became_active":"2024-09-13T14:39:42.768034+0000","last_became_peered":"2024-09-13T14:39:42.768034+0000","last_unstale":"2024-09-13T14:42:09.311167+0000","last_undegraded":"2024-09-13T14:42:09.311167+0000","last_fullsized":"2024-09-13T14:42:09.311167+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:28:55.502108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":41,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.275206+0000","last_change":"2024-09-13T14:39:42.772096+0000","last_active":"2024-09-13T14:42:07.275206+0000","last_peered":"2024-09-13T14:42:07.275206+0000","last_clean":"2024-09-13T14:42:07.275206+0000","last_became_active":"2024-09-13T14:39:42.771762+0000","last_became_peered":"2024-09-13T14:39:42.771762+0000","last_unstale":"2024-09-13T14:42:07.275206+0000","last_undegraded":"2024-09-13T14:42:07.275206+0000","last_fullsized":"2024-09-13T14:42:07.275206+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:16.172571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":244,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.310581+0000","last_change":"2024-09-13T14:40:23.784403+0000","last_active":"2024-09-13T14:42:09.310581+0000","last_peered":"2024-09-13T14:42:09.310581+0000","last_clean":"2024-09-13T14:42:09.310581+0000","last_became_active":"2024-09-13T14:40:22.951246+0000","last_became_peered":"2024-09-13T14:40:22.951246+0000","last_unstale":"2024-09-13T14:42:09.310581+0000","last_undegraded":"2024-09-13T14:42:09.310581+0000","last_fullsized":"2024-09-13T14:42:09.310581+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.784263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.784263+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:48:09.437261+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.7","version":"135'11363","reported_seq":11399,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:16.649797+0000","last_change":"2024-09-13T14:39:42.773740+0000","last_active":"2024-09-13T14:42:16.649797+0000","last_peered":"2024-09-13T14:42:16.649797+0000","last_clean":"2024-09-13T14:42:16.649797+0000","last_became_active":"2024-09-13T14:39:42.773444+0000","last_became_peered":"2024-09-13T14:39:42.773444+0000","last_unstale":"2024-09-13T14:42:16.649797+0000","last_undegraded":"2024-09-13T14:42:16.649797+0000","last_fullsized":"2024-09-13T14:42:16.649797+0000","mapping_epoch":107,"log_start":"118'4000","ondisk_log_start":"118'4000","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":7363,"log_dups_size":0,"ondisk_log_size":7363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:09:16.690630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309788672,"num_objects":4727,"num_object_clones":0,"num_object_copies":14181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11363,"num_write_kb":514880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":346,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.272610+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:42:07.272610+0000","last_peered":"2024-09-13T14:42:07.272610+0000","last_clean":"2024-09-13T14:42:07.272610+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:42:07.272610+0000","last_undegraded":"2024-09-13T14:42:07.272610+0000","last_fullsized":"2024-09-13T14:42:07.272610+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":136,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.329448+0000","last_change":"2024-09-13T14:38:13.977962+0000","last_active":"2024-09-13T14:42:09.329448+0000","last_peered":"2024-09-13T14:42:09.329448+0000","last_clean":"2024-09-13T14:42:09.329448+0000","last_became_active":"2024-09-13T14:38:13.977638+0000","last_became_peered":"2024-09-13T14:38:13.977638+0000","last_unstale":"2024-09-13T14:42:09.329448+0000","last_undegraded":"2024-09-13T14:42:09.329448+0000","last_fullsized":"2024-09-13T14:42:09.329448+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:53:16.902157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":140,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.339140+0000","last_change":"2024-09-13T14:37:40.927444+0000","last_active":"2024-09-13T14:42:09.339140+0000","last_peered":"2024-09-13T14:42:09.339140+0000","last_clean":"2024-09-13T14:42:09.339140+0000","last_became_active":"2024-09-13T14:37:40.927140+0000","last_became_peered":"2024-09-13T14:37:40.927140+0000","last_unstale":"2024-09-13T14:42:09.339140+0000","last_undegraded":"2024-09-13T14:42:09.339140+0000","last_fullsized":"2024-09-13T14:42:09.339140+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:32:54.207568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":262,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.272725+0000","last_change":"2024-09-13T14:40:28.602999+0000","last_active":"2024-09-13T14:42:07.272725+0000","last_peered":"2024-09-13T14:42:07.272725+0000","last_clean":"2024-09-13T14:42:07.272725+0000","last_became_active":"2024-09-13T14:40:22.949002+0000","last_became_peered":"2024-09-13T14:40:22.949002+0000","last_unstale":"2024-09-13T14:42:07.272725+0000","last_undegraded":"2024-09-13T14:42:07.272725+0000","last_fullsized":"2024-09-13T14:42:07.272725+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:28.602942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:28.602942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:29:46.037596+0000","scrub_duration":0.00945176,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.6","version":"135'11502","reported_seq":11570,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.047750+0000","last_change":"2024-09-13T14:39:42.759808+0000","last_active":"2024-09-13T14:42:19.047750+0000","last_peered":"2024-09-13T14:42:19.047750+0000","last_clean":"2024-09-13T14:42:19.047750+0000","last_became_active":"2024-09-13T14:39:42.759199+0000","last_became_peered":"2024-09-13T14:39:42.759199+0000","last_unstale":"2024-09-13T14:42:19.047750+0000","last_undegraded":"2024-09-13T14:42:19.047750+0000","last_fullsized":"2024-09-13T14:42:19.047750+0000","mapping_epoch":107,"log_start":"114'3300","ondisk_log_start":"114'3300","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":8202,"log_dups_size":0,"ondisk_log_size":8202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:55:55.573511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302383104,"num_objects":4614,"num_object_clones":0,"num_object_copies":13842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11502,"num_write_kb":515712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":540,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.318544+0000","last_change":"2024-09-13T14:38:13.978679+0000","last_active":"2024-09-13T14:42:09.318544+0000","last_peered":"2024-09-13T14:42:09.318544+0000","last_clean":"2024-09-13T14:42:09.318544+0000","last_became_active":"2024-09-13T14:38:13.978171+0000","last_became_peered":"2024-09-13T14:38:13.978171+0000","last_unstale":"2024-09-13T14:42:09.318544+0000","last_undegraded":"2024-09-13T14:42:09.318544+0000","last_fullsized":"2024-09-13T14:42:09.318544+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:57.691198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":123,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.313118+0000","last_change":"2024-09-13T14:39:42.769691+0000","last_active":"2024-09-13T14:42:09.313118+0000","last_peered":"2024-09-13T14:42:09.313118+0000","last_clean":"2024-09-13T14:42:09.313118+0000","last_became_active":"2024-09-13T14:39:42.769266+0000","last_became_peered":"2024-09-13T14:39:42.769266+0000","last_unstale":"2024-09-13T14:42:09.313118+0000","last_undegraded":"2024-09-13T14:42:09.313118+0000","last_fullsized":"2024-09-13T14:42:09.313118+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:11.637732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":149,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312835+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:42:09.312835+0000","last_peered":"2024-09-13T14:42:09.312835+0000","last_clean":"2024-09-13T14:42:09.312835+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:42:09.312835+0000","last_undegraded":"2024-09-13T14:42:09.312835+0000","last_fullsized":"2024-09-13T14:42:09.312835+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":36,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.264457+0000","last_change":"2024-09-13T14:40:26.896691+0000","last_active":"2024-09-13T14:42:07.264457+0000","last_peered":"2024-09-13T14:42:07.264457+0000","last_clean":"2024-09-13T14:42:07.264457+0000","last_became_active":"2024-09-13T14:40:22.949772+0000","last_became_peered":"2024-09-13T14:40:22.949772+0000","last_unstale":"2024-09-13T14:42:07.264457+0000","last_undegraded":"2024-09-13T14:42:07.264457+0000","last_fullsized":"2024-09-13T14:42:07.264457+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:26.896613+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:26.896613+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:01.615433+0000","scrub_duration":0.004120828,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.5","version":"135'11412","reported_seq":11480,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:18.300747+0000","last_change":"2024-09-13T14:39:42.776210+0000","last_active":"2024-09-13T14:42:18.300747+0000","last_peered":"2024-09-13T14:42:18.300747+0000","last_clean":"2024-09-13T14:42:18.300747+0000","last_became_active":"2024-09-13T14:39:42.775823+0000","last_became_peered":"2024-09-13T14:39:42.775823+0000","last_unstale":"2024-09-13T14:42:18.300747+0000","last_undegraded":"2024-09-13T14:42:18.300747+0000","last_fullsized":"2024-09-13T14:42:18.300747+0000","mapping_epoch":107,"log_start":"120'5200","ondisk_log_start":"120'5200","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6212,"log_dups_size":0,"ondisk_log_size":6212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:39.020681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310116352,"num_objects":4732,"num_object_clones":0,"num_object_copies":14196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11412,"num_write_kb":516608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":726,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.264757+0000","last_change":"2024-09-13T14:39:42.805055+0000","last_active":"2024-09-13T14:42:07.264757+0000","last_peered":"2024-09-13T14:42:07.264757+0000","last_clean":"2024-09-13T14:42:07.264757+0000","last_became_active":"2024-09-13T14:39:42.762456+0000","last_became_peered":"2024-09-13T14:39:42.762456+0000","last_unstale":"2024-09-13T14:42:07.264757+0000","last_undegraded":"2024-09-13T14:42:07.264757+0000","last_fullsized":"2024-09-13T14:42:07.264757+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:54:40.839991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":110,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.275041+0000","last_change":"2024-09-13T14:39:42.775159+0000","last_active":"2024-09-13T14:42:07.275041+0000","last_peered":"2024-09-13T14:42:07.275041+0000","last_clean":"2024-09-13T14:42:07.275041+0000","last_became_active":"2024-09-13T14:39:42.774841+0000","last_became_peered":"2024-09-13T14:39:42.774841+0000","last_unstale":"2024-09-13T14:42:07.275041+0000","last_undegraded":"2024-09-13T14:42:07.275041+0000","last_fullsized":"2024-09-13T14:42:07.275041+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:35:45.579397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":127,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.311575+0000","last_change":"2024-09-13T14:38:13.981294+0000","last_active":"2024-09-13T14:42:09.311575+0000","last_peered":"2024-09-13T14:42:09.311575+0000","last_clean":"2024-09-13T14:42:09.311575+0000","last_became_active":"2024-09-13T14:38:13.980975+0000","last_became_peered":"2024-09-13T14:38:13.980975+0000","last_unstale":"2024-09-13T14:42:09.311575+0000","last_undegraded":"2024-09-13T14:42:09.311575+0000","last_fullsized":"2024-09-13T14:42:09.311575+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:17.545179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":53,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312569+0000","last_change":"2024-09-13T14:40:23.093425+0000","last_active":"2024-09-13T14:42:09.312569+0000","last_peered":"2024-09-13T14:42:09.312569+0000","last_clean":"2024-09-13T14:42:09.312569+0000","last_became_active":"2024-09-13T14:40:22.947358+0000","last_became_peered":"2024-09-13T14:40:22.947358+0000","last_unstale":"2024-09-13T14:42:09.312569+0000","last_undegraded":"2024-09-13T14:42:09.312569+0000","last_fullsized":"2024-09-13T14:42:09.312569+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.093356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.093356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:38:28.248105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.3","version":"135'11405","reported_seq":11473,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:18.307746+0000","last_change":"2024-09-13T14:39:42.770363+0000","last_active":"2024-09-13T14:42:18.307746+0000","last_peered":"2024-09-13T14:42:18.307746+0000","last_clean":"2024-09-13T14:42:18.307746+0000","last_became_active":"2024-09-13T14:39:42.770092+0000","last_became_peered":"2024-09-13T14:39:42.770092+0000","last_unstale":"2024-09-13T14:42:18.307746+0000","last_undegraded":"2024-09-13T14:42:18.307746+0000","last_fullsized":"2024-09-13T14:42:18.307746+0000","mapping_epoch":107,"log_start":"120'5200","ondisk_log_start":"120'5200","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6205,"log_dups_size":0,"ondisk_log_size":6205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:20:20.375252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304545792,"num_objects":4647,"num_object_clones":0,"num_object_copies":13941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11405,"num_write_kb":513664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":117,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.264858+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:42:07.264858+0000","last_peered":"2024-09-13T14:42:07.264858+0000","last_clean":"2024-09-13T14:42:07.264858+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:42:07.264858+0000","last_undegraded":"2024-09-13T14:42:07.264858+0000","last_fullsized":"2024-09-13T14:42:07.264858+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":633,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.311747+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:42:09.311747+0000","last_peered":"2024-09-13T14:42:09.311747+0000","last_clean":"2024-09-13T14:42:09.311747+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:42:09.311747+0000","last_undegraded":"2024-09-13T14:42:09.311747+0000","last_fullsized":"2024-09-13T14:42:09.311747+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":105,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.313698+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:42:09.313698+0000","last_peered":"2024-09-13T14:42:09.313698+0000","last_clean":"2024-09-13T14:42:09.313698+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:42:09.313698+0000","last_undegraded":"2024-09-13T14:42:09.313698+0000","last_fullsized":"2024-09-13T14:42:09.313698+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":199,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.330274+0000","last_change":"2024-09-13T14:40:23.334441+0000","last_active":"2024-09-13T14:42:09.330274+0000","last_peered":"2024-09-13T14:42:09.330274+0000","last_clean":"2024-09-13T14:42:09.330274+0000","last_became_active":"2024-09-13T14:40:22.952908+0000","last_became_peered":"2024-09-13T14:40:22.952908+0000","last_unstale":"2024-09-13T14:42:09.330274+0000","last_undegraded":"2024-09-13T14:42:09.330274+0000","last_fullsized":"2024-09-13T14:42:09.330274+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.334379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.334379+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:58:19.770506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"135'11606","reported_seq":11674,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:18.289737+0000","last_change":"2024-09-13T14:39:42.770399+0000","last_active":"2024-09-13T14:42:18.289737+0000","last_peered":"2024-09-13T14:42:18.289737+0000","last_clean":"2024-09-13T14:42:18.289737+0000","last_became_active":"2024-09-13T14:39:42.770074+0000","last_became_peered":"2024-09-13T14:39:42.770074+0000","last_unstale":"2024-09-13T14:42:18.289737+0000","last_undegraded":"2024-09-13T14:42:18.289737+0000","last_fullsized":"2024-09-13T14:42:18.289737+0000","mapping_epoch":107,"log_start":"120'5400","ondisk_log_start":"120'5400","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6206,"log_dups_size":0,"ondisk_log_size":6206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:28:03.030813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309985280,"num_objects":4730,"num_object_clones":0,"num_object_copies":14190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11606,"num_write_kb":522752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":116,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.311657+0000","last_change":"2024-09-13T14:37:40.921861+0000","last_active":"2024-09-13T14:42:09.311657+0000","last_peered":"2024-09-13T14:42:09.311657+0000","last_clean":"2024-09-13T14:42:09.311657+0000","last_became_active":"2024-09-13T14:37:40.921587+0000","last_became_peered":"2024-09-13T14:37:40.921587+0000","last_unstale":"2024-09-13T14:42:09.311657+0000","last_undegraded":"2024-09-13T14:42:09.311657+0000","last_fullsized":"2024-09-13T14:42:09.311657+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:24.619352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":348,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.273186+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:42:07.273186+0000","last_peered":"2024-09-13T14:42:07.273186+0000","last_clean":"2024-09-13T14:42:07.273186+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:42:07.273186+0000","last_undegraded":"2024-09-13T14:42:07.273186+0000","last_fullsized":"2024-09-13T14:42:07.273186+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":118,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.329964+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:42:09.329964+0000","last_peered":"2024-09-13T14:42:09.329964+0000","last_clean":"2024-09-13T14:42:09.329964+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:42:09.329964+0000","last_undegraded":"2024-09-13T14:42:09.329964+0000","last_fullsized":"2024-09-13T14:42:09.329964+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":36,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.274914+0000","last_change":"2024-09-13T14:40:30.500108+0000","last_active":"2024-09-13T14:42:07.274914+0000","last_peered":"2024-09-13T14:42:07.274914+0000","last_clean":"2024-09-13T14:42:07.274914+0000","last_became_active":"2024-09-13T14:40:22.949882+0000","last_became_peered":"2024-09-13T14:40:22.949882+0000","last_unstale":"2024-09-13T14:42:07.274914+0000","last_undegraded":"2024-09-13T14:42:07.274914+0000","last_fullsized":"2024-09-13T14:42:07.274914+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:40:30.500046+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:22:58.480202+0000","scrub_duration":0.00248832,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1","version":"135'11505","reported_seq":11541,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:16.657460+0000","last_change":"2024-09-13T14:39:42.772158+0000","last_active":"2024-09-13T14:42:16.657460+0000","last_peered":"2024-09-13T14:42:16.657460+0000","last_clean":"2024-09-13T14:42:16.657460+0000","last_became_active":"2024-09-13T14:39:42.771189+0000","last_became_peered":"2024-09-13T14:39:42.771189+0000","last_unstale":"2024-09-13T14:42:16.657460+0000","last_undegraded":"2024-09-13T14:42:16.657460+0000","last_fullsized":"2024-09-13T14:42:16.657460+0000","mapping_epoch":107,"log_start":"120'4600","ondisk_log_start":"120'4600","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6905,"log_dups_size":0,"ondisk_log_size":6905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:58:08.882288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329056256,"num_objects":5021,"num_object_clones":0,"num_object_copies":15063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11505,"num_write_kb":528832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":131,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312730+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:42:09.312730+0000","last_peered":"2024-09-13T14:42:09.312730+0000","last_clean":"2024-09-13T14:42:09.312730+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:42:09.312730+0000","last_undegraded":"2024-09-13T14:42:09.312730+0000","last_fullsized":"2024-09-13T14:42:09.312730+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":454,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.310239+0000","last_change":"2024-09-13T14:39:44.990216+0000","last_active":"2024-09-13T14:42:09.310239+0000","last_peered":"2024-09-13T14:42:09.310239+0000","last_clean":"2024-09-13T14:42:09.310239+0000","last_became_active":"2024-09-13T14:39:44.989433+0000","last_became_peered":"2024-09-13T14:39:44.989433+0000","last_unstale":"2024-09-13T14:42:09.310239+0000","last_undegraded":"2024-09-13T14:42:09.310239+0000","last_fullsized":"2024-09-13T14:42:09.310239+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:00.426042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":129,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312417+0000","last_change":"2024-09-13T14:38:13.979255+0000","last_active":"2024-09-13T14:42:09.312417+0000","last_peered":"2024-09-13T14:42:09.312417+0000","last_clean":"2024-09-13T14:42:09.312417+0000","last_became_active":"2024-09-13T14:38:13.978868+0000","last_became_peered":"2024-09-13T14:38:13.978868+0000","last_unstale":"2024-09-13T14:42:09.312417+0000","last_undegraded":"2024-09-13T14:42:09.312417+0000","last_fullsized":"2024-09-13T14:42:09.312417+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:15:26.003538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":58,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.311182+0000","last_change":"2024-09-13T14:40:24.124750+0000","last_active":"2024-09-13T14:42:09.311182+0000","last_peered":"2024-09-13T14:42:09.311182+0000","last_clean":"2024-09-13T14:42:09.311182+0000","last_became_active":"2024-09-13T14:40:22.958820+0000","last_became_peered":"2024-09-13T14:40:22.958820+0000","last_unstale":"2024-09-13T14:42:09.311182+0000","last_undegraded":"2024-09-13T14:42:09.311182+0000","last_fullsized":"2024-09-13T14:42:09.311182+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:24.124531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:24.124531+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:36:35.757492+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"135'11445","reported_seq":11512,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:16.692467+0000","last_change":"2024-09-13T14:39:42.770912+0000","last_active":"2024-09-13T14:42:16.692467+0000","last_peered":"2024-09-13T14:42:16.692467+0000","last_clean":"2024-09-13T14:42:16.692467+0000","last_became_active":"2024-09-13T14:39:42.770646+0000","last_became_peered":"2024-09-13T14:39:42.770646+0000","last_unstale":"2024-09-13T14:42:16.692467+0000","last_undegraded":"2024-09-13T14:42:16.692467+0000","last_fullsized":"2024-09-13T14:42:16.692467+0000","mapping_epoch":107,"log_start":"120'4600","ondisk_log_start":"120'4600","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6845,"log_dups_size":0,"ondisk_log_size":6845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:54:13.100820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321060864,"num_objects":4899,"num_object_clones":0,"num_object_copies":14697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11445,"num_write_kb":523008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":140,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.338585+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:42:09.338585+0000","last_peered":"2024-09-13T14:42:09.338585+0000","last_clean":"2024-09-13T14:42:09.338585+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:42:09.338585+0000","last_undegraded":"2024-09-13T14:42:09.338585+0000","last_fullsized":"2024-09-13T14:42:09.338585+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1052,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.273188+0000","last_change":"2024-09-13T14:39:44.990389+0000","last_active":"2024-09-13T14:42:07.273188+0000","last_peered":"2024-09-13T14:42:07.273188+0000","last_clean":"2024-09-13T14:42:07.273188+0000","last_became_active":"2024-09-13T14:39:44.989805+0000","last_became_peered":"2024-09-13T14:39:44.989805+0000","last_unstale":"2024-09-13T14:42:07.273188+0000","last_undegraded":"2024-09-13T14:42:07.273188+0000","last_fullsized":"2024-09-13T14:42:07.273188+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:15.392445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":92,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.274607+0000","last_change":"2024-09-13T14:39:42.769954+0000","last_active":"2024-09-13T14:42:07.274607+0000","last_peered":"2024-09-13T14:42:07.274607+0000","last_clean":"2024-09-13T14:42:07.274607+0000","last_became_active":"2024-09-13T14:39:42.769635+0000","last_became_peered":"2024-09-13T14:39:42.769635+0000","last_unstale":"2024-09-13T14:42:07.274607+0000","last_undegraded":"2024-09-13T14:42:07.274607+0000","last_fullsized":"2024-09-13T14:42:07.274607+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:51.588071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":109,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.275006+0000","last_change":"2024-09-13T14:39:42.775357+0000","last_active":"2024-09-13T14:42:07.275006+0000","last_peered":"2024-09-13T14:42:07.275006+0000","last_clean":"2024-09-13T14:42:07.275006+0000","last_became_active":"2024-09-13T14:39:42.775041+0000","last_became_peered":"2024-09-13T14:39:42.775041+0000","last_unstale":"2024-09-13T14:42:07.275006+0000","last_undegraded":"2024-09-13T14:42:07.275006+0000","last_fullsized":"2024-09-13T14:42:07.275006+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:44.768910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":130,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.329244+0000","last_change":"2024-09-13T14:39:42.787151+0000","last_active":"2024-09-13T14:42:09.329244+0000","last_peered":"2024-09-13T14:42:09.329244+0000","last_clean":"2024-09-13T14:42:09.329244+0000","last_became_active":"2024-09-13T14:39:42.771712+0000","last_became_peered":"2024-09-13T14:39:42.771712+0000","last_unstale":"2024-09-13T14:42:09.329244+0000","last_undegraded":"2024-09-13T14:42:09.329244+0000","last_fullsized":"2024-09-13T14:42:09.329244+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:54.216663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":123,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.313544+0000","last_change":"2024-09-13T14:39:42.772974+0000","last_active":"2024-09-13T14:42:09.313544+0000","last_peered":"2024-09-13T14:42:09.313544+0000","last_clean":"2024-09-13T14:42:09.313544+0000","last_became_active":"2024-09-13T14:39:42.771449+0000","last_became_peered":"2024-09-13T14:39:42.771449+0000","last_unstale":"2024-09-13T14:42:09.313544+0000","last_undegraded":"2024-09-13T14:42:09.313544+0000","last_fullsized":"2024-09-13T14:42:09.313544+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:46.441079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":115,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.272955+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:42:07.272955+0000","last_peered":"2024-09-13T14:42:07.272955+0000","last_clean":"2024-09-13T14:42:07.272955+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:42:07.272955+0000","last_undegraded":"2024-09-13T14:42:07.272955+0000","last_fullsized":"2024-09-13T14:42:07.272955+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":100,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.264897+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:42:07.264897+0000","last_peered":"2024-09-13T14:42:07.264897+0000","last_clean":"2024-09-13T14:42:07.264897+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:42:07.264897+0000","last_undegraded":"2024-09-13T14:42:07.264897+0000","last_fullsized":"2024-09-13T14:42:07.264897+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":92,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312498+0000","last_change":"2024-09-13T14:39:42.767764+0000","last_active":"2024-09-13T14:42:09.312498+0000","last_peered":"2024-09-13T14:42:09.312498+0000","last_clean":"2024-09-13T14:42:09.312498+0000","last_became_active":"2024-09-13T14:39:42.767450+0000","last_became_peered":"2024-09-13T14:39:42.767450+0000","last_unstale":"2024-09-13T14:42:09.312498+0000","last_undegraded":"2024-09-13T14:42:09.312498+0000","last_fullsized":"2024-09-13T14:42:09.312498+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:47.729382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":127,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.330128+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:42:09.330128+0000","last_peered":"2024-09-13T14:42:09.330128+0000","last_clean":"2024-09-13T14:42:09.330128+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:42:09.330128+0000","last_undegraded":"2024-09-13T14:42:09.330128+0000","last_fullsized":"2024-09-13T14:42:09.330128+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":141,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.311766+0000","last_change":"2024-09-13T14:39:42.772266+0000","last_active":"2024-09-13T14:42:09.311766+0000","last_peered":"2024-09-13T14:42:09.311766+0000","last_clean":"2024-09-13T14:42:09.311766+0000","last_became_active":"2024-09-13T14:39:42.771880+0000","last_became_peered":"2024-09-13T14:39:42.771880+0000","last_unstale":"2024-09-13T14:42:09.311766+0000","last_undegraded":"2024-09-13T14:42:09.311766+0000","last_fullsized":"2024-09-13T14:42:09.311766+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:41:01.160105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":90,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.264970+0000","last_change":"2024-09-13T14:39:42.773890+0000","last_active":"2024-09-13T14:42:07.264970+0000","last_peered":"2024-09-13T14:42:07.264970+0000","last_clean":"2024-09-13T14:42:07.264970+0000","last_became_active":"2024-09-13T14:39:42.773523+0000","last_became_peered":"2024-09-13T14:39:42.773523+0000","last_unstale":"2024-09-13T14:42:07.264970+0000","last_undegraded":"2024-09-13T14:42:07.264970+0000","last_fullsized":"2024-09-13T14:42:07.264970+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:13.156261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":152,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.311667+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:42:09.311667+0000","last_peered":"2024-09-13T14:42:09.311667+0000","last_clean":"2024-09-13T14:42:09.311667+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:42:09.311667+0000","last_undegraded":"2024-09-13T14:42:09.311667+0000","last_fullsized":"2024-09-13T14:42:09.311667+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":129,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.339032+0000","last_change":"2024-09-13T14:38:13.979604+0000","last_active":"2024-09-13T14:42:09.339032+0000","last_peered":"2024-09-13T14:42:09.339032+0000","last_clean":"2024-09-13T14:42:09.339032+0000","last_became_active":"2024-09-13T14:38:13.979288+0000","last_became_peered":"2024-09-13T14:38:13.979288+0000","last_unstale":"2024-09-13T14:42:09.339032+0000","last_undegraded":"2024-09-13T14:42:09.339032+0000","last_fullsized":"2024-09-13T14:42:09.339032+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:21.031527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":116,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.310590+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:42:09.310590+0000","last_peered":"2024-09-13T14:42:09.310590+0000","last_clean":"2024-09-13T14:42:09.310590+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:42:09.310590+0000","last_undegraded":"2024-09-13T14:42:09.310590+0000","last_fullsized":"2024-09-13T14:42:09.310590+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":148,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.311754+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:42:09.311754+0000","last_peered":"2024-09-13T14:42:09.311754+0000","last_clean":"2024-09-13T14:42:09.311754+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:42:09.311754+0000","last_undegraded":"2024-09-13T14:42:09.311754+0000","last_fullsized":"2024-09-13T14:42:09.311754+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":118,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.275001+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:42:07.275001+0000","last_peered":"2024-09-13T14:42:07.275001+0000","last_clean":"2024-09-13T14:42:07.275001+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:42:07.275001+0000","last_undegraded":"2024-09-13T14:42:07.275001+0000","last_fullsized":"2024-09-13T14:42:07.275001+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":154,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.319470+0000","last_change":"2024-09-13T14:38:13.976950+0000","last_active":"2024-09-13T14:42:09.319470+0000","last_peered":"2024-09-13T14:42:09.319470+0000","last_clean":"2024-09-13T14:42:09.319470+0000","last_became_active":"2024-09-13T14:38:13.976639+0000","last_became_peered":"2024-09-13T14:38:13.976639+0000","last_unstale":"2024-09-13T14:42:09.319470+0000","last_undegraded":"2024-09-13T14:42:09.319470+0000","last_fullsized":"2024-09-13T14:42:09.319470+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:24.126890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":120,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.329874+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:42:09.329874+0000","last_peered":"2024-09-13T14:42:09.329874+0000","last_clean":"2024-09-13T14:42:09.329874+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:42:09.329874+0000","last_undegraded":"2024-09-13T14:42:09.329874+0000","last_fullsized":"2024-09-13T14:42:09.329874+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":116,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.274955+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:42:07.274955+0000","last_peered":"2024-09-13T14:42:07.274955+0000","last_clean":"2024-09-13T14:42:07.274955+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:42:07.274955+0000","last_undegraded":"2024-09-13T14:42:07.274955+0000","last_fullsized":"2024-09-13T14:42:07.274955+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":102,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.264978+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:42:07.264978+0000","last_peered":"2024-09-13T14:42:07.264978+0000","last_clean":"2024-09-13T14:42:07.264978+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:42:07.264978+0000","last_undegraded":"2024-09-13T14:42:07.264978+0000","last_fullsized":"2024-09-13T14:42:07.264978+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":116,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.274902+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:42:07.274902+0000","last_peered":"2024-09-13T14:42:07.274902+0000","last_clean":"2024-09-13T14:42:07.274902+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:42:07.274902+0000","last_undegraded":"2024-09-13T14:42:07.274902+0000","last_fullsized":"2024-09-13T14:42:07.274902+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":127,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.264813+0000","last_change":"2024-09-13T14:39:42.778413+0000","last_active":"2024-09-13T14:42:07.264813+0000","last_peered":"2024-09-13T14:42:07.264813+0000","last_clean":"2024-09-13T14:42:07.264813+0000","last_became_active":"2024-09-13T14:39:42.766624+0000","last_became_peered":"2024-09-13T14:39:42.766624+0000","last_unstale":"2024-09-13T14:42:07.264813+0000","last_undegraded":"2024-09-13T14:42:07.264813+0000","last_fullsized":"2024-09-13T14:42:07.264813+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:13.435604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":91,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.264534+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:42:07.264534+0000","last_peered":"2024-09-13T14:42:07.264534+0000","last_clean":"2024-09-13T14:42:07.264534+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:42:07.264534+0000","last_undegraded":"2024-09-13T14:42:07.264534+0000","last_fullsized":"2024-09-13T14:42:07.264534+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":148,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312973+0000","last_change":"2024-09-13T14:39:42.767451+0000","last_active":"2024-09-13T14:42:09.312973+0000","last_peered":"2024-09-13T14:42:09.312973+0000","last_clean":"2024-09-13T14:42:09.312973+0000","last_became_active":"2024-09-13T14:39:42.767086+0000","last_became_peered":"2024-09-13T14:39:42.767086+0000","last_unstale":"2024-09-13T14:42:09.312973+0000","last_undegraded":"2024-09-13T14:42:09.312973+0000","last_fullsized":"2024-09-13T14:42:09.312973+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:02.674069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":102,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-13T14:42:07.274534+0000","last_change":"2024-09-13T14:39:42.768653+0000","last_active":"2024-09-13T14:42:07.274534+0000","last_peered":"2024-09-13T14:42:07.274534+0000","last_clean":"2024-09-13T14:42:07.274534+0000","last_became_active":"2024-09-13T14:39:42.768333+0000","last_became_peered":"2024-09-13T14:39:42.768333+0000","last_unstale":"2024-09-13T14:42:07.274534+0000","last_undegraded":"2024-09-13T14:42:07.274534+0000","last_fullsized":"2024-09-13T14:42:07.274534+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:25.655793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":132,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.330041+0000","last_change":"2024-09-13T14:37:40.924860+0000","last_active":"2024-09-13T14:42:09.330041+0000","last_peered":"2024-09-13T14:42:09.330041+0000","last_clean":"2024-09-13T14:42:09.330041+0000","last_became_active":"2024-09-13T14:37:40.924522+0000","last_became_peered":"2024-09-13T14:37:40.924522+0000","last_unstale":"2024-09-13T14:42:09.330041+0000","last_undegraded":"2024-09-13T14:42:09.330041+0000","last_fullsized":"2024-09-13T14:42:09.330041+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:10.340511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":5005377560,"num_objects":76377,"num_object_clones":0,"num_object_copies":229131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76377,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":184011,"num_write_kb":8332353,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15088431104,"data_stored":15088418888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117511,"ondisk_log_size":117511,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":9648,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397700,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1865268,"kb_used_data":1559596,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91879884,"statfs":{"total":95995035648,"available":94085001216,"internally_reserved":0,"allocated":1597026304,"data_stored":1596224969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":184683593929,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790158,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3091304,"kb_used_data":2785632,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":90653848,"statfs":{"total":95995035648,"available":92829540352,"internally_reserved":0,"allocated":2852487168,"data_stored":2851694134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986387,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2814884,"kb_used_data":2513372,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":90930268,"statfs":{"total":95995035648,"available":93112594432,"internally_reserved":0,"allocated":2573692928,"data_stored":2572903153,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598835,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1341148,"kb_used_data":1269204,"kb_used_omap":6,"kb_used_meta":71929,"kb_avail":92404004,"statfs":{"total":95995035648,"available":94621700096,"internally_reserved":0,"allocated":1299664896,"data_stored":1298883565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":73656288},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762362,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2562676,"kb_used_data":2418476,"kb_used_omap":8,"kb_used_meta":144119,"kb_avail":91182476,"statfs":{"total":95995035648,"available":93370855424,"internally_reserved":0,"allocated":2476519424,"data_stored":2475718841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":147578314},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":94,"seq":403726925888,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1959260,"kb_used_data":1847060,"kb_used_omap":7,"kb_used_meta":112184,"kb_avail":91785892,"statfs":{"total":95995035648,"available":93988753408,"internally_reserved":0,"allocated":1891389440,"data_stored":1890607758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":114877129},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122128,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2561716,"kb_used_data":2421740,"kb_used_omap":7,"kb_used_meta":139960,"kb_avail":91183436,"statfs":{"total":95995035648,"available":93371838464,"internally_reserved":0,"allocated":2479861760,"data_stored":2479060928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":143319631},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2459631616,"data_stored":2459631616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1884815360,"data_stored":1884815360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2456883200,"data_stored":2456879128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1285619712,"data_stored":1285619712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2569601024,"data_stored":2569601024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2838171648,"data_stored":2838167576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1593708544,"data_stored":1593704472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:42:20.520 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:42:20.521 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:20.547 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:20.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:20.649 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:20.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.701 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:20.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:20.802 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:20.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:20 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:42:20.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:20 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:42:20.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:20 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:20.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:20 smithi096 ceph-mon[102928]: osdmap e136: 8 total, 8 up, 7 in 2024-09-13T14:42:20.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:20 smithi096 ceph-mon[102928]: from='client.54751 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:42:20.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:20 smithi096 ceph-mon[102928]: pgmap v349: 99 pgs: 99 active+clean; 4.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 956 op/s 2024-09-13T14:42:20.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:20 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:42:20.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:20 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:42:20.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:20 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:20.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:20 smithi096 ceph-mon[98901]: osdmap e136: 8 total, 8 up, 7 in 2024-09-13T14:42:20.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:20 smithi096 ceph-mon[98901]: from='client.54751 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:42:20.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:20 smithi096 ceph-mon[98901]: pgmap v349: 99 pgs: 99 active+clean; 4.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 956 op/s 2024-09-13T14:42:20.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.853 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:20.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:20.954 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:20.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:20 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:42:20.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:20 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:42:20.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:20 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:20.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:20 smithi204 ceph-mon[87071]: osdmap e136: 8 total, 8 up, 7 in 2024-09-13T14:42:20.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:20 smithi204 ceph-mon[87071]: from='client.54751 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:42:20.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:20 smithi204 ceph-mon[87071]: pgmap v349: 99 pgs: 99 active+clean; 4.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 956 op/s 2024-09-13T14:42:20.996 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:20.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:21.098 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:21.132 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:21.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:21.234 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:21.270 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:21.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:21.375 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:21.396 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:21.437 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:21.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:21.539 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:21.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.590 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:21.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:21.691 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:21.722 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:21.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:21.824 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:21.855 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:21.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:21.957 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:21.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:21.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:21.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.987 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:21.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:22.089 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:22.125 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:22.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:22.161 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:22.165 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:22.227 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:22.253 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:22.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:22.308 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:22.308 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:22.309 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:22.309 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:22.356 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:22.385 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:22.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:22.487 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:22.513 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:22.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:22.615 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:22.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.654 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:22.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:22.756 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:22.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.797 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:22.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:22.899 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:22.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.929 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:22.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:23.023 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:42:23.030 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:23.056 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:23.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:23.158 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:23.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.186 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:23.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:23.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:22 smithi204 ceph-mon[87071]: pgmap v350: 99 pgs: 99 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:42:23.288 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:23.315 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:23.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:23.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:22 smithi096 ceph-mon[98901]: pgmap v350: 99 pgs: 99 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:42:23.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:22 smithi096 ceph-mon[102928]: pgmap v350: 99 pgs: 99 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:42:23.417 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:23.445 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:23.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:23.547 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:23.578 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:23.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:23.679 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:23.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.710 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:23.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:23.811 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:23.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.841 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:23.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:23.942 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:23.993 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:23.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:24.096 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:24.127 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:24.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:24.229 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:24.272 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:24.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:24.374 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:24.409 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:24.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:24.511 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:24.555 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:24.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:24.657 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:24.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.696 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:24.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:24.797 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:24.830 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:24.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:24.932 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:24.970 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:24.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:25.021 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:24 smithi096 ceph-mon[98901]: pgmap v351: 99 pgs: 99 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 970 op/s 2024-09-13T14:42:25.021 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:24 smithi096 ceph-mon[102928]: pgmap v351: 99 pgs: 99 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 970 op/s 2024-09-13T14:42:25.072 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:25.103 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:25.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:25.205 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:24 smithi204 ceph-mon[87071]: pgmap v351: 99 pgs: 99 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 970 op/s 2024-09-13T14:42:25.236 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:25.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:25.338 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:25.372 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:25.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:25.474 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:25.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.510 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:25.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:25.611 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:25.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.640 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:25.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:25.741 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:25.771 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:25.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:25.875 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:25.904 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:25.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:26.005 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:26.040 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:26.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:26.142 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:26.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.180 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:26.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:26.281 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:26.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.318 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:26.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:26.419 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:26.444 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:26.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:26.546 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:26.572 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:26.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:26.674 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:26.700 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:26.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:26.802 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:26.831 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:26.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:26.933 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:26.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.959 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:26.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:27.060 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:27.108 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:27.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:27.210 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:27.256 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:27.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:27.358 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:27.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.393 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:27.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:27.494 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:27.527 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:27.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:27.629 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:27.658 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:27.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:27.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:27 smithi204 ceph-mon[87071]: pgmap v352: 99 pgs: 99 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 975 op/s 2024-09-13T14:42:27.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:27 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:27.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:27 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:27.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:27 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:42:27.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:27 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:42:27.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:27 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:27.739 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:27 smithi096 ceph-mon[98901]: pgmap v352: 99 pgs: 99 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 975 op/s 2024-09-13T14:42:27.739 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:27 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:27.739 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:27 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:27.739 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:27 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:42:27.739 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:27 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:42:27.739 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:27 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:27.740 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:27 smithi096 ceph-mon[102928]: pgmap v352: 99 pgs: 99 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 975 op/s 2024-09-13T14:42:27.740 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:27 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:27.740 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:27 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:27.740 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:27 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:42:27.740 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:27 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:42:27.740 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:27 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:27.759 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:27.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.791 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:27.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:27.892 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:27.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.930 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:27.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:28.031 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:28.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.063 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:28.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:28.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:42:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:42:27] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.51.0" 2024-09-13T14:42:28.164 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:28.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.206 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:28.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:28.307 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:28.339 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:42:28.340 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:42:28.352 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:28.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:28.453 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:28.482 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:28.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:28.584 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:28.614 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:28.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:28.716 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:28.750 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:28.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:28.852 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:28.883 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:28.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:28.984 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:29.018 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:29.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:29.120 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:29.147 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:29.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:29.250 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:29.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.286 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:29.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:29.387 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:29.426 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:29.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:29.528 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:29.531 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:29 smithi096 ceph-mon[98901]: pgmap v353: 99 pgs: 99 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 974 op/s 2024-09-13T14:42:29.531 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:29 smithi096 ceph-mon[102928]: pgmap v353: 99 pgs: 99 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 974 op/s 2024-09-13T14:42:29.557 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:29.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:29.659 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:29.687 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:29.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:29.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:29 smithi204 ceph-mon[87071]: pgmap v353: 99 pgs: 99 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 974 op/s 2024-09-13T14:42:29.790 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:29.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.824 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:29.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:29.926 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:29.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.960 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:29.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:30.061 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:30.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.100 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:30.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:30.201 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:30.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.249 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:30.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:30.350 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:30.387 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:30.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:30.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:42:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:42:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:30.489 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:30.517 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:30.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:30.619 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:30.647 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:30.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:30.749 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:30.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.779 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:30.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:30.873 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:42:30.880 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:30.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.915 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:30.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:30.952 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:42:30.954 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:42:31.016 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:31.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.043 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:31.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:31.144 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:31.172 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:31.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:31.274 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:31.314 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:31.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:31.416 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:31.448 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:31.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:31.550 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:31.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:31 smithi096 ceph-mon[98901]: pgmap v354: 99 pgs: 99 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 955 op/s 2024-09-13T14:42:31.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:31 smithi096 ceph-mon[102928]: pgmap v354: 99 pgs: 99 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 955 op/s 2024-09-13T14:42:31.583 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:31.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:31.685 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:31.706 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:31 smithi204 ceph-mon[87071]: pgmap v354: 99 pgs: 99 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 955 op/s 2024-09-13T14:42:31.726 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:31.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:31.757 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":354,"stamp":"2024-09-13T14:42:29.687734+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4374927333,"num_objects":66712,"num_object_clones":0,"num_object_copies":200136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66712,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":194611,"num_write_kb":8344976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":4203971,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119536,"ondisk_log_size":119536,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14348688,"kb_used_data":12971544,"kb_used_omap":34,"kb_used_meta":1376861,"kb_avail":641867376,"statfs":{"total":671965249536,"available":657272193024,"internally_reserved":0,"allocated":13282861056,"data_stored":13277257793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35126,"internal_metadata":1409906378},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-635240448,"num_objects":-9693,"num_object_clones":0,"num_object_copies":-29079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9693,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.146870"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":136,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.558983+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:42:19.558983+0000","last_peered":"2024-09-13T14:42:19.558983+0000","last_clean":"2024-09-13T14:42:19.558983+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:42:19.558983+0000","last_undegraded":"2024-09-13T14:42:19.558983+0000","last_fullsized":"2024-09-13T14:42:19.558983+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":102,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312220+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:42:09.312220+0000","last_peered":"2024-09-13T14:42:09.312220+0000","last_clean":"2024-09-13T14:42:09.312220+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:42:09.312220+0000","last_undegraded":"2024-09-13T14:42:09.312220+0000","last_fullsized":"2024-09-13T14:42:09.312220+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":100,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.317133+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:42:09.317133+0000","last_peered":"2024-09-13T14:42:09.317133+0000","last_clean":"2024-09-13T14:42:09.317133+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:42:09.317133+0000","last_undegraded":"2024-09-13T14:42:09.317133+0000","last_fullsized":"2024-09-13T14:42:09.317133+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":129,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.556573+0000","last_change":"2024-09-13T14:38:13.977954+0000","last_active":"2024-09-13T14:42:19.556573+0000","last_peered":"2024-09-13T14:42:19.556573+0000","last_clean":"2024-09-13T14:42:19.556573+0000","last_became_active":"2024-09-13T14:38:13.977619+0000","last_became_peered":"2024-09-13T14:38:13.977619+0000","last_unstale":"2024-09-13T14:42:19.556573+0000","last_undegraded":"2024-09-13T14:42:19.556573+0000","last_fullsized":"2024-09-13T14:42:19.556573+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:58:30.259967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":130,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.564525+0000","last_change":"2024-09-13T14:38:13.978232+0000","last_active":"2024-09-13T14:42:19.564525+0000","last_peered":"2024-09-13T14:42:19.564525+0000","last_clean":"2024-09-13T14:42:19.564525+0000","last_became_active":"2024-09-13T14:38:13.977921+0000","last_became_peered":"2024-09-13T14:38:13.977921+0000","last_unstale":"2024-09-13T14:42:19.564525+0000","last_undegraded":"2024-09-13T14:42:19.564525+0000","last_fullsized":"2024-09-13T14:42:19.564525+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:10.127626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":93,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.315481+0000","last_change":"2024-09-13T14:39:42.765301+0000","last_active":"2024-09-13T14:42:09.315481+0000","last_peered":"2024-09-13T14:42:09.315481+0000","last_clean":"2024-09-13T14:42:09.315481+0000","last_became_active":"2024-09-13T14:39:42.764977+0000","last_became_peered":"2024-09-13T14:39:42.764977+0000","last_unstale":"2024-09-13T14:42:09.315481+0000","last_undegraded":"2024-09-13T14:42:09.315481+0000","last_fullsized":"2024-09-13T14:42:09.315481+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:25.223972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.563670+0000","last_change":"2024-09-13T14:37:40.925691+0000","last_active":"2024-09-13T14:42:19.563670+0000","last_peered":"2024-09-13T14:42:19.563670+0000","last_clean":"2024-09-13T14:42:19.563670+0000","last_became_active":"2024-09-13T14:37:40.925360+0000","last_became_peered":"2024-09-13T14:37:40.925360+0000","last_unstale":"2024-09-13T14:42:19.563670+0000","last_undegraded":"2024-09-13T14:42:19.563670+0000","last_fullsized":"2024-09-13T14:42:19.563670+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:07.337967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":91,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.555997+0000","last_change":"2024-09-13T14:39:42.781872+0000","last_active":"2024-09-13T14:42:19.555997+0000","last_peered":"2024-09-13T14:42:19.555997+0000","last_clean":"2024-09-13T14:42:19.555997+0000","last_became_active":"2024-09-13T14:39:42.770254+0000","last_became_peered":"2024-09-13T14:39:42.770254+0000","last_unstale":"2024-09-13T14:42:19.555997+0000","last_undegraded":"2024-09-13T14:42:19.555997+0000","last_fullsized":"2024-09-13T14:42:19.555997+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:31.211081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.9","version":"136'11951","reported_seq":11988,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:26.622408+0000","last_change":"2024-09-13T14:39:42.764535+0000","last_active":"2024-09-13T14:42:26.622408+0000","last_peered":"2024-09-13T14:42:26.622408+0000","last_clean":"2024-09-13T14:42:26.622408+0000","last_became_active":"2024-09-13T14:39:42.764101+0000","last_became_peered":"2024-09-13T14:39:42.764101+0000","last_unstale":"2024-09-13T14:42:26.622408+0000","last_undegraded":"2024-09-13T14:42:26.622408+0000","last_fullsized":"2024-09-13T14:42:26.622408+0000","mapping_epoch":107,"log_start":"119'4000","ondisk_log_start":"119'4000","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":7951,"log_dups_size":0,"ondisk_log_size":7951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:22.498969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276627456,"num_objects":4221,"num_object_clones":0,"num_object_copies":12663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11951,"num_write_kb":517504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":121,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.560781+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:42:19.560781+0000","last_peered":"2024-09-13T14:42:19.560781+0000","last_clean":"2024-09-13T14:42:19.560781+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:42:19.560781+0000","last_undegraded":"2024-09-13T14:42:19.560781+0000","last_fullsized":"2024-09-13T14:42:19.560781+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":93,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.316273+0000","last_change":"2024-09-13T14:39:42.764876+0000","last_active":"2024-09-13T14:42:09.316273+0000","last_peered":"2024-09-13T14:42:09.316273+0000","last_clean":"2024-09-13T14:42:09.316273+0000","last_became_active":"2024-09-13T14:39:42.764082+0000","last_became_peered":"2024-09-13T14:39:42.764082+0000","last_unstale":"2024-09-13T14:42:09.316273+0000","last_undegraded":"2024-09-13T14:42:09.316273+0000","last_fullsized":"2024-09-13T14:42:09.316273+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:10:07.458952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"136'12008","reported_seq":12077,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:29.070056+0000","last_change":"2024-09-13T14:39:42.767715+0000","last_active":"2024-09-13T14:42:29.070056+0000","last_peered":"2024-09-13T14:42:29.070056+0000","last_clean":"2024-09-13T14:42:29.070056+0000","last_became_active":"2024-09-13T14:39:42.766678+0000","last_became_peered":"2024-09-13T14:39:42.766678+0000","last_unstale":"2024-09-13T14:42:29.070056+0000","last_undegraded":"2024-09-13T14:42:29.070056+0000","last_fullsized":"2024-09-13T14:42:29.070056+0000","mapping_epoch":107,"log_start":"117'3800","ondisk_log_start":"117'3800","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":8208,"log_dups_size":0,"ondisk_log_size":8208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:30:38.645181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261619712,"num_objects":3992,"num_object_clones":0,"num_object_copies":11976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12008,"num_write_kb":512000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":102,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312653+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:42:09.312653+0000","last_peered":"2024-09-13T14:42:09.312653+0000","last_clean":"2024-09-13T14:42:09.312653+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:42:09.312653+0000","last_undegraded":"2024-09-13T14:42:09.312653+0000","last_fullsized":"2024-09-13T14:42:09.312653+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":127,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.315121+0000","last_change":"2024-09-13T14:38:13.979677+0000","last_active":"2024-09-13T14:42:09.315121+0000","last_peered":"2024-09-13T14:42:09.315121+0000","last_clean":"2024-09-13T14:42:09.315121+0000","last_became_active":"2024-09-13T14:38:13.979323+0000","last_became_peered":"2024-09-13T14:38:13.979323+0000","last_unstale":"2024-09-13T14:42:09.315121+0000","last_undegraded":"2024-09-13T14:42:09.315121+0000","last_fullsized":"2024-09-13T14:42:09.315121+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:29:03.491254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"136'12144","reported_seq":12214,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:29.059311+0000","last_change":"2024-09-13T14:39:42.766942+0000","last_active":"2024-09-13T14:42:29.059311+0000","last_peered":"2024-09-13T14:42:29.059311+0000","last_clean":"2024-09-13T14:42:29.059311+0000","last_became_active":"2024-09-13T14:39:42.766647+0000","last_became_peered":"2024-09-13T14:39:42.766647+0000","last_unstale":"2024-09-13T14:42:29.059311+0000","last_undegraded":"2024-09-13T14:42:29.059311+0000","last_fullsized":"2024-09-13T14:42:29.059311+0000","mapping_epoch":107,"log_start":"118'4000","ondisk_log_start":"118'4000","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":8144,"log_dups_size":0,"ondisk_log_size":8144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:16.267901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258473984,"num_objects":3944,"num_object_clones":0,"num_object_copies":11832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12144,"num_write_kb":514816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":98,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312563+0000","last_change":"2024-09-13T14:39:42.774547+0000","last_active":"2024-09-13T14:42:09.312563+0000","last_peered":"2024-09-13T14:42:09.312563+0000","last_clean":"2024-09-13T14:42:09.312563+0000","last_became_active":"2024-09-13T14:39:42.766006+0000","last_became_peered":"2024-09-13T14:39:42.766006+0000","last_unstale":"2024-09-13T14:42:09.312563+0000","last_undegraded":"2024-09-13T14:42:09.312563+0000","last_fullsized":"2024-09-13T14:42:09.312563+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:06:29.632163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":120,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.545978+0000","last_change":"2024-09-13T14:39:42.765763+0000","last_active":"2024-09-13T14:42:19.545978+0000","last_peered":"2024-09-13T14:42:19.545978+0000","last_clean":"2024-09-13T14:42:19.545978+0000","last_became_active":"2024-09-13T14:39:42.765441+0000","last_became_peered":"2024-09-13T14:39:42.765441+0000","last_unstale":"2024-09-13T14:42:19.545978+0000","last_undegraded":"2024-09-13T14:42:19.545978+0000","last_fullsized":"2024-09-13T14:42:19.545978+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:06.483638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.a","version":"136'12504","reported_seq":12574,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:29.041705+0000","last_change":"2024-09-13T14:39:42.763571+0000","last_active":"2024-09-13T14:42:29.041705+0000","last_peered":"2024-09-13T14:42:29.041705+0000","last_clean":"2024-09-13T14:42:29.041705+0000","last_became_active":"2024-09-13T14:39:42.763270+0000","last_became_peered":"2024-09-13T14:39:42.763270+0000","last_unstale":"2024-09-13T14:42:29.041705+0000","last_undegraded":"2024-09-13T14:42:29.041705+0000","last_fullsized":"2024-09-13T14:42:29.041705+0000","mapping_epoch":107,"log_start":"119'4300","ondisk_log_start":"119'4300","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":8204,"log_dups_size":0,"ondisk_log_size":8204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:38:09.281466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274202624,"num_objects":4184,"num_object_clones":0,"num_object_copies":12552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12504,"num_write_kb":534016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":111,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.316906+0000","last_change":"2024-09-13T14:39:42.765558+0000","last_active":"2024-09-13T14:42:09.316906+0000","last_peered":"2024-09-13T14:42:09.316906+0000","last_clean":"2024-09-13T14:42:09.316906+0000","last_became_active":"2024-09-13T14:39:42.765246+0000","last_became_peered":"2024-09-13T14:39:42.765246+0000","last_unstale":"2024-09-13T14:42:09.316906+0000","last_undegraded":"2024-09-13T14:42:09.316906+0000","last_fullsized":"2024-09-13T14:42:09.316906+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:08.640610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":130,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.557253+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:42:19.557253+0000","last_peered":"2024-09-13T14:42:19.557253+0000","last_clean":"2024-09-13T14:42:19.557253+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:42:19.557253+0000","last_undegraded":"2024-09-13T14:42:19.557253+0000","last_fullsized":"2024-09-13T14:42:19.557253+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":263,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.315232+0000","last_change":"2024-09-13T14:40:32.590204+0000","last_active":"2024-09-13T14:42:09.315232+0000","last_peered":"2024-09-13T14:42:09.315232+0000","last_clean":"2024-09-13T14:42:09.315232+0000","last_became_active":"2024-09-13T14:40:22.949830+0000","last_became_peered":"2024-09-13T14:40:22.949830+0000","last_unstale":"2024-09-13T14:42:09.315232+0000","last_undegraded":"2024-09-13T14:42:09.315232+0000","last_fullsized":"2024-09-13T14:42:09.315232+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:32.472827+0000","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.d","version":"136'12231","reported_seq":12302,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:29.070396+0000","last_change":"2024-09-13T14:39:42.759038+0000","last_active":"2024-09-13T14:42:29.070396+0000","last_peered":"2024-09-13T14:42:29.070396+0000","last_clean":"2024-09-13T14:42:29.070396+0000","last_became_active":"2024-09-13T14:39:42.758753+0000","last_became_peered":"2024-09-13T14:39:42.758753+0000","last_unstale":"2024-09-13T14:42:29.070396+0000","last_undegraded":"2024-09-13T14:42:29.070396+0000","last_fullsized":"2024-09-13T14:42:29.070396+0000","mapping_epoch":107,"log_start":"119'4100","ondisk_log_start":"119'4100","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":8131,"log_dups_size":0,"ondisk_log_size":8131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:14:07.557396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270532632,"num_objects":4129,"num_object_clones":0,"num_object_copies":12387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4129,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12231,"num_write_kb":523457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":133,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.558867+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:42:19.558867+0000","last_peered":"2024-09-13T14:42:19.558867+0000","last_clean":"2024-09-13T14:42:19.558867+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:42:19.558867+0000","last_undegraded":"2024-09-13T14:42:19.558867+0000","last_fullsized":"2024-09-13T14:42:19.558867+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":110,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.313899+0000","last_change":"2024-09-13T14:39:42.775584+0000","last_active":"2024-09-13T14:42:09.313899+0000","last_peered":"2024-09-13T14:42:09.313899+0000","last_clean":"2024-09-13T14:42:09.313899+0000","last_became_active":"2024-09-13T14:39:42.775254+0000","last_became_peered":"2024-09-13T14:39:42.775254+0000","last_unstale":"2024-09-13T14:42:09.313899+0000","last_undegraded":"2024-09-13T14:42:09.313899+0000","last_fullsized":"2024-09-13T14:42:09.313899+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:44.607098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":44,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.316092+0000","last_change":"2024-09-13T14:40:27.868402+0000","last_active":"2024-09-13T14:42:09.316092+0000","last_peered":"2024-09-13T14:42:09.316092+0000","last_clean":"2024-09-13T14:42:09.316092+0000","last_became_active":"2024-09-13T14:40:22.966301+0000","last_became_peered":"2024-09-13T14:40:22.966301+0000","last_unstale":"2024-09-13T14:42:09.316092+0000","last_undegraded":"2024-09-13T14:42:09.316092+0000","last_fullsized":"2024-09-13T14:42:09.316092+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:18.660186+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.c","version":"136'12058","reported_seq":12128,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:27.043070+0000","last_change":"2024-09-13T14:39:42.772481+0000","last_active":"2024-09-13T14:42:27.043070+0000","last_peered":"2024-09-13T14:42:27.043070+0000","last_clean":"2024-09-13T14:42:27.043070+0000","last_became_active":"2024-09-13T14:39:42.772211+0000","last_became_peered":"2024-09-13T14:39:42.772211+0000","last_unstale":"2024-09-13T14:42:27.043070+0000","last_undegraded":"2024-09-13T14:42:27.043070+0000","last_fullsized":"2024-09-13T14:42:27.043070+0000","mapping_epoch":107,"log_start":"120'5200","ondisk_log_start":"120'5200","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6858,"log_dups_size":0,"ondisk_log_size":6858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:30:16.860807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276430848,"num_objects":4218,"num_object_clones":0,"num_object_copies":12654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12058,"num_write_kb":520832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":92,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.314161+0000","last_change":"2024-09-13T14:39:42.776751+0000","last_active":"2024-09-13T14:42:09.314161+0000","last_peered":"2024-09-13T14:42:09.314161+0000","last_clean":"2024-09-13T14:42:09.314161+0000","last_became_active":"2024-09-13T14:39:42.774855+0000","last_became_peered":"2024-09-13T14:39:42.774855+0000","last_unstale":"2024-09-13T14:42:09.314161+0000","last_undegraded":"2024-09-13T14:42:09.314161+0000","last_fullsized":"2024-09-13T14:42:09.314161+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:11.022013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":117,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.318903+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:42:09.318903+0000","last_peered":"2024-09-13T14:42:09.318903+0000","last_clean":"2024-09-13T14:42:09.318903+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:42:09.318903+0000","last_undegraded":"2024-09-13T14:42:09.318903+0000","last_fullsized":"2024-09-13T14:42:09.318903+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"136'12027","reported_seq":12064,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:26.643959+0000","last_change":"2024-09-13T14:39:42.764380+0000","last_active":"2024-09-13T14:42:26.643959+0000","last_peered":"2024-09-13T14:42:26.643959+0000","last_clean":"2024-09-13T14:42:26.643959+0000","last_became_active":"2024-09-13T14:39:42.764127+0000","last_became_peered":"2024-09-13T14:39:42.764127+0000","last_unstale":"2024-09-13T14:42:26.643959+0000","last_undegraded":"2024-09-13T14:42:26.643959+0000","last_fullsized":"2024-09-13T14:42:26.643959+0000","mapping_epoch":107,"log_start":"120'4700","ondisk_log_start":"120'4700","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":7327,"log_dups_size":0,"ondisk_log_size":7327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:37:32.713989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285147136,"num_objects":4351,"num_object_clones":0,"num_object_copies":13053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12027,"num_write_kb":524096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":117,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.556049+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:42:19.556049+0000","last_peered":"2024-09-13T14:42:19.556049+0000","last_clean":"2024-09-13T14:42:19.556049+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:42:19.556049+0000","last_undegraded":"2024-09-13T14:42:19.556049+0000","last_fullsized":"2024-09-13T14:42:19.556049+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.322388+0000","last_change":"2024-09-13T14:39:42.776187+0000","last_active":"2024-09-13T14:42:09.322388+0000","last_peered":"2024-09-13T14:42:09.322388+0000","last_clean":"2024-09-13T14:42:09.322388+0000","last_became_active":"2024-09-13T14:39:42.775751+0000","last_became_peered":"2024-09-13T14:39:42.775751+0000","last_unstale":"2024-09-13T14:42:09.322388+0000","last_undegraded":"2024-09-13T14:42:09.322388+0000","last_fullsized":"2024-09-13T14:42:09.322388+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:35.155717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":37,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.316412+0000","last_change":"2024-09-13T14:40:25.861254+0000","last_active":"2024-09-13T14:42:09.316412+0000","last_peered":"2024-09-13T14:42:09.316412+0000","last_clean":"2024-09-13T14:42:09.316412+0000","last_became_active":"2024-09-13T14:40:22.957614+0000","last_became_peered":"2024-09-13T14:40:22.957614+0000","last_unstale":"2024-09-13T14:42:09.316412+0000","last_undegraded":"2024-09-13T14:42:09.316412+0000","last_fullsized":"2024-09-13T14:42:09.316412+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:53.388190+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.e","version":"136'12332","reported_seq":12369,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:26.655926+0000","last_change":"2024-09-13T14:39:42.772007+0000","last_active":"2024-09-13T14:42:26.655926+0000","last_peered":"2024-09-13T14:42:26.655926+0000","last_clean":"2024-09-13T14:42:26.655926+0000","last_became_active":"2024-09-13T14:39:42.771672+0000","last_became_peered":"2024-09-13T14:39:42.771672+0000","last_unstale":"2024-09-13T14:42:26.655926+0000","last_undegraded":"2024-09-13T14:42:26.655926+0000","last_fullsized":"2024-09-13T14:42:26.655926+0000","mapping_epoch":107,"log_start":"120'5500","ondisk_log_start":"120'5500","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6832,"log_dups_size":0,"ondisk_log_size":6832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:26:12.134635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280231936,"num_objects":4276,"num_object_clones":0,"num_object_copies":12828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12332,"num_write_kb":531456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":137,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.571539+0000","last_change":"2024-09-13T14:38:13.981576+0000","last_active":"2024-09-13T14:42:19.571539+0000","last_peered":"2024-09-13T14:42:19.571539+0000","last_clean":"2024-09-13T14:42:19.571539+0000","last_became_active":"2024-09-13T14:38:13.981262+0000","last_became_peered":"2024-09-13T14:38:13.981262+0000","last_unstale":"2024-09-13T14:42:19.571539+0000","last_undegraded":"2024-09-13T14:42:19.571539+0000","last_fullsized":"2024-09-13T14:42:19.571539+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:20.284250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":118,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.553351+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:42:19.553351+0000","last_peered":"2024-09-13T14:42:19.553351+0000","last_clean":"2024-09-13T14:42:19.553351+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:42:19.553351+0000","last_undegraded":"2024-09-13T14:42:19.553351+0000","last_fullsized":"2024-09-13T14:42:19.553351+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.4","version":"136'12009","reported_seq":12046,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:26.633494+0000","last_change":"2024-09-13T14:39:42.765514+0000","last_active":"2024-09-13T14:42:26.633494+0000","last_peered":"2024-09-13T14:42:26.633494+0000","last_clean":"2024-09-13T14:42:26.633494+0000","last_became_active":"2024-09-13T14:39:42.765209+0000","last_became_peered":"2024-09-13T14:39:42.765209+0000","last_unstale":"2024-09-13T14:42:26.633494+0000","last_undegraded":"2024-09-13T14:42:26.633494+0000","last_fullsized":"2024-09-13T14:42:26.633494+0000","mapping_epoch":107,"log_start":"119'4100","ondisk_log_start":"119'4100","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":7909,"log_dups_size":0,"ondisk_log_size":7909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:59.785672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275316736,"num_objects":4201,"num_object_clones":0,"num_object_copies":12603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12009,"num_write_kb":518720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":254,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312154+0000","last_change":"2024-09-13T14:40:22.639864+0000","last_active":"2024-09-13T14:42:09.312154+0000","last_peered":"2024-09-13T14:42:09.312154+0000","last_clean":"2024-09-13T14:42:09.312154+0000","last_became_active":"2024-09-13T14:40:20.926503+0000","last_became_peered":"2024-09-13T14:40:20.926503+0000","last_unstale":"2024-09-13T14:42:09.312154+0000","last_undegraded":"2024-09-13T14:42:09.312154+0000","last_fullsized":"2024-09-13T14:42:09.312154+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:22.639760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:22.639760+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:08.523563+0000","scrub_duration":0.0051120610000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":663,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.319604+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:42:09.319604+0000","last_peered":"2024-09-13T14:42:09.319604+0000","last_clean":"2024-09-13T14:42:09.319604+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:42:09.319604+0000","last_undegraded":"2024-09-13T14:42:09.319604+0000","last_fullsized":"2024-09-13T14:42:09.319604+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":95,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.560051+0000","last_change":"2024-09-13T14:39:42.768341+0000","last_active":"2024-09-13T14:42:19.560051+0000","last_peered":"2024-09-13T14:42:19.560051+0000","last_clean":"2024-09-13T14:42:19.560051+0000","last_became_active":"2024-09-13T14:39:42.768034+0000","last_became_peered":"2024-09-13T14:39:42.768034+0000","last_unstale":"2024-09-13T14:42:19.560051+0000","last_undegraded":"2024-09-13T14:42:19.560051+0000","last_fullsized":"2024-09-13T14:42:19.560051+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:28:55.502108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":42,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.314208+0000","last_change":"2024-09-13T14:39:42.772096+0000","last_active":"2024-09-13T14:42:09.314208+0000","last_peered":"2024-09-13T14:42:09.314208+0000","last_clean":"2024-09-13T14:42:09.314208+0000","last_became_active":"2024-09-13T14:39:42.771762+0000","last_became_peered":"2024-09-13T14:39:42.771762+0000","last_unstale":"2024-09-13T14:42:09.314208+0000","last_undegraded":"2024-09-13T14:42:09.314208+0000","last_fullsized":"2024-09-13T14:42:09.314208+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:16.172571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":246,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.551696+0000","last_change":"2024-09-13T14:40:23.784403+0000","last_active":"2024-09-13T14:42:19.551696+0000","last_peered":"2024-09-13T14:42:19.551696+0000","last_clean":"2024-09-13T14:42:19.551696+0000","last_became_active":"2024-09-13T14:40:22.951246+0000","last_became_peered":"2024-09-13T14:40:22.951246+0000","last_unstale":"2024-09-13T14:42:19.551696+0000","last_undegraded":"2024-09-13T14:42:19.551696+0000","last_fullsized":"2024-09-13T14:42:19.551696+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.784263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.784263+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:48:09.437261+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.7","version":"136'11934","reported_seq":11971,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:26.670852+0000","last_change":"2024-09-13T14:39:42.773740+0000","last_active":"2024-09-13T14:42:26.670852+0000","last_peered":"2024-09-13T14:42:26.670852+0000","last_clean":"2024-09-13T14:42:26.670852+0000","last_became_active":"2024-09-13T14:39:42.773444+0000","last_became_peered":"2024-09-13T14:39:42.773444+0000","last_unstale":"2024-09-13T14:42:26.670852+0000","last_undegraded":"2024-09-13T14:42:26.670852+0000","last_fullsized":"2024-09-13T14:42:26.670852+0000","mapping_epoch":107,"log_start":"120'4600","ondisk_log_start":"120'4600","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":7334,"log_dups_size":0,"ondisk_log_size":7334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:09:16.690630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272367616,"num_objects":4156,"num_object_clones":0,"num_object_copies":12468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11934,"num_write_kb":514880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":347,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312074+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:42:09.312074+0000","last_peered":"2024-09-13T14:42:09.312074+0000","last_clean":"2024-09-13T14:42:09.312074+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:42:09.312074+0000","last_undegraded":"2024-09-13T14:42:09.312074+0000","last_fullsized":"2024-09-13T14:42:09.312074+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":138,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.556100+0000","last_change":"2024-09-13T14:38:13.977962+0000","last_active":"2024-09-13T14:42:19.556100+0000","last_peered":"2024-09-13T14:42:19.556100+0000","last_clean":"2024-09-13T14:42:19.556100+0000","last_became_active":"2024-09-13T14:38:13.977638+0000","last_became_peered":"2024-09-13T14:38:13.977638+0000","last_unstale":"2024-09-13T14:42:19.556100+0000","last_undegraded":"2024-09-13T14:42:19.556100+0000","last_fullsized":"2024-09-13T14:42:19.556100+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:53:16.902157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":142,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.564692+0000","last_change":"2024-09-13T14:37:40.927444+0000","last_active":"2024-09-13T14:42:19.564692+0000","last_peered":"2024-09-13T14:42:19.564692+0000","last_clean":"2024-09-13T14:42:19.564692+0000","last_became_active":"2024-09-13T14:37:40.927140+0000","last_became_peered":"2024-09-13T14:37:40.927140+0000","last_unstale":"2024-09-13T14:42:19.564692+0000","last_undegraded":"2024-09-13T14:42:19.564692+0000","last_fullsized":"2024-09-13T14:42:19.564692+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:32:54.207568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":263,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312062+0000","last_change":"2024-09-13T14:40:28.602999+0000","last_active":"2024-09-13T14:42:09.312062+0000","last_peered":"2024-09-13T14:42:09.312062+0000","last_clean":"2024-09-13T14:42:09.312062+0000","last_became_active":"2024-09-13T14:40:22.949002+0000","last_became_peered":"2024-09-13T14:40:22.949002+0000","last_unstale":"2024-09-13T14:42:09.312062+0000","last_undegraded":"2024-09-13T14:42:09.312062+0000","last_fullsized":"2024-09-13T14:42:09.312062+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:28.602942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:28.602942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:29:46.037596+0000","scrub_duration":0.00945176,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.6","version":"136'12081","reported_seq":12151,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:29.021685+0000","last_change":"2024-09-13T14:39:42.759808+0000","last_active":"2024-09-13T14:42:29.021685+0000","last_peered":"2024-09-13T14:42:29.021685+0000","last_clean":"2024-09-13T14:42:29.021685+0000","last_became_active":"2024-09-13T14:39:42.759199+0000","last_became_peered":"2024-09-13T14:39:42.759199+0000","last_unstale":"2024-09-13T14:42:29.021685+0000","last_undegraded":"2024-09-13T14:42:29.021685+0000","last_fullsized":"2024-09-13T14:42:29.021685+0000","mapping_epoch":107,"log_start":"117'3900","ondisk_log_start":"117'3900","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":8181,"log_dups_size":0,"ondisk_log_size":8181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:55:55.573511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264437760,"num_objects":4035,"num_object_clones":0,"num_object_copies":12105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12081,"num_write_kb":515712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":542,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.555520+0000","last_change":"2024-09-13T14:38:13.978679+0000","last_active":"2024-09-13T14:42:19.555520+0000","last_peered":"2024-09-13T14:42:19.555520+0000","last_clean":"2024-09-13T14:42:19.555520+0000","last_became_active":"2024-09-13T14:38:13.978171+0000","last_became_peered":"2024-09-13T14:38:13.978171+0000","last_unstale":"2024-09-13T14:42:19.555520+0000","last_undegraded":"2024-09-13T14:42:19.555520+0000","last_fullsized":"2024-09-13T14:42:19.555520+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:57.691198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":125,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.553868+0000","last_change":"2024-09-13T14:39:42.769691+0000","last_active":"2024-09-13T14:42:19.553868+0000","last_peered":"2024-09-13T14:42:19.553868+0000","last_clean":"2024-09-13T14:42:19.553868+0000","last_became_active":"2024-09-13T14:39:42.769266+0000","last_became_peered":"2024-09-13T14:39:42.769266+0000","last_unstale":"2024-09-13T14:42:19.553868+0000","last_undegraded":"2024-09-13T14:42:19.553868+0000","last_fullsized":"2024-09-13T14:42:19.553868+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:11.637732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":151,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.553193+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:42:19.553193+0000","last_peered":"2024-09-13T14:42:19.553193+0000","last_clean":"2024-09-13T14:42:19.553193+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:42:19.553193+0000","last_undegraded":"2024-09-13T14:42:19.553193+0000","last_fullsized":"2024-09-13T14:42:19.553193+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":37,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.316093+0000","last_change":"2024-09-13T14:40:26.896691+0000","last_active":"2024-09-13T14:42:09.316093+0000","last_peered":"2024-09-13T14:42:09.316093+0000","last_clean":"2024-09-13T14:42:09.316093+0000","last_became_active":"2024-09-13T14:40:22.949772+0000","last_became_peered":"2024-09-13T14:40:22.949772+0000","last_unstale":"2024-09-13T14:42:09.316093+0000","last_undegraded":"2024-09-13T14:42:09.316093+0000","last_fullsized":"2024-09-13T14:42:09.316093+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:26.896613+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:26.896613+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:01.615433+0000","scrub_duration":0.004120828,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.5","version":"136'12055","reported_seq":12125,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:28.303602+0000","last_change":"2024-09-13T14:39:42.776210+0000","last_active":"2024-09-13T14:42:28.303602+0000","last_peered":"2024-09-13T14:42:28.303602+0000","last_clean":"2024-09-13T14:42:28.303602+0000","last_became_active":"2024-09-13T14:39:42.775823+0000","last_became_peered":"2024-09-13T14:39:42.775823+0000","last_unstale":"2024-09-13T14:42:28.303602+0000","last_undegraded":"2024-09-13T14:42:28.303602+0000","last_fullsized":"2024-09-13T14:42:28.303602+0000","mapping_epoch":107,"log_start":"121'5900","ondisk_log_start":"121'5900","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6155,"log_dups_size":0,"ondisk_log_size":6155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:39.020681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267976704,"num_objects":4089,"num_object_clones":0,"num_object_copies":12267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12055,"num_write_kb":516608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":727,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.316696+0000","last_change":"2024-09-13T14:39:42.805055+0000","last_active":"2024-09-13T14:42:09.316696+0000","last_peered":"2024-09-13T14:42:09.316696+0000","last_clean":"2024-09-13T14:42:09.316696+0000","last_became_active":"2024-09-13T14:39:42.762456+0000","last_became_peered":"2024-09-13T14:39:42.762456+0000","last_unstale":"2024-09-13T14:42:09.316696+0000","last_undegraded":"2024-09-13T14:42:09.316696+0000","last_fullsized":"2024-09-13T14:42:09.316696+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:54:40.839991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":111,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.314009+0000","last_change":"2024-09-13T14:39:42.775159+0000","last_active":"2024-09-13T14:42:09.314009+0000","last_peered":"2024-09-13T14:42:09.314009+0000","last_clean":"2024-09-13T14:42:09.314009+0000","last_became_active":"2024-09-13T14:39:42.774841+0000","last_became_peered":"2024-09-13T14:39:42.774841+0000","last_unstale":"2024-09-13T14:42:09.314009+0000","last_undegraded":"2024-09-13T14:42:09.314009+0000","last_fullsized":"2024-09-13T14:42:09.314009+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:35:45.579397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":129,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.552970+0000","last_change":"2024-09-13T14:38:13.981294+0000","last_active":"2024-09-13T14:42:19.552970+0000","last_peered":"2024-09-13T14:42:19.552970+0000","last_clean":"2024-09-13T14:42:19.552970+0000","last_became_active":"2024-09-13T14:38:13.980975+0000","last_became_peered":"2024-09-13T14:38:13.980975+0000","last_unstale":"2024-09-13T14:42:19.552970+0000","last_undegraded":"2024-09-13T14:42:19.552970+0000","last_fullsized":"2024-09-13T14:42:19.552970+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:17.545179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":55,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.559540+0000","last_change":"2024-09-13T14:40:23.093425+0000","last_active":"2024-09-13T14:42:19.559540+0000","last_peered":"2024-09-13T14:42:19.559540+0000","last_clean":"2024-09-13T14:42:19.559540+0000","last_became_active":"2024-09-13T14:40:22.947358+0000","last_became_peered":"2024-09-13T14:40:22.947358+0000","last_unstale":"2024-09-13T14:42:19.559540+0000","last_undegraded":"2024-09-13T14:42:19.559540+0000","last_fullsized":"2024-09-13T14:42:19.559540+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.093356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.093356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:38:28.248105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.3","version":"136'12000","reported_seq":12068,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:28.272572+0000","last_change":"2024-09-13T14:39:42.770363+0000","last_active":"2024-09-13T14:42:28.272572+0000","last_peered":"2024-09-13T14:42:28.272572+0000","last_clean":"2024-09-13T14:42:28.272572+0000","last_became_active":"2024-09-13T14:39:42.770092+0000","last_became_peered":"2024-09-13T14:39:42.770092+0000","last_unstale":"2024-09-13T14:42:28.272572+0000","last_undegraded":"2024-09-13T14:42:28.272572+0000","last_fullsized":"2024-09-13T14:42:28.272572+0000","mapping_epoch":107,"log_start":"121'5800","ondisk_log_start":"121'5800","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6200,"log_dups_size":0,"ondisk_log_size":6200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:20:20.375252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265551872,"num_objects":4052,"num_object_clones":0,"num_object_copies":12156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12000,"num_write_kb":513664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":118,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.317267+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:42:09.317267+0000","last_peered":"2024-09-13T14:42:09.317267+0000","last_clean":"2024-09-13T14:42:09.317267+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:42:09.317267+0000","last_undegraded":"2024-09-13T14:42:09.317267+0000","last_fullsized":"2024-09-13T14:42:09.317267+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":635,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.553134+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:42:19.553134+0000","last_peered":"2024-09-13T14:42:19.553134+0000","last_clean":"2024-09-13T14:42:19.553134+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:42:19.553134+0000","last_undegraded":"2024-09-13T14:42:19.553134+0000","last_fullsized":"2024-09-13T14:42:19.553134+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":107,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.546298+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:42:19.546298+0000","last_peered":"2024-09-13T14:42:19.546298+0000","last_clean":"2024-09-13T14:42:19.546298+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:42:19.546298+0000","last_undegraded":"2024-09-13T14:42:19.546298+0000","last_fullsized":"2024-09-13T14:42:19.546298+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":201,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.564283+0000","last_change":"2024-09-13T14:40:23.334441+0000","last_active":"2024-09-13T14:42:19.564283+0000","last_peered":"2024-09-13T14:42:19.564283+0000","last_clean":"2024-09-13T14:42:19.564283+0000","last_became_active":"2024-09-13T14:40:22.952908+0000","last_became_peered":"2024-09-13T14:40:22.952908+0000","last_unstale":"2024-09-13T14:42:19.564283+0000","last_undegraded":"2024-09-13T14:42:19.564283+0000","last_fullsized":"2024-09-13T14:42:19.564283+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.334379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.334379+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:58:19.770506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"136'12207","reported_seq":12277,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:28.293356+0000","last_change":"2024-09-13T14:39:42.770399+0000","last_active":"2024-09-13T14:42:28.293356+0000","last_peered":"2024-09-13T14:42:28.293356+0000","last_clean":"2024-09-13T14:42:28.293356+0000","last_became_active":"2024-09-13T14:39:42.770074+0000","last_became_peered":"2024-09-13T14:39:42.770074+0000","last_unstale":"2024-09-13T14:42:28.293356+0000","last_undegraded":"2024-09-13T14:42:28.293356+0000","last_fullsized":"2024-09-13T14:42:28.293356+0000","mapping_epoch":107,"log_start":"121'6000","ondisk_log_start":"121'6000","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6207,"log_dups_size":0,"ondisk_log_size":6207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:28:03.030813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270598144,"num_objects":4129,"num_object_clones":0,"num_object_copies":12387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12207,"num_write_kb":522752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":118,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.553048+0000","last_change":"2024-09-13T14:37:40.921861+0000","last_active":"2024-09-13T14:42:19.553048+0000","last_peered":"2024-09-13T14:42:19.553048+0000","last_clean":"2024-09-13T14:42:19.553048+0000","last_became_active":"2024-09-13T14:37:40.921587+0000","last_became_peered":"2024-09-13T14:37:40.921587+0000","last_unstale":"2024-09-13T14:42:19.553048+0000","last_undegraded":"2024-09-13T14:42:19.553048+0000","last_fullsized":"2024-09-13T14:42:19.553048+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:24.619352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":349,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312645+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:42:09.312645+0000","last_peered":"2024-09-13T14:42:09.312645+0000","last_clean":"2024-09-13T14:42:09.312645+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:42:09.312645+0000","last_undegraded":"2024-09-13T14:42:09.312645+0000","last_fullsized":"2024-09-13T14:42:09.312645+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":120,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.557169+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:42:19.557169+0000","last_peered":"2024-09-13T14:42:19.557169+0000","last_clean":"2024-09-13T14:42:19.557169+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:42:19.557169+0000","last_undegraded":"2024-09-13T14:42:19.557169+0000","last_fullsized":"2024-09-13T14:42:19.557169+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":37,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.313830+0000","last_change":"2024-09-13T14:40:30.500108+0000","last_active":"2024-09-13T14:42:09.313830+0000","last_peered":"2024-09-13T14:42:09.313830+0000","last_clean":"2024-09-13T14:42:09.313830+0000","last_became_active":"2024-09-13T14:40:22.949882+0000","last_became_peered":"2024-09-13T14:40:22.949882+0000","last_unstale":"2024-09-13T14:42:09.313830+0000","last_undegraded":"2024-09-13T14:42:09.313830+0000","last_fullsized":"2024-09-13T14:42:09.313830+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:40:30.500046+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:22:58.480202+0000","scrub_duration":0.00248832,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1","version":"136'12108","reported_seq":12145,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:26.671224+0000","last_change":"2024-09-13T14:39:42.772158+0000","last_active":"2024-09-13T14:42:26.671224+0000","last_peered":"2024-09-13T14:42:26.671224+0000","last_clean":"2024-09-13T14:42:26.671224+0000","last_became_active":"2024-09-13T14:39:42.771189+0000","last_became_peered":"2024-09-13T14:39:42.771189+0000","last_unstale":"2024-09-13T14:42:26.671224+0000","last_undegraded":"2024-09-13T14:42:26.671224+0000","last_fullsized":"2024-09-13T14:42:26.671224+0000","mapping_epoch":107,"log_start":"120'5200","ondisk_log_start":"120'5200","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6908,"log_dups_size":0,"ondisk_log_size":6908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:58:08.882288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289538048,"num_objects":4418,"num_object_clones":0,"num_object_copies":13254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12108,"num_write_kb":528832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":133,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.559700+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:42:19.559700+0000","last_peered":"2024-09-13T14:42:19.559700+0000","last_clean":"2024-09-13T14:42:19.559700+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:42:19.559700+0000","last_undegraded":"2024-09-13T14:42:19.559700+0000","last_fullsized":"2024-09-13T14:42:19.559700+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":456,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.552464+0000","last_change":"2024-09-13T14:39:44.990216+0000","last_active":"2024-09-13T14:42:19.552464+0000","last_peered":"2024-09-13T14:42:19.552464+0000","last_clean":"2024-09-13T14:42:19.552464+0000","last_became_active":"2024-09-13T14:39:44.989433+0000","last_became_peered":"2024-09-13T14:39:44.989433+0000","last_unstale":"2024-09-13T14:42:19.552464+0000","last_undegraded":"2024-09-13T14:42:19.552464+0000","last_fullsized":"2024-09-13T14:42:19.552464+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:00.426042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":131,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.572574+0000","last_change":"2024-09-13T14:38:13.979255+0000","last_active":"2024-09-13T14:42:19.572574+0000","last_peered":"2024-09-13T14:42:19.572574+0000","last_clean":"2024-09-13T14:42:19.572574+0000","last_became_active":"2024-09-13T14:38:13.978868+0000","last_became_peered":"2024-09-13T14:38:13.978868+0000","last_unstale":"2024-09-13T14:42:19.572574+0000","last_undegraded":"2024-09-13T14:42:19.572574+0000","last_fullsized":"2024-09-13T14:42:19.572574+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:15:26.003538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":60,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.558774+0000","last_change":"2024-09-13T14:40:24.124750+0000","last_active":"2024-09-13T14:42:19.558774+0000","last_peered":"2024-09-13T14:42:19.558774+0000","last_clean":"2024-09-13T14:42:19.558774+0000","last_became_active":"2024-09-13T14:40:22.958820+0000","last_became_peered":"2024-09-13T14:40:22.958820+0000","last_unstale":"2024-09-13T14:42:19.558774+0000","last_undegraded":"2024-09-13T14:42:19.558774+0000","last_fullsized":"2024-09-13T14:42:19.558774+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:24.124531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:24.124531+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:36:35.757492+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"136'12055","reported_seq":12122,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:26.707055+0000","last_change":"2024-09-13T14:39:42.770912+0000","last_active":"2024-09-13T14:42:26.707055+0000","last_peered":"2024-09-13T14:42:26.707055+0000","last_clean":"2024-09-13T14:42:26.707055+0000","last_became_active":"2024-09-13T14:39:42.770646+0000","last_became_peered":"2024-09-13T14:39:42.770646+0000","last_unstale":"2024-09-13T14:42:26.707055+0000","last_undegraded":"2024-09-13T14:42:26.707055+0000","last_fullsized":"2024-09-13T14:42:26.707055+0000","mapping_epoch":107,"log_start":"120'5200","ondisk_log_start":"120'5200","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6855,"log_dups_size":0,"ondisk_log_size":6855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:54:13.100820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281083904,"num_objects":4289,"num_object_clones":0,"num_object_copies":12867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12055,"num_write_kb":523008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":142,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.564432+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:42:19.564432+0000","last_peered":"2024-09-13T14:42:19.564432+0000","last_clean":"2024-09-13T14:42:19.564432+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:42:19.564432+0000","last_undegraded":"2024-09-13T14:42:19.564432+0000","last_fullsized":"2024-09-13T14:42:19.564432+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1053,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312601+0000","last_change":"2024-09-13T14:39:44.990389+0000","last_active":"2024-09-13T14:42:09.312601+0000","last_peered":"2024-09-13T14:42:09.312601+0000","last_clean":"2024-09-13T14:42:09.312601+0000","last_became_active":"2024-09-13T14:39:44.989805+0000","last_became_peered":"2024-09-13T14:39:44.989805+0000","last_unstale":"2024-09-13T14:42:09.312601+0000","last_undegraded":"2024-09-13T14:42:09.312601+0000","last_fullsized":"2024-09-13T14:42:09.312601+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:15.392445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":93,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.315004+0000","last_change":"2024-09-13T14:39:42.769954+0000","last_active":"2024-09-13T14:42:09.315004+0000","last_peered":"2024-09-13T14:42:09.315004+0000","last_clean":"2024-09-13T14:42:09.315004+0000","last_became_active":"2024-09-13T14:39:42.769635+0000","last_became_peered":"2024-09-13T14:39:42.769635+0000","last_unstale":"2024-09-13T14:42:09.315004+0000","last_undegraded":"2024-09-13T14:42:09.315004+0000","last_fullsized":"2024-09-13T14:42:09.315004+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:51.588071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":110,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.313936+0000","last_change":"2024-09-13T14:39:42.775357+0000","last_active":"2024-09-13T14:42:09.313936+0000","last_peered":"2024-09-13T14:42:09.313936+0000","last_clean":"2024-09-13T14:42:09.313936+0000","last_became_active":"2024-09-13T14:39:42.775041+0000","last_became_peered":"2024-09-13T14:39:42.775041+0000","last_unstale":"2024-09-13T14:42:09.313936+0000","last_undegraded":"2024-09-13T14:42:09.313936+0000","last_fullsized":"2024-09-13T14:42:09.313936+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:44.768910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":132,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.561069+0000","last_change":"2024-09-13T14:39:42.787151+0000","last_active":"2024-09-13T14:42:19.561069+0000","last_peered":"2024-09-13T14:42:19.561069+0000","last_clean":"2024-09-13T14:42:19.561069+0000","last_became_active":"2024-09-13T14:39:42.771712+0000","last_became_peered":"2024-09-13T14:39:42.771712+0000","last_unstale":"2024-09-13T14:42:19.561069+0000","last_undegraded":"2024-09-13T14:42:19.561069+0000","last_fullsized":"2024-09-13T14:42:19.561069+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:54.216663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":125,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.546156+0000","last_change":"2024-09-13T14:39:42.772974+0000","last_active":"2024-09-13T14:42:19.546156+0000","last_peered":"2024-09-13T14:42:19.546156+0000","last_clean":"2024-09-13T14:42:19.546156+0000","last_became_active":"2024-09-13T14:39:42.771449+0000","last_became_peered":"2024-09-13T14:39:42.771449+0000","last_unstale":"2024-09-13T14:42:19.546156+0000","last_undegraded":"2024-09-13T14:42:19.546156+0000","last_fullsized":"2024-09-13T14:42:19.546156+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:46.441079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":116,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.312430+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:42:09.312430+0000","last_peered":"2024-09-13T14:42:09.312430+0000","last_clean":"2024-09-13T14:42:09.312430+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:42:09.312430+0000","last_undegraded":"2024-09-13T14:42:09.312430+0000","last_fullsized":"2024-09-13T14:42:09.312430+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":101,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.318390+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:42:09.318390+0000","last_peered":"2024-09-13T14:42:09.318390+0000","last_clean":"2024-09-13T14:42:09.318390+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:42:09.318390+0000","last_undegraded":"2024-09-13T14:42:09.318390+0000","last_fullsized":"2024-09-13T14:42:09.318390+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":94,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.560487+0000","last_change":"2024-09-13T14:39:42.767764+0000","last_active":"2024-09-13T14:42:19.560487+0000","last_peered":"2024-09-13T14:42:19.560487+0000","last_clean":"2024-09-13T14:42:19.560487+0000","last_became_active":"2024-09-13T14:39:42.767450+0000","last_became_peered":"2024-09-13T14:39:42.767450+0000","last_unstale":"2024-09-13T14:42:19.560487+0000","last_undegraded":"2024-09-13T14:42:19.560487+0000","last_fullsized":"2024-09-13T14:42:19.560487+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:47.729382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":129,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.563893+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:42:19.563893+0000","last_peered":"2024-09-13T14:42:19.563893+0000","last_clean":"2024-09-13T14:42:19.563893+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:42:19.563893+0000","last_undegraded":"2024-09-13T14:42:19.563893+0000","last_fullsized":"2024-09-13T14:42:19.563893+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":143,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.573049+0000","last_change":"2024-09-13T14:39:42.772266+0000","last_active":"2024-09-13T14:42:19.573049+0000","last_peered":"2024-09-13T14:42:19.573049+0000","last_clean":"2024-09-13T14:42:19.573049+0000","last_became_active":"2024-09-13T14:39:42.771880+0000","last_became_peered":"2024-09-13T14:39:42.771880+0000","last_unstale":"2024-09-13T14:42:19.573049+0000","last_undegraded":"2024-09-13T14:42:19.573049+0000","last_fullsized":"2024-09-13T14:42:19.573049+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:41:01.160105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":91,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.318631+0000","last_change":"2024-09-13T14:39:42.773890+0000","last_active":"2024-09-13T14:42:09.318631+0000","last_peered":"2024-09-13T14:42:09.318631+0000","last_clean":"2024-09-13T14:42:09.318631+0000","last_became_active":"2024-09-13T14:39:42.773523+0000","last_became_peered":"2024-09-13T14:39:42.773523+0000","last_unstale":"2024-09-13T14:42:09.318631+0000","last_undegraded":"2024-09-13T14:42:09.318631+0000","last_fullsized":"2024-09-13T14:42:09.318631+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:13.156261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":154,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.572081+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:42:19.572081+0000","last_peered":"2024-09-13T14:42:19.572081+0000","last_clean":"2024-09-13T14:42:19.572081+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:42:19.572081+0000","last_undegraded":"2024-09-13T14:42:19.572081+0000","last_fullsized":"2024-09-13T14:42:19.572081+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":131,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.564614+0000","last_change":"2024-09-13T14:38:13.979604+0000","last_active":"2024-09-13T14:42:19.564614+0000","last_peered":"2024-09-13T14:42:19.564614+0000","last_clean":"2024-09-13T14:42:19.564614+0000","last_became_active":"2024-09-13T14:38:13.979288+0000","last_became_peered":"2024-09-13T14:38:13.979288+0000","last_unstale":"2024-09-13T14:42:19.564614+0000","last_undegraded":"2024-09-13T14:42:19.564614+0000","last_fullsized":"2024-09-13T14:42:19.564614+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:21.031527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":118,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.551224+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:42:19.551224+0000","last_peered":"2024-09-13T14:42:19.551224+0000","last_clean":"2024-09-13T14:42:19.551224+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:42:19.551224+0000","last_undegraded":"2024-09-13T14:42:19.551224+0000","last_fullsized":"2024-09-13T14:42:19.551224+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":150,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.563148+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:42:19.563148+0000","last_peered":"2024-09-13T14:42:19.563148+0000","last_clean":"2024-09-13T14:42:19.563148+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:42:19.563148+0000","last_undegraded":"2024-09-13T14:42:19.563148+0000","last_fullsized":"2024-09-13T14:42:19.563148+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":119,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.314344+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:42:09.314344+0000","last_peered":"2024-09-13T14:42:09.314344+0000","last_clean":"2024-09-13T14:42:09.314344+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:42:09.314344+0000","last_undegraded":"2024-09-13T14:42:09.314344+0000","last_fullsized":"2024-09-13T14:42:09.314344+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":156,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.555030+0000","last_change":"2024-09-13T14:38:13.976950+0000","last_active":"2024-09-13T14:42:19.555030+0000","last_peered":"2024-09-13T14:42:19.555030+0000","last_clean":"2024-09-13T14:42:19.555030+0000","last_became_active":"2024-09-13T14:38:13.976639+0000","last_became_peered":"2024-09-13T14:38:13.976639+0000","last_unstale":"2024-09-13T14:42:19.555030+0000","last_undegraded":"2024-09-13T14:42:19.555030+0000","last_fullsized":"2024-09-13T14:42:19.555030+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:24.126890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":122,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.557069+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:42:19.557069+0000","last_peered":"2024-09-13T14:42:19.557069+0000","last_clean":"2024-09-13T14:42:19.557069+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:42:19.557069+0000","last_undegraded":"2024-09-13T14:42:19.557069+0000","last_fullsized":"2024-09-13T14:42:19.557069+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":117,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.314058+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:42:09.314058+0000","last_peered":"2024-09-13T14:42:09.314058+0000","last_clean":"2024-09-13T14:42:09.314058+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:42:09.314058+0000","last_undegraded":"2024-09-13T14:42:09.314058+0000","last_fullsized":"2024-09-13T14:42:09.314058+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":103,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.316839+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:42:09.316839+0000","last_peered":"2024-09-13T14:42:09.316839+0000","last_clean":"2024-09-13T14:42:09.316839+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:42:09.316839+0000","last_undegraded":"2024-09-13T14:42:09.316839+0000","last_fullsized":"2024-09-13T14:42:09.316839+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":117,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.314270+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:42:09.314270+0000","last_peered":"2024-09-13T14:42:09.314270+0000","last_clean":"2024-09-13T14:42:09.314270+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:42:09.314270+0000","last_undegraded":"2024-09-13T14:42:09.314270+0000","last_fullsized":"2024-09-13T14:42:09.314270+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":128,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.317203+0000","last_change":"2024-09-13T14:39:42.778413+0000","last_active":"2024-09-13T14:42:09.317203+0000","last_peered":"2024-09-13T14:42:09.317203+0000","last_clean":"2024-09-13T14:42:09.317203+0000","last_became_active":"2024-09-13T14:39:42.766624+0000","last_became_peered":"2024-09-13T14:39:42.766624+0000","last_unstale":"2024-09-13T14:42:09.317203+0000","last_undegraded":"2024-09-13T14:42:09.317203+0000","last_fullsized":"2024-09-13T14:42:09.317203+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:13.435604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.316497+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:42:09.316497+0000","last_peered":"2024-09-13T14:42:09.316497+0000","last_clean":"2024-09-13T14:42:09.316497+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:42:09.316497+0000","last_undegraded":"2024-09-13T14:42:09.316497+0000","last_fullsized":"2024-09-13T14:42:09.316497+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":150,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.559324+0000","last_change":"2024-09-13T14:39:42.767451+0000","last_active":"2024-09-13T14:42:19.559324+0000","last_peered":"2024-09-13T14:42:19.559324+0000","last_clean":"2024-09-13T14:42:19.559324+0000","last_became_active":"2024-09-13T14:39:42.767086+0000","last_became_peered":"2024-09-13T14:39:42.767086+0000","last_unstale":"2024-09-13T14:42:19.559324+0000","last_undegraded":"2024-09-13T14:42:19.559324+0000","last_fullsized":"2024-09-13T14:42:19.559324+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:02.674069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":103,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-09-13T14:42:09.313721+0000","last_change":"2024-09-13T14:39:42.768653+0000","last_active":"2024-09-13T14:42:09.313721+0000","last_peered":"2024-09-13T14:42:09.313721+0000","last_clean":"2024-09-13T14:42:09.313721+0000","last_became_active":"2024-09-13T14:39:42.768333+0000","last_became_peered":"2024-09-13T14:39:42.768333+0000","last_unstale":"2024-09-13T14:42:09.313721+0000","last_undegraded":"2024-09-13T14:42:09.313721+0000","last_fullsized":"2024-09-13T14:42:09.313721+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:25.655793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":134,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:42:19.563805+0000","last_change":"2024-09-13T14:37:40.924860+0000","last_active":"2024-09-13T14:42:19.563805+0000","last_peered":"2024-09-13T14:42:19.563805+0000","last_clean":"2024-09-13T14:42:19.563805+0000","last_became_active":"2024-09-13T14:37:40.924522+0000","last_became_peered":"2024-09-13T14:37:40.924522+0000","last_unstale":"2024-09-13T14:42:19.563805+0000","last_undegraded":"2024-09-13T14:42:19.563805+0000","last_fullsized":"2024-09-13T14:42:19.563805+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:10.340511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":4370137112,"num_objects":66684,"num_object_clones":0,"num_object_copies":200052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66684,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":193704,"num_write_kb":8332353,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13172617216,"data_stored":13172605000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117404,"ondisk_log_size":117404,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":9648,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397702,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1667780,"kb_used_data":1366268,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92077372,"statfs":{"total":95995035648,"available":94287228928,"internally_reserved":0,"allocated":1399058432,"data_stored":1398249284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":184683593931,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790160,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2744888,"kb_used_data":2439152,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91000264,"statfs":{"total":95995035648,"available":93184270336,"internally_reserved":0,"allocated":2497691648,"data_stored":2496890801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986389,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2504884,"kb_used_data":2199212,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91240268,"statfs":{"total":95995035648,"available":93430034432,"internally_reserved":0,"allocated":2251993088,"data_stored":2251195500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598837,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1182572,"kb_used_data":1114788,"kb_used_omap":6,"kb_used_meta":67769,"kb_avail":92562580,"statfs":{"total":95995035648,"available":94784081920,"internally_reserved":0,"allocated":1141542912,"data_stored":1140753768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":69396448},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762364,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2257668,"kb_used_data":2113468,"kb_used_omap":8,"kb_used_meta":144119,"kb_avail":91487484,"statfs":{"total":95995035648,"available":93683183616,"internally_reserved":0,"allocated":2164191232,"data_stored":2163382836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":147578314},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":94,"seq":403726925890,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1740972,"kb_used_data":1632868,"kb_used_omap":7,"kb_used_meta":108024,"kb_avail":92004180,"statfs":{"total":95995035648,"available":94212280320,"internally_reserved":0,"allocated":1672056832,"data_stored":1671267337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":110617289},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122130,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2249924,"kb_used_data":2105788,"kb_used_omap":7,"kb_used_meta":144120,"kb_avail":91495228,"statfs":{"total":95995035648,"available":93691113472,"internally_reserved":0,"allocated":2156326912,"data_stored":2155518267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":147579471},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2140274688,"data_stored":2140274688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1644691456,"data_stored":1644691456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2138116096,"data_stored":2138112024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1125122048,"data_stored":1125122048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2247688192,"data_stored":2247688192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2481721344,"data_stored":2481717272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1395003392,"data_stored":1394999320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:42:31.761 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T14:42:31.761 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:42:31.762 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-09-13T14:42:31.762 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:42:31.762 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:42:31.762 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-09-13T14:42:31.829 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:31.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.872 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:31.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:31.973 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:32.010 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:32.013 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:32.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:32.022 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:42:32.023 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:42:32.068 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:42:32.068 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:42:32.110 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:42:32.110 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:42:32.114 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:32.131 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:42:32.142 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:42:32.142 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:42:32.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.154 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:32.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:32.173 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:42:32.173 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:42:32.202 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:42:32.202 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:42:32.233 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:42:32.233 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:42:32.255 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:32.264 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:42:32.265 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:42:32.281 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:32.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:32.383 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:32.415 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:32.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:32.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:32.544 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:32 smithi204 ceph-mon[87071]: from='client.35211 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:42:32.545 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:32 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/178990663' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:32.550 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:32.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:32.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:32 smithi096 ceph-mon[98901]: from='client.35211 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:42:32.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:32 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/178990663' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:32.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:32 smithi096 ceph-mon[102928]: from='client.35211 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:42:32.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:32 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/178990663' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:32.652 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:32.693 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:32.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:32.794 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:32.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.830 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:32.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:32.931 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:32.965 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:32.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:33.066 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:33.098 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:33.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:33.200 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:33.233 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:33.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:33.335 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:33.370 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:33.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:33.427 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:33.472 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:33.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.513 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:33.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:33.614 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:33.655 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:33.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:33.679 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:33 smithi204 ceph-mon[87071]: pgmap v355: 99 pgs: 99 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 859 op/s 2024-09-13T14:42:33.757 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:33.788 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:33.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:33 smithi096 ceph-mon[98901]: pgmap v355: 99 pgs: 99 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 859 op/s 2024-09-13T14:42:33.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:33 smithi096 ceph-mon[102928]: pgmap v355: 99 pgs: 99 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 859 op/s 2024-09-13T14:42:33.890 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:33.921 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:33.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:33.978 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:42:33.979 INFO:teuthology.orchestra.run.smithi204.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:42:33.979 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:42:34.023 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:34.061 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:34.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:34.163 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:34.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.192 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:34.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:34.293 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:34.322 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:34.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:34.424 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:34.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.465 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:34.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:34.566 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:34.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.595 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:34.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:34.696 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:34.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.731 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:34.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:34.832 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:34.864 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:34.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:34.966 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:35.000 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:35.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:35.102 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:35.134 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:35.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:35.236 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:35.275 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:35.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:35.377 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:35.414 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:35.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:35.435 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:42:35.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:35.543 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:35.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:35.646 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:35.680 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:35.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:35.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:35 smithi204 ceph-mon[87071]: pgmap v356: 99 pgs: 99 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:42:35.782 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:35.815 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:35.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:35.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:35 smithi096 ceph-mon[102928]: pgmap v356: 99 pgs: 99 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:42:35.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:35 smithi096 ceph-mon[98901]: pgmap v356: 99 pgs: 99 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:42:35.917 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:35.953 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:35.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:36.055 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:36.097 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:36.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:36.199 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:36.225 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:36.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:36.327 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:36.354 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:36.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:36.456 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:36.496 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:36.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:36.584 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:36.584 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:36.598 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:36.644 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:36.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:36.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:36.746 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:36.770 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:36.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:36.872 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:36.898 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:36.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:36.999 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:37.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.043 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:37.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:37.065 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:42:37.065 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:42:37.065 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:42:37.065 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:42:37.065 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:42:37.145 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:37.182 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:37.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:37.284 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:37.310 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:37.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:37.412 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:37.432 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:42:37.439 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:37.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:37.541 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:37.569 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:37.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:37.672 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:37.715 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:37.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:37.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:37 smithi204 ceph-mon[87071]: pgmap v357: 99 pgs: 99 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 955 op/s 2024-09-13T14:42:37.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:37 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2060282746' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:42:37.728 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:37 smithi096 ceph-mon[98901]: pgmap v357: 99 pgs: 99 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 955 op/s 2024-09-13T14:42:37.728 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:37 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2060282746' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:42:37.728 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:37 smithi096 ceph-mon[102928]: pgmap v357: 99 pgs: 99 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 955 op/s 2024-09-13T14:42:37.728 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:37 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2060282746' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:42:37.768 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:42:38.145 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:38.147 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:42:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:42:37] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.51.0" 2024-09-13T14:42:38.187 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:38.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:38.289 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:38.329 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:38.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:38.431 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:38.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.480 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:38.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:38.581 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:38.614 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:38.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:38.689 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:38.716 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:38.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:38 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:38.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:38 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2060282746' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:42:38.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:38 smithi204 ceph-mon[87071]: osdmap e137: 8 total, 8 up, 7 in 2024-09-13T14:42:38.742 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:38.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:38.764 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:38.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:38 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:38.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:38 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2060282746' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:42:38.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:38 smithi096 ceph-mon[98901]: osdmap e137: 8 total, 8 up, 7 in 2024-09-13T14:42:38.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:38 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:38.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:38 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2060282746' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:42:38.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:38 smithi096 ceph-mon[102928]: osdmap e137: 8 total, 8 up, 7 in 2024-09-13T14:42:38.844 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:38.873 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:38.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:38.974 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:39.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.000 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:39.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:39.102 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:39.131 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:39.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:39.233 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:39.260 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:39.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:39.362 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:39.393 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:39.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:39.495 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:39.528 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:39.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:39.629 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:39.655 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:39.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:39.662 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:39 smithi096 ceph-mon[98901]: pgmap v359: 99 pgs: 99 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 994 op/s 2024-09-13T14:42:39.662 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:39 smithi096 ceph-mon[102928]: pgmap v359: 99 pgs: 99 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 994 op/s 2024-09-13T14:42:39.718 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:42:39.718 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:42:39.719 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-09-13T14:42:39.719 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:42:39.719 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:42:39.719 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-09-13T14:42:39.757 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:39.782 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:39.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:39.884 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:39.912 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:39.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:39.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:39 smithi204 ceph-mon[87071]: pgmap v359: 99 pgs: 99 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 994 op/s 2024-09-13T14:42:40.015 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:40.054 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:40.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:40.156 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:40.187 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:40.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:40.289 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:40.316 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:40.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:40.418 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:40.452 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:40.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:40.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:42:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:42:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:40.554 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:40.588 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:40.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:40.658 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:42:40.689 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:40.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.761 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:40.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:40.862 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:40.922 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:40.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:41.024 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:41.081 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:41.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:41.183 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:41.229 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:41.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:41.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:41.365 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:41.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:41.467 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:41.519 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:41.530 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:41.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:41.575 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:41.632 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:41.682 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:41.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:41.784 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:41.790 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:41 smithi096 ceph-mon[98901]: pgmap v360: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 976 op/s 2024-09-13T14:42:41.791 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:41 smithi096 ceph-mon[102928]: pgmap v360: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 976 op/s 2024-09-13T14:42:41.837 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:41.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:41.941 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:41.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:41 smithi204 ceph-mon[87071]: pgmap v360: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 976 op/s 2024-09-13T14:42:41.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.976 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:41.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:42.077 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:42.105 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:42.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:42.207 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:42.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.249 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:42.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:42.350 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:42.393 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:42.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:42.495 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:42.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.526 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:42.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:42.627 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:42.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.666 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:42.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:42.768 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:42.804 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:42.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:42.906 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:42.936 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:42.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:43.038 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:43.081 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:43.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:43.148 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:42:43.149 INFO:teuthology.orchestra.run.smithi096.stdout: "size": 20, 2024-09-13T14:42:43.149 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 600, 2024-09-13T14:42:43.149 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:42:43.149 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.149 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:67766 5.3 5:c1b2f022:::benchmark_data_smithi096_138841_object67765:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:42:43.149 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.201070+0000", 2024-09-13T14:42:43.149 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 137.944673524, 2024-09-13T14:42:43.149 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.13832765799999999, 2024-09-13T14:42:43.149 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.149 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.149 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.149 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:42:43.150 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:42:43.150 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:42:43.150 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 67766 2024-09-13T14:42:43.150 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.150 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.150 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.150 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.150 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201070+0000", 2024-09-13T14:42:43.150 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.150 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.150 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.150 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.151 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201070+0000", 2024-09-13T14:42:43.151 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.151 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.151 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.151 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.151 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201071+0000", 2024-09-13T14:42:43.151 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.051e-06 2024-09-13T14:42:43.151 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.151 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.151 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.151 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201091+0000", 2024-09-13T14:42:43.151 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0605999999999999e-05 2024-09-13T14:42:43.151 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.152 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.152 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.152 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201103+0000", 2024-09-13T14:42:43.152 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1843e-05 2024-09-13T14:42:43.152 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.152 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.152 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.152 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201110+0000", 2024-09-13T14:42:43.152 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1980000000000004e-06 2024-09-13T14:42:43.152 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.152 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.152 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.153 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.322156+0000", 2024-09-13T14:42:43.153 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.12104585700000001 2024-09-13T14:42:43.153 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.153 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.153 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.153 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.322296+0000", 2024-09-13T14:42:43.153 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000140419 2024-09-13T14:42:43.153 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.153 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.153 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 3,5", 2024-09-13T14:42:43.153 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.322442+0000", 2024-09-13T14:42:43.153 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000145084 2024-09-13T14:42:43.153 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.154 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.154 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.154 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.324985+0000", 2024-09-13T14:42:43.154 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0025436059999999999 2024-09-13T14:42:43.154 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.154 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.154 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.154 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.330119+0000", 2024-09-13T14:42:43.154 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0051342039999999999 2024-09-13T14:42:43.154 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.154 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.154 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:42:43.154 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.339248+0000", 2024-09-13T14:42:43.155 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0091287810000000007 2024-09-13T14:42:43.155 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.155 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.155 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.155 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.339344+0000", 2024-09-13T14:42:43.155 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.5706999999999997e-05 2024-09-13T14:42:43.155 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.155 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.155 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.155 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.339397+0000", 2024-09-13T14:42:43.155 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3301999999999999e-05 2024-09-13T14:42:43.155 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.155 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.156 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.156 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.156 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.156 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:68079 5.3 5:cc82a13d:::benchmark_data_smithi096_138841_object68078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:42:43.156 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.517423+0000", 2024-09-13T14:42:43.156 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 137.62832015199999, 2024-09-13T14:42:43.156 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16008386699999999, 2024-09-13T14:42:43.156 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.156 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.156 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.156 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:42:43.156 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:42:43.156 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:42:43.157 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 68079 2024-09-13T14:42:43.157 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.157 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.157 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.157 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.157 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517423+0000", 2024-09-13T14:42:43.157 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.157 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.157 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.157 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.157 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517423+0000", 2024-09-13T14:42:43.157 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.157 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.157 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.158 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.158 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517424+0000", 2024-09-13T14:42:43.158 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1540000000000001e-06 2024-09-13T14:42:43.158 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.158 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.158 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.158 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517443+0000", 2024-09-13T14:42:43.158 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8576e-05 2024-09-13T14:42:43.158 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.158 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.158 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.158 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517452+0000", 2024-09-13T14:42:43.158 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.4329999999999993e-06 2024-09-13T14:42:43.159 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.159 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.159 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.159 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517458+0000", 2024-09-13T14:42:43.159 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9030000000000001e-06 2024-09-13T14:42:43.159 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.159 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.159 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.159 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.518942+0000", 2024-09-13T14:42:43.159 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0014841469999999999 2024-09-13T14:42:43.159 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.159 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.159 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.159 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519105+0000", 2024-09-13T14:42:43.160 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000162495 2024-09-13T14:42:43.160 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.160 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.160 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 3,5", 2024-09-13T14:42:43.160 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519231+0000", 2024-09-13T14:42:43.160 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012689 2024-09-13T14:42:43.160 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.160 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.160 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.160 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.521772+0000", 2024-09-13T14:42:43.160 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.002540212 2024-09-13T14:42:43.160 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.160 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.161 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:42:43.161 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.521866+0000", 2024-09-13T14:42:43.161 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3863999999999994e-05 2024-09-13T14:42:43.161 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.161 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.161 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.161 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677396+0000", 2024-09-13T14:42:43.161 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.155530265 2024-09-13T14:42:43.161 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.161 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.161 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.161 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677458+0000", 2024-09-13T14:42:43.161 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1790000000000003e-05 2024-09-13T14:42:43.161 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.162 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.162 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.162 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677507+0000", 2024-09-13T14:42:43.162 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9138000000000003e-05 2024-09-13T14:42:43.162 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.162 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.162 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.162 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.162 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.162 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:68081 5.5 5:a2088a5b:::benchmark_data_smithi096_138841_object68080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:42:43.162 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.517472+0000", 2024-09-13T14:42:43.162 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 137.62827070500001, 2024-09-13T14:42:43.162 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.164906097, 2024-09-13T14:42:43.163 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.163 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.163 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.163 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:42:43.163 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:42:43.164 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:42:43.164 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 68081 2024-09-13T14:42:43.164 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.164 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.164 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.164 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.164 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517472+0000", 2024-09-13T14:42:43.164 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.164 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.164 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.164 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.164 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517472+0000", 2024-09-13T14:42:43.165 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.165 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.165 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.165 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.165 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517473+0000", 2024-09-13T14:42:43.165 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7300000000000001e-07 2024-09-13T14:42:43.165 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.165 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.165 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.165 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517489+0000", 2024-09-13T14:42:43.165 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6595999999999998e-05 2024-09-13T14:42:43.165 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.165 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.166 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.166 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517496+0000", 2024-09-13T14:42:43.166 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6590000000000001e-06 2024-09-13T14:42:43.166 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.166 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.166 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.166 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517501+0000", 2024-09-13T14:42:43.166 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8110000000000002e-06 2024-09-13T14:42:43.166 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.166 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.166 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.166 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517713+0000", 2024-09-13T14:42:43.166 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00021189200000000001 2024-09-13T14:42:43.167 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.167 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.167 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.167 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517828+0000", 2024-09-13T14:42:43.167 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000115171 2024-09-13T14:42:43.167 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.167 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.167 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:42:43.167 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517950+0000", 2024-09-13T14:42:43.167 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012224499999999999 2024-09-13T14:42:43.167 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.167 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.167 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:42:43.168 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.681738+0000", 2024-09-13T14:42:43.168 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.163787502 2024-09-13T14:42:43.168 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.168 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.168 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.168 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.681777+0000", 2024-09-13T14:42:43.168 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9407999999999999e-05 2024-09-13T14:42:43.168 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.168 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.168 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.168 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.682179+0000", 2024-09-13T14:42:43.168 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00040160599999999999 2024-09-13T14:42:43.168 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.169 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.169 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.169 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.682331+0000", 2024-09-13T14:42:43.169 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000152631 2024-09-13T14:42:43.169 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.169 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.169 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.169 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.682378+0000", 2024-09-13T14:42:43.169 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7102999999999999e-05 2024-09-13T14:42:43.169 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.169 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.169 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.169 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.169 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.170 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:68082 5.5 5:ab56da9e:::benchmark_data_smithi096_138841_object68081:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:42:43.170 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.517514+0000", 2024-09-13T14:42:43.170 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 137.62822876999999, 2024-09-13T14:42:43.170 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16716524299999999, 2024-09-13T14:42:43.170 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.170 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.170 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.170 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:42:43.170 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:42:43.170 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:42:43.170 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 68082 2024-09-13T14:42:43.170 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.170 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.171 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.171 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.171 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517514+0000", 2024-09-13T14:42:43.171 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.171 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.171 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.171 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.171 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517514+0000", 2024-09-13T14:42:43.171 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.171 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.171 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.171 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.171 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517515+0000", 2024-09-13T14:42:43.172 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9500000000000002e-07 2024-09-13T14:42:43.172 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.172 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.172 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.172 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517532+0000", 2024-09-13T14:42:43.172 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6756e-05 2024-09-13T14:42:43.172 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.172 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.172 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.172 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517538+0000", 2024-09-13T14:42:43.172 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2729999999999999e-06 2024-09-13T14:42:43.172 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.172 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.172 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.173 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517543+0000", 2024-09-13T14:42:43.173 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6319999999999999e-06 2024-09-13T14:42:43.173 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.173 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.173 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.173 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522076+0000", 2024-09-13T14:42:43.173 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0045333719999999999 2024-09-13T14:42:43.173 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.173 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.173 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.173 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677838+0000", 2024-09-13T14:42:43.173 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15576197 2024-09-13T14:42:43.173 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.173 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.174 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:42:43.174 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677976+0000", 2024-09-13T14:42:43.174 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013789 2024-09-13T14:42:43.174 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.174 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.174 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:42:43.174 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.681787+0000", 2024-09-13T14:42:43.174 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0038115979999999998 2024-09-13T14:42:43.174 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.174 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.174 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.174 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.682419+0000", 2024-09-13T14:42:43.174 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000631541 2024-09-13T14:42:43.175 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.175 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.175 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.175 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684445+0000", 2024-09-13T14:42:43.175 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0020261459999999999 2024-09-13T14:42:43.175 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.175 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.175 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.175 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684625+0000", 2024-09-13T14:42:43.175 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00017956600000000001 2024-09-13T14:42:43.175 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.175 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.175 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.175 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684680+0000", 2024-09-13T14:42:43.176 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4904000000000002e-05 2024-09-13T14:42:43.176 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.176 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.176 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.176 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.176 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.176 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:68084 5.2 5:41fe4da9:::benchmark_data_smithi096_138841_object68083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:42:43.176 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.519853+0000", 2024-09-13T14:42:43.176 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 137.625890535, 2024-09-13T14:42:43.176 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.18042530400000001, 2024-09-13T14:42:43.176 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.176 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.176 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.177 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:42:43.177 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:42:43.177 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:42:43.177 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 68084 2024-09-13T14:42:43.177 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.177 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.177 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.177 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.177 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519853+0000", 2024-09-13T14:42:43.177 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.177 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.177 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.177 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.177 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519853+0000", 2024-09-13T14:42:43.178 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.178 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.178 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.178 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.178 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519854+0000", 2024-09-13T14:42:43.178 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.446e-06 2024-09-13T14:42:43.178 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.178 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.178 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.178 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519875+0000", 2024-09-13T14:42:43.178 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0798999999999999e-05 2024-09-13T14:42:43.178 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.178 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.179 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.179 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519887+0000", 2024-09-13T14:42:43.179 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1921e-05 2024-09-13T14:42:43.179 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.179 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.179 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.179 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519905+0000", 2024-09-13T14:42:43.179 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7994000000000001e-05 2024-09-13T14:42:43.179 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.179 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.179 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.179 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520192+0000", 2024-09-13T14:42:43.179 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00028749299999999999 2024-09-13T14:42:43.179 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.180 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.180 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.180 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520321+0000", 2024-09-13T14:42:43.180 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012930799999999999 2024-09-13T14:42:43.180 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.180 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.180 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:42:43.180 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520447+0000", 2024-09-13T14:42:43.180 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012507100000000001 2024-09-13T14:42:43.180 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.180 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.180 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.180 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.680495+0000", 2024-09-13T14:42:43.181 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16004827699999999 2024-09-13T14:42:43.181 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.181 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.181 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:42:43.181 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684061+0000", 2024-09-13T14:42:43.181 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0035666180000000001 2024-09-13T14:42:43.181 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.181 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.181 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.181 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.700185+0000", 2024-09-13T14:42:43.181 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.016123723 2024-09-13T14:42:43.181 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.181 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.182 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.182 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.700239+0000", 2024-09-13T14:42:43.182 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3553e-05 2024-09-13T14:42:43.182 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.182 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.182 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.182 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.700278+0000", 2024-09-13T14:42:43.182 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9100999999999997e-05 2024-09-13T14:42:43.182 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.182 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.182 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.182 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.182 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.182 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:68087 5.2 5:44b2e474:::benchmark_data_smithi096_138841_object68086:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:42:43.183 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.519923+0000", 2024-09-13T14:42:43.183 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 137.625820221, 2024-09-13T14:42:43.183 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.18477276300000001, 2024-09-13T14:42:43.183 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.183 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.183 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.183 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:42:43.183 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:42:43.183 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:42:43.183 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 68087 2024-09-13T14:42:43.183 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.183 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.183 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.184 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.184 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519923+0000", 2024-09-13T14:42:43.184 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.184 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.184 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.184 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.184 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519923+0000", 2024-09-13T14:42:43.184 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.184 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.184 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.184 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.184 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519923+0000", 2024-09-13T14:42:43.184 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3099999999999998e-07 2024-09-13T14:42:43.185 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.185 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.185 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.185 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519940+0000", 2024-09-13T14:42:43.185 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6365e-05 2024-09-13T14:42:43.185 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.185 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.185 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.185 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519947+0000", 2024-09-13T14:42:43.185 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9489999999999999e-06 2024-09-13T14:42:43.185 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.185 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.185 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.186 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519951+0000", 2024-09-13T14:42:43.186 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7840000000000003e-06 2024-09-13T14:42:43.186 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.186 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.186 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.186 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677824+0000", 2024-09-13T14:42:43.186 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15787304099999999 2024-09-13T14:42:43.186 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.186 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.186 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.186 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677959+0000", 2024-09-13T14:42:43.186 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000134148 2024-09-13T14:42:43.186 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.186 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.187 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:42:43.187 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.678095+0000", 2024-09-13T14:42:43.187 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000136837 2024-09-13T14:42:43.187 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.187 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.187 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.187 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684103+0000", 2024-09-13T14:42:43.187 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0060078400000000004 2024-09-13T14:42:43.187 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.187 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.187 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:42:43.187 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684196+0000", 2024-09-13T14:42:43.187 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2334000000000001e-05 2024-09-13T14:42:43.188 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.188 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.188 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.188 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.704598+0000", 2024-09-13T14:42:43.188 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.020402453000000001 2024-09-13T14:42:43.188 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.188 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.188 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.188 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.704646+0000", 2024-09-13T14:42:43.188 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7812000000000002e-05 2024-09-13T14:42:43.188 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.188 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.188 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.189 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.704696+0000", 2024-09-13T14:42:43.189 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9669000000000002e-05 2024-09-13T14:42:43.189 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.189 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.189 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.189 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.189 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.189 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.35049.0:68081 5.5 e119/107 ondisk, result = 0)", 2024-09-13T14:42:43.189 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.520949+0000", 2024-09-13T14:42:43.189 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 137.62479389000001, 2024-09-13T14:42:43.189 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.160838853, 2024-09-13T14:42:43.189 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.189 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.190 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:42:43.190 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.190 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.190 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.190 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520949+0000", 2024-09-13T14:42:43.190 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.190 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.190 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.190 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.190 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520949+0000", 2024-09-13T14:42:43.190 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.190 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.190 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.191 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.191 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520950+0000", 2024-09-13T14:42:43.191 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9699999999999995e-07 2024-09-13T14:42:43.191 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.191 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.191 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.191 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520953+0000", 2024-09-13T14:42:43.191 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7989999999999998e-06 2024-09-13T14:42:43.191 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.191 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.191 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.191 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520962+0000", 2024-09-13T14:42:43.191 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0659999999999993e-06 2024-09-13T14:42:43.191 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.192 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.192 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.192 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520968+0000", 2024-09-13T14:42:43.192 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0750000000000003e-06 2024-09-13T14:42:43.192 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.192 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.192 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.192 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.681765+0000", 2024-09-13T14:42:43.192 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:43.193 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16079722900000001 2024-09-13T14:42:43.193 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.194 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.194 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.194 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.681772+0000", 2024-09-13T14:42:43.194 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8090000000000004e-06 2024-09-13T14:42:43.194 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.194 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.194 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.194 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.681788+0000", 2024-09-13T14:42:43.194 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6078000000000001e-05 2024-09-13T14:42:43.194 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.194 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.194 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.194 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.195 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.195 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.35049.0:68079 5.3 e119/107 ondisk, result = 0)", 2024-09-13T14:42:43.195 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.522506+0000", 2024-09-13T14:42:43.195 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 137.62323714199999, 2024-09-13T14:42:43.195 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15500866399999999, 2024-09-13T14:42:43.195 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.195 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.195 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:42:43.195 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.195 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.195 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.195 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522506+0000", 2024-09-13T14:42:43.195 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.196 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.196 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.196 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.196 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522506+0000", 2024-09-13T14:42:43.196 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.196 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.196 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.196 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.196 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522507+0000", 2024-09-13T14:42:43.196 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5600000000000004e-07 2024-09-13T14:42:43.196 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.196 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.196 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.197 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522511+0000", 2024-09-13T14:42:43.197 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3200000000000001e-06 2024-09-13T14:42:43.197 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.197 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.197 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.197 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677247+0000", 2024-09-13T14:42:43.197 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15473636299999999 2024-09-13T14:42:43.197 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.197 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.197 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.197 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677263+0000", 2024-09-13T14:42:43.197 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5777000000000002e-05 2024-09-13T14:42:43.197 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.198 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.198 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.198 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677381+0000", 2024-09-13T14:42:43.198 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000117434 2024-09-13T14:42:43.198 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.198 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.198 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.198 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677389+0000", 2024-09-13T14:42:43.198 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8219999999999996e-06 2024-09-13T14:42:43.198 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.198 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.198 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.198 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677515+0000", 2024-09-13T14:42:43.198 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012509199999999999 2024-09-13T14:42:43.199 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.199 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.199 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.199 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.199 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.199 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.35049.0:68084 5.2 e119/107 ondisk, result = 0)", 2024-09-13T14:42:43.199 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.522895+0000", 2024-09-13T14:42:43.199 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 137.62284784900001, 2024-09-13T14:42:43.199 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15760656300000001, 2024-09-13T14:42:43.199 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.199 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.199 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:42:43.199 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.199 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.200 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.200 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522895+0000", 2024-09-13T14:42:43.200 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.200 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.200 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.200 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.200 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522895+0000", 2024-09-13T14:42:43.200 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.200 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.200 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.200 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.200 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522896+0000", 2024-09-13T14:42:43.200 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3799999999999997e-07 2024-09-13T14:42:43.201 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.201 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.201 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.201 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522905+0000", 2024-09-13T14:42:43.201 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7100000000000002e-06 2024-09-13T14:42:43.201 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.201 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.201 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.201 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677276+0000", 2024-09-13T14:42:43.201 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.154370968 2024-09-13T14:42:43.201 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.201 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.201 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.201 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677287+0000", 2024-09-13T14:42:43.202 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0641e-05 2024-09-13T14:42:43.202 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.202 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.202 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.202 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.680481+0000", 2024-09-13T14:42:43.202 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0031934720000000002 2024-09-13T14:42:43.202 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.202 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.202 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.202 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.680489+0000", 2024-09-13T14:42:43.202 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0590000000000003e-06 2024-09-13T14:42:43.202 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.202 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.202 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.203 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.680502+0000", 2024-09-13T14:42:43.203 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3175e-05 2024-09-13T14:42:43.203 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.203 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.203 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.203 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.203 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.203 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:155320 5.7 e130/107 5:ebe30544:::benchmark_data_smithi096_138841_object25124:head v 130'9638, mlcod=130'9635)", 2024-09-13T14:42:43.203 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.976910+0000", 2024-09-13T14:42:43.203 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 55.168832748, 2024-09-13T14:42:43.203 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.145356561, 2024-09-13T14:42:43.203 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.203 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.204 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.204 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.204 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.204 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.204 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.976910+0000", 2024-09-13T14:42:43.204 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.204 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.204 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.204 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.204 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.976910+0000", 2024-09-13T14:42:43.204 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.204 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.204 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.204 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.205 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.976911+0000", 2024-09-13T14:42:43.205 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.9999999999999995e-07 2024-09-13T14:42:43.205 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.205 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.205 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.205 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.976919+0000", 2024-09-13T14:42:43.205 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0690000000000002e-06 2024-09-13T14:42:43.205 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.205 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.205 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.205 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.976931+0000", 2024-09-13T14:42:43.205 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1506e-05 2024-09-13T14:42:43.205 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.205 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.206 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.206 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.976938+0000", 2024-09-13T14:42:43.206 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1339999999999998e-06 2024-09-13T14:42:43.206 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.206 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.206 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.206 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.977990+0000", 2024-09-13T14:42:43.206 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0010518190000000001 2024-09-13T14:42:43.206 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.206 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.206 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.206 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.978018+0000", 2024-09-13T14:42:43.206 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7798e-05 2024-09-13T14:42:43.207 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.207 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.207 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.207 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.122093+0000", 2024-09-13T14:42:43.207 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.144075693 2024-09-13T14:42:43.207 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.207 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.207 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.207 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.122267+0000", 2024-09-13T14:42:43.207 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000173542 2024-09-13T14:42:43.207 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.207 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.207 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.207 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.207 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.208 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:155322 5.7 e130/107 5:eae81fbc:::benchmark_data_smithi096_138841_object25126:head v 130'9639, mlcod=130'9635)", 2024-09-13T14:42:43.208 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.978364+0000", 2024-09-13T14:42:43.208 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 55.167378827999997, 2024-09-13T14:42:43.208 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15947017999999999, 2024-09-13T14:42:43.208 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.208 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.208 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.208 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.208 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.208 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.208 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.978364+0000", 2024-09-13T14:42:43.208 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.208 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.209 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.209 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.209 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.978364+0000", 2024-09-13T14:42:43.209 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.209 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.209 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.209 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.209 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.978365+0000", 2024-09-13T14:42:43.209 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4700000000000001e-07 2024-09-13T14:42:43.209 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.209 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.209 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.209 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.978369+0000", 2024-09-13T14:42:43.210 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0860000000000004e-06 2024-09-13T14:42:43.210 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.210 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.210 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.210 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.978379+0000", 2024-09-13T14:42:43.210 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0566e-05 2024-09-13T14:42:43.210 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.210 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.210 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.210 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.978385+0000", 2024-09-13T14:42:43.210 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7169999999999996e-06 2024-09-13T14:42:43.210 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.210 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.210 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.211 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982196+0000", 2024-09-13T14:42:43.211 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0038109509999999999 2024-09-13T14:42:43.211 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.211 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.211 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.211 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982525+0000", 2024-09-13T14:42:43.211 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00032845200000000002 2024-09-13T14:42:43.211 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.211 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.211 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.211 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.137777+0000", 2024-09-13T14:42:43.211 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15525257000000001 2024-09-13T14:42:43.211 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.212 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.212 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.212 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.137834+0000", 2024-09-13T14:42:43.212 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7290999999999997e-05 2024-09-13T14:42:43.212 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.212 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.212 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.212 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.212 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.212 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:155332 5.5 5:af4c7fbd:::benchmark_data_smithi096_138841_object25136:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:42:43.212 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.981878+0000", 2024-09-13T14:42:43.212 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 55.1638655, 2024-09-13T14:42:43.212 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.14758105599999999, 2024-09-13T14:42:43.213 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.213 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.213 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.213 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:42:43.213 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:42:43.213 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:42:43.213 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 155332 2024-09-13T14:42:43.213 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.213 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.213 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.213 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.213 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.981878+0000", 2024-09-13T14:42:43.213 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.214 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.214 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.214 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.214 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.981878+0000", 2024-09-13T14:42:43.214 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.214 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.214 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.214 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.214 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.981879+0000", 2024-09-13T14:42:43.214 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0330000000000001e-06 2024-09-13T14:42:43.214 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.214 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.214 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.214 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.981882+0000", 2024-09-13T14:42:43.215 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6109999999999998e-06 2024-09-13T14:42:43.215 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.215 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.215 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.215 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.981891+0000", 2024-09-13T14:42:43.215 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2070000000000001e-06 2024-09-13T14:42:43.215 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.215 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.215 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.215 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.981898+0000", 2024-09-13T14:42:43.215 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4080000000000003e-06 2024-09-13T14:42:43.215 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.215 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.216 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.216 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982086+0000", 2024-09-13T14:42:43.216 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00018797200000000001 2024-09-13T14:42:43.216 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.216 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.216 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.216 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986738+0000", 2024-09-13T14:42:43.216 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0046517449999999997 2024-09-13T14:42:43.216 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.216 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.216 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:42:43.216 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986844+0000", 2024-09-13T14:42:43.216 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000106773 2024-09-13T14:42:43.216 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.217 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.217 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.217 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129296+0000", 2024-09-13T14:42:43.217 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.142451877 2024-09-13T14:42:43.217 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.217 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.217 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.217 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129341+0000", 2024-09-13T14:42:43.217 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5243e-05 2024-09-13T14:42:43.217 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.217 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.217 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:42:43.217 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129357+0000", 2024-09-13T14:42:43.217 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5495e-05 2024-09-13T14:42:43.218 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.218 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.218 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.218 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129416+0000", 2024-09-13T14:42:43.218 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8953999999999998e-05 2024-09-13T14:42:43.218 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.218 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.218 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.218 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129459+0000", 2024-09-13T14:42:43.218 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2738000000000003e-05 2024-09-13T14:42:43.218 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.218 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.218 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.218 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.219 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.219 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:155335 5.3 5:c44414a0:::benchmark_data_smithi096_138841_object25139:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:42:43.219 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.982035+0000", 2024-09-13T14:42:43.219 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 55.163707541999997, 2024-09-13T14:42:43.219 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.14205281, 2024-09-13T14:42:43.219 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.219 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.219 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.219 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:42:43.219 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:42:43.219 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:42:43.219 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 155335 2024-09-13T14:42:43.219 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.220 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.220 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.220 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.220 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982035+0000", 2024-09-13T14:42:43.220 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.220 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.220 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.220 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.220 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982035+0000", 2024-09-13T14:42:43.220 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.220 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.220 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.220 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.220 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982036+0000", 2024-09-13T14:42:43.221 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9999999999999997e-07 2024-09-13T14:42:43.221 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.221 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.221 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.221 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982040+0000", 2024-09-13T14:42:43.221 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.3519999999999998e-06 2024-09-13T14:42:43.221 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.221 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.221 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.221 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982048+0000", 2024-09-13T14:42:43.221 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6780000000000006e-06 2024-09-13T14:42:43.221 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.221 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.222 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.222 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982054+0000", 2024-09-13T14:42:43.222 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0310000000000004e-06 2024-09-13T14:42:43.222 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.222 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.222 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.222 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982249+0000", 2024-09-13T14:42:43.222 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00019445899999999999 2024-09-13T14:42:43.222 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.222 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.222 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.222 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982377+0000", 2024-09-13T14:42:43.222 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012871200000000001 2024-09-13T14:42:43.223 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.223 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.223 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 3,5", 2024-09-13T14:42:43.223 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982458+0000", 2024-09-13T14:42:43.223 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0708e-05 2024-09-13T14:42:43.223 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.223 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.223 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.223 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989435+0000", 2024-09-13T14:42:43.223 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0069764129999999999 2024-09-13T14:42:43.223 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.223 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.223 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.223 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.119639+0000", 2024-09-13T14:42:43.224 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.130204137 2024-09-13T14:42:43.224 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.224 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.224 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:42:43.224 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124001+0000", 2024-09-13T14:42:43.224 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0043621049999999998 2024-09-13T14:42:43.224 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.224 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.224 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.224 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124047+0000", 2024-09-13T14:42:43.224 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6313000000000002e-05 2024-09-13T14:42:43.224 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.224 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.225 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.225 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124088+0000", 2024-09-13T14:42:43.225 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1202000000000003e-05 2024-09-13T14:42:43.225 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.225 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.225 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.225 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.225 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.225 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:155336 5.3 5:c313698d:::benchmark_data_smithi096_138841_object25140:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:42:43.225 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.982278+0000", 2024-09-13T14:42:43.225 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 55.163464689000001, 2024-09-13T14:42:43.225 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.14239633900000001, 2024-09-13T14:42:43.225 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.226 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.226 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.226 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:42:43.226 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:42:43.226 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:42:43.226 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 155336 2024-09-13T14:42:43.226 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.226 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.226 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.226 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.226 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982278+0000", 2024-09-13T14:42:43.226 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.226 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.227 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.227 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.227 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982278+0000", 2024-09-13T14:42:43.227 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.227 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.227 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.227 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.227 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982279+0000", 2024-09-13T14:42:43.227 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.9399999999999993e-07 2024-09-13T14:42:43.227 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.227 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.227 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.227 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982285+0000", 2024-09-13T14:42:43.227 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2789999999999998e-06 2024-09-13T14:42:43.228 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.228 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.228 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.228 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982296+0000", 2024-09-13T14:42:43.228 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.092e-05 2024-09-13T14:42:43.228 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.228 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.228 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.228 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982303+0000", 2024-09-13T14:42:43.228 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9689999999999997e-06 2024-09-13T14:42:43.228 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.228 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.229 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.229 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.985618+0000", 2024-09-13T14:42:43.229 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.003315597 2024-09-13T14:42:43.229 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.229 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.229 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.229 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.985751+0000", 2024-09-13T14:42:43.229 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013257899999999999 2024-09-13T14:42:43.229 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.229 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.229 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 3,5", 2024-09-13T14:42:43.229 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.985862+0000", 2024-09-13T14:42:43.229 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000110851 2024-09-13T14:42:43.229 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.230 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.230 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.230 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.119699+0000", 2024-09-13T14:42:43.230 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.13383772899999999 2024-09-13T14:42:43.230 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.230 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.230 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:42:43.230 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124097+0000", 2024-09-13T14:42:43.230 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0043978649999999999 2024-09-13T14:42:43.230 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.230 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.230 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.230 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124580+0000", 2024-09-13T14:42:43.231 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00048257600000000002 2024-09-13T14:42:43.231 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.231 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.231 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.231 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124633+0000", 2024-09-13T14:42:43.231 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3705000000000002e-05 2024-09-13T14:42:43.231 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.231 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.231 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.231 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124675+0000", 2024-09-13T14:42:43.231 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1275e-05 2024-09-13T14:42:43.231 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.231 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.231 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.232 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.232 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.232 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:155325 5.7 e130/107 5:ee3ba1a9:::benchmark_data_smithi096_138841_object25129:head v 130'9640, mlcod=130'9636)", 2024-09-13T14:42:43.232 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.982625+0000", 2024-09-13T14:42:43.232 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 55.163118054000002, 2024-09-13T14:42:43.232 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.155244469, 2024-09-13T14:42:43.232 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.232 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.232 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.232 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.232 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.233 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.233 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982625+0000", 2024-09-13T14:42:43.233 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.233 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.233 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.233 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.233 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982625+0000", 2024-09-13T14:42:43.233 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.233 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.233 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.233 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.233 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982626+0000", 2024-09-13T14:42:43.234 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0800000000000004e-07 2024-09-13T14:42:43.234 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.234 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.234 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.234 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982632+0000", 2024-09-13T14:42:43.234 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6590000000000001e-06 2024-09-13T14:42:43.234 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.234 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.234 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.234 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982641+0000", 2024-09-13T14:42:43.234 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5799999999999992e-06 2024-09-13T14:42:43.234 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.234 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.234 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.235 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982647+0000", 2024-09-13T14:42:43.235 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7169999999999996e-06 2024-09-13T14:42:43.235 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.235 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.235 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.235 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986298+0000", 2024-09-13T14:42:43.235 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0036512010000000002 2024-09-13T14:42:43.235 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.235 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.235 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.235 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986445+0000", 2024-09-13T14:42:43.235 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00014707699999999999 2024-09-13T14:42:43.235 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.236 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.236 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.236 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.137839+0000", 2024-09-13T14:42:43.236 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15139466800000001 2024-09-13T14:42:43.236 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.236 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.236 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.236 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.137869+0000", 2024-09-13T14:42:43.236 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.9958999999999999e-05 2024-09-13T14:42:43.236 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.236 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.236 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.236 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.236 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.237 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:155327 5.7 e130/107 5:eb2cb127:::benchmark_data_smithi096_138841_object25131:head v 130'9641, mlcod=130'9636)", 2024-09-13T14:42:43.237 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.986380+0000", 2024-09-13T14:42:43.237 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 55.159362682999998, 2024-09-13T14:42:43.237 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16496414300000001, 2024-09-13T14:42:43.237 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.237 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.238 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.238 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.238 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.238 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.238 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986380+0000", 2024-09-13T14:42:43.238 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.238 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.238 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.239 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.239 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986380+0000", 2024-09-13T14:42:43.239 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.239 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.239 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.239 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.239 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986381+0000", 2024-09-13T14:42:43.239 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3099999999999998e-07 2024-09-13T14:42:43.240 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.240 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.240 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.240 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986386+0000", 2024-09-13T14:42:43.240 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6569999999999996e-06 2024-09-13T14:42:43.240 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.240 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.240 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.240 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986397+0000", 2024-09-13T14:42:43.240 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1121000000000001e-05 2024-09-13T14:42:43.240 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.240 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.240 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.241 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986402+0000", 2024-09-13T14:42:43.241 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7119999999999997e-06 2024-09-13T14:42:43.241 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.241 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.241 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.241 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.122374+0000", 2024-09-13T14:42:43.241 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.135971707 2024-09-13T14:42:43.241 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.241 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.241 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.241 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.122417+0000", 2024-09-13T14:42:43.241 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3392000000000001e-05 2024-09-13T14:42:43.241 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.242 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.242 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.242 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.151302+0000", 2024-09-13T14:42:43.242 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.028884310999999999 2024-09-13T14:42:43.242 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.242 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.242 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.242 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.151345+0000", 2024-09-13T14:42:43.242 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2712e-05 2024-09-13T14:42:43.243 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.243 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.243 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.243 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.243 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.243 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:155339 5.3 5:c0e941be:::benchmark_data_smithi096_138841_object25143:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:42:43.243 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.989369+0000", 2024-09-13T14:42:43.243 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 55.156374245000002, 2024-09-13T14:42:43.243 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.138433308, 2024-09-13T14:42:43.243 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.243 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.243 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.243 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:42:43.244 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:42:43.244 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:42:43.244 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 155339 2024-09-13T14:42:43.244 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.244 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.244 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.244 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.244 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989369+0000", 2024-09-13T14:42:43.244 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.244 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.244 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.244 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.244 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989369+0000", 2024-09-13T14:42:43.244 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.245 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.245 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.245 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.245 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989370+0000", 2024-09-13T14:42:43.245 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.145e-06 2024-09-13T14:42:43.245 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.245 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.245 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.245 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989375+0000", 2024-09-13T14:42:43.245 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0189999999999997e-06 2024-09-13T14:42:43.245 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.246 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.246 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.246 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989385+0000", 2024-09-13T14:42:43.246 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.5629999999999998e-06 2024-09-13T14:42:43.246 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.246 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.246 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.246 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989393+0000", 2024-09-13T14:42:43.246 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0770000000000008e-06 2024-09-13T14:42:43.246 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.246 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.246 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.247 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120707+0000", 2024-09-13T14:42:43.247 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.13131430699999999 2024-09-13T14:42:43.247 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.247 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.247 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.247 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120838+0000", 2024-09-13T14:42:43.247 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000131471 2024-09-13T14:42:43.247 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.247 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.247 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 3,5", 2024-09-13T14:42:43.247 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120966+0000", 2024-09-13T14:42:43.247 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012741400000000001 2024-09-13T14:42:43.247 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.247 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.248 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.248 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.127627+0000", 2024-09-13T14:42:43.248 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0066615809999999998 2024-09-13T14:42:43.248 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.248 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.248 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:42:43.248 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.127644+0000", 2024-09-13T14:42:43.248 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6426000000000002e-05 2024-09-13T14:42:43.248 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.248 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.248 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:43.248 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.127685+0000", 2024-09-13T14:42:43.248 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1085999999999999e-05 2024-09-13T14:42:43.249 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.249 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.249 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.249 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.127755+0000", 2024-09-13T14:42:43.249 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.0322999999999997e-05 2024-09-13T14:42:43.249 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.249 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.249 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.249 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.127802+0000", 2024-09-13T14:42:43.249 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6896e-05 2024-09-13T14:42:43.249 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.249 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.249 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.249 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.250 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.250 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:155343 5.8 e130/107 5:1b6701d1:::benchmark_data_smithi096_138841_object25147:head v 130'9533, mlcod=130'9531)", 2024-09-13T14:42:43.250 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.990682+0000", 2024-09-13T14:42:43.250 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 55.155060937999998, 2024-09-13T14:42:43.250 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.138397784, 2024-09-13T14:42:43.250 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.250 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.250 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.250 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.250 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.250 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.250 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.990682+0000", 2024-09-13T14:42:43.250 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.251 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.251 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.251 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.251 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.990682+0000", 2024-09-13T14:42:43.251 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.251 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.251 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.251 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.251 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.990683+0000", 2024-09-13T14:42:43.251 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.75e-07 2024-09-13T14:42:43.251 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.251 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.251 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.252 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.990687+0000", 2024-09-13T14:42:43.252 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4270000000000001e-06 2024-09-13T14:42:43.252 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.252 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.252 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.252 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120445+0000", 2024-09-13T14:42:43.252 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.12975792899999999 2024-09-13T14:42:43.252 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.252 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.252 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.252 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120451+0000", 2024-09-13T14:42:43.252 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6659999999999996e-06 2024-09-13T14:42:43.252 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.252 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.253 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.253 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.121219+0000", 2024-09-13T14:42:43.253 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00076832500000000004 2024-09-13T14:42:43.253 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.253 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.253 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.253 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.121239+0000", 2024-09-13T14:42:43.253 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0063000000000001e-05 2024-09-13T14:42:43.253 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.253 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.253 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.253 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129032+0000", 2024-09-13T14:42:43.253 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.007792635 2024-09-13T14:42:43.254 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.254 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.254 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.254 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129080+0000", 2024-09-13T14:42:43.254 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8164000000000002e-05 2024-09-13T14:42:43.254 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.254 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.254 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.254 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.254 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.254 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.35049.0:155336 5.3 e130/107 ondisk, result = 0)", 2024-09-13T14:42:43.254 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.991017+0000", 2024-09-13T14:42:43.255 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 55.154725894999999, 2024-09-13T14:42:43.255 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.13366312299999999, 2024-09-13T14:42:43.255 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.255 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.255 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:42:43.255 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.255 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.255 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.255 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.991017+0000", 2024-09-13T14:42:43.255 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.255 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.255 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.255 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.256 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.991017+0000", 2024-09-13T14:42:43.256 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.256 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.256 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.256 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.256 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.991018+0000", 2024-09-13T14:42:43.256 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4899999999999995e-07 2024-09-13T14:42:43.256 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.256 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.256 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.256 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.991021+0000", 2024-09-13T14:42:43.256 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4189999999999998e-06 2024-09-13T14:42:43.256 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.256 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.257 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.257 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120165+0000", 2024-09-13T14:42:43.257 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.129144174 2024-09-13T14:42:43.257 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.257 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.257 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.257 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120249+0000", 2024-09-13T14:42:43.257 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.3357999999999997e-05 2024-09-13T14:42:43.257 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.257 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.257 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.257 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124565+0000", 2024-09-13T14:42:43.257 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0043166400000000001 2024-09-13T14:42:43.258 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.258 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.258 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.258 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124574+0000", 2024-09-13T14:42:43.258 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8359999999999998e-06 2024-09-13T14:42:43.258 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.258 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.258 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.258 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124680+0000", 2024-09-13T14:42:43.258 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000106047 2024-09-13T14:42:43.258 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.258 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.258 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.259 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.259 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.259 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:155348 5.7 e130/107 5:e352345d:::benchmark_data_smithi096_138841_object25152:head v 130'9642, mlcod=130'9636)", 2024-09-13T14:42:43.259 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.999666+0000", 2024-09-13T14:42:43.259 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 55.146077448, 2024-09-13T14:42:43.259 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15170647500000001, 2024-09-13T14:42:43.259 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:42:43.259 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:42:43.259 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:43.259 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:42:43.259 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.259 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:42:43.259 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.999666+0000", 2024-09-13T14:42:43.260 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.260 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.260 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.260 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:42:43.260 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.999666+0000", 2024-09-13T14:42:43.260 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:42:43.260 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.260 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.260 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:42:43.260 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.999667+0000", 2024-09-13T14:42:43.260 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.265e-06 2024-09-13T14:42:43.260 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.260 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.261 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:42:43.261 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.999677+0000", 2024-09-13T14:42:43.261 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.876e-06 2024-09-13T14:42:43.261 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.261 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.261 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:42:43.261 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120515+0000", 2024-09-13T14:42:43.261 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.120838062 2024-09-13T14:42:43.261 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.261 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.261 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:42:43.261 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120520+0000", 2024-09-13T14:42:43.261 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3279999999999996e-06 2024-09-13T14:42:43.262 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.262 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.262 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:42:43.262 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.132225+0000", 2024-09-13T14:42:43.262 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011705357 2024-09-13T14:42:43.262 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.262 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.262 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:42:43.262 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.132324+0000", 2024-09-13T14:42:43.262 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.8665999999999994e-05 2024-09-13T14:42:43.262 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.262 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.262 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:42:43.263 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.151348+0000", 2024-09-13T14:42:43.263 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.019023786000000001 2024-09-13T14:42:43.263 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:42:43.263 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:42:43.263 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:42:43.263 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.151372+0000", 2024-09-13T14:42:43.263 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4134999999999998e-05 2024-09-13T14:42:43.263 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.263 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.263 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.263 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:42:43.263 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:42:43.263 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:42:43.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.265 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:43.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:43.366 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:43.395 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:43.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:43.497 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:43.526 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:43.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:43.628 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:43.662 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:43.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:43.764 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:43.807 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:43.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:43.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:43 smithi096 ceph-mon[98901]: pgmap v361: 99 pgs: 99 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 972 op/s 2024-09-13T14:42:43.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:43 smithi096 ceph-mon[102928]: pgmap v361: 99 pgs: 99 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 972 op/s 2024-09-13T14:42:43.909 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:43.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.940 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:43.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:43.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:43 smithi204 ceph-mon[87071]: pgmap v361: 99 pgs: 99 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 972 op/s 2024-09-13T14:42:44.041 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:44.073 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:44.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:44.175 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:44.214 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:44.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:44.316 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:44.343 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:44.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:44.446 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:44.477 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:44.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:44.579 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:44.612 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:44.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:44.714 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:44.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.746 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:44.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:44.847 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:44.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.869 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:44.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:44.970 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:45.007 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:45.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:45.109 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:45.133 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:45.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:45.235 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:45.264 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:45.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:45.365 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:45.396 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:45.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:45.498 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:45.534 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:45.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:45.636 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:45.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.663 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:45.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:45.764 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:45.800 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:45.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:45.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:45 smithi096 ceph-mon[98901]: pgmap v362: 99 pgs: 99 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 964 op/s 2024-09-13T14:42:45.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:45 smithi096 ceph-mon[102928]: pgmap v362: 99 pgs: 99 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 964 op/s 2024-09-13T14:42:45.902 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:45.933 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:45.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:45.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:45 smithi204 ceph-mon[87071]: pgmap v362: 99 pgs: 99 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 964 op/s 2024-09-13T14:42:46.035 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:46.066 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:46.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:46.168 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:46.194 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:46.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:46.296 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:46.326 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:46.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:46.428 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:46.467 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:46.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:46.568 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:46.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.599 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:46.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:46.700 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:46.734 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:46.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:46.836 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:46.876 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:46.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:46.978 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:47.008 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:47.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:47.110 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:47.170 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:47.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:47.271 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:47.295 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:42:47.295 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:42:47.296 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:47.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:47.329 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:42:47.330 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:42:47.366 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:42:47.366 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:42:47.398 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:47.405 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:42:47.405 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:42:47.434 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:42:47.434 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:42:47.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.444 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:47.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:47.462 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:42:47.462 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:42:47.490 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:42:47.491 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:42:47.522 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:42:47.522 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:42:47.545 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:47.590 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:47.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:47.693 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:42:47.693 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:47.719 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:47.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:47.731 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:47 smithi096 ceph-mon[98901]: pgmap v363: 99 pgs: 99 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 967 op/s 2024-09-13T14:42:47.731 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:47 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/201728692' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:47.731 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:47 smithi096 ceph-mon[102928]: pgmap v363: 99 pgs: 99 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 967 op/s 2024-09-13T14:42:47.732 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:47 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/201728692' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:47.821 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:47.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.863 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:47.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:47.964 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:47.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:47 smithi204 ceph-mon[87071]: pgmap v363: 99 pgs: 99 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 967 op/s 2024-09-13T14:42:47.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:47 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/201728692' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:47.991 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:42:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:42:47] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.51.0" 2024-09-13T14:42:48.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.025 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:48.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:48.126 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:48.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.158 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:48.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:48.260 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:48.287 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:48.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:48.389 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:48.421 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:48.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:48.523 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:48.560 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:48.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:48.662 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:48.690 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:48.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:48.792 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:48.827 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:48.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:48.929 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:48.968 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:48.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:49.072 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:49.102 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:49.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:49.205 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:49.237 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:49.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:49.339 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:49.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.374 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:49.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:49.475 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:49.523 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:49.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:49.626 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:49.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.653 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:49.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:49.754 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:49.791 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:49.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:49.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:49 smithi096 ceph-mon[98901]: pgmap v364: 99 pgs: 99 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 941 op/s 2024-09-13T14:42:49.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:49 smithi096 ceph-mon[102928]: pgmap v364: 99 pgs: 99 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 941 op/s 2024-09-13T14:42:49.894 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:49.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.920 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:49.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:49.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:49 smithi204 ceph-mon[87071]: pgmap v364: 99 pgs: 99 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 941 op/s 2024-09-13T14:42:50.022 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:50.062 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:50.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:50.164 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:50.189 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:50.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:50.291 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:50.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.325 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:50.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:50.329 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:42:50.426 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:50.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.469 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:50.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:50.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:42:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:42:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:50.570 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:50.609 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:50.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:50.711 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:50.763 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:50.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:50.808 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:42:50.808 INFO:teuthology.orchestra.run.smithi096.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:42:50.808 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:42:50.865 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:50.897 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:50.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:50.999 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:51.033 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:51.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:51.136 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:51.165 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:51.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:51.267 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:51.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.308 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:51.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:51.409 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:51.448 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:51.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:51.550 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:51.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.586 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:51.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:51.687 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:51.710 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:51.719 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:51.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:51.797 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:51.821 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:51.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:51 smithi096 ceph-mon[98901]: pgmap v365: 99 pgs: 99 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:42:51.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:51.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:51.826 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:51 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1500363349' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:51.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:51 smithi096 ceph-mon[102928]: pgmap v365: 99 pgs: 99 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:42:51.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:51.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:51.827 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:51 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1500363349' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:51.827 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:42:51.850 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:51.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:51.952 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:51.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:51 smithi204 ceph-mon[87071]: pgmap v365: 99 pgs: 99 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:42:51.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:42:51.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:51.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:51 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1500363349' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:51.982 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:51.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:52.084 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:52.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.132 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:52.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:52.233 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:52.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.266 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:52.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:52.367 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:52.402 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:52.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:52.504 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:52.539 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:52.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:52.641 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:52.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.669 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:52.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:52.770 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:52.804 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:52.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:52.906 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:52.936 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:52.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:53.038 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:53.071 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:52 smithi096 ceph-mon[98901]: pgmap v366: 99 pgs: 99 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 866 op/s 2024-09-13T14:42:53.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:52 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:53.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:52 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1500363349' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:53.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:52 smithi096 ceph-mon[98901]: osdmap e138: 8 total, 8 up, 7 in 2024-09-13T14:42:53.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:52 smithi096 ceph-mon[102928]: pgmap v366: 99 pgs: 99 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 866 op/s 2024-09-13T14:42:53.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:52 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:53.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:52 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1500363349' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:53.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:52 smithi096 ceph-mon[102928]: osdmap e138: 8 total, 8 up, 7 in 2024-09-13T14:42:53.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.076 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:53.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:53.178 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:53.205 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:53.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:53.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:52 smithi204 ceph-mon[87071]: pgmap v366: 99 pgs: 99 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 866 op/s 2024-09-13T14:42:53.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:52 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:53.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:52 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1500363349' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:53.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:52 smithi204 ceph-mon[87071]: osdmap e138: 8 total, 8 up, 7 in 2024-09-13T14:42:53.307 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:53.343 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:53.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:53.446 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:53.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.490 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:53.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:53.591 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:53.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.622 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:53.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:53.723 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:53.749 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:53.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:53.851 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:53.888 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:53.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:53.990 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:54.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.019 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:54.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:54.120 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:54.167 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:54.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:54.269 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:54.309 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:54.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:54.411 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:54.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.449 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:54.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:54.550 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:54.580 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:54.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:54.681 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:54.708 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:54.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:54.800 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:42:54.809 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:54.835 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:54.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:54.937 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:54.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.979 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:54.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:55.080 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:55.106 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:55.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:55.152 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:54 smithi096 ceph-mon[98901]: pgmap v368: 99 pgs: 99 active+clean; 2.7 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 976 op/s 2024-09-13T14:42:55.153 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:54 smithi096 ceph-mon[102928]: pgmap v368: 99 pgs: 99 active+clean; 2.7 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 976 op/s 2024-09-13T14:42:55.207 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:55.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:54 smithi204 ceph-mon[87071]: pgmap v368: 99 pgs: 99 active+clean; 2.7 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 976 op/s 2024-09-13T14:42:55.248 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:55.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:55.350 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:55.385 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:55.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:55.429 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:42:55.429 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:42:55.487 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:55.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.533 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:55.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:55.634 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:55.665 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:55.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:55.767 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:55.797 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:55.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:55.899 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:55.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.936 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:55.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:56.037 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:56.074 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:56.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:56.176 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:56.208 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:56.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:56.310 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:56.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.351 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:56.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:56.452 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:56.488 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:56.488 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:42:56.488 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:42:56.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:56.515 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:56.531 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:42:56.532 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:42:56.571 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:42:56.571 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:42:56.590 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:56.603 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:42:56.604 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:42:56.620 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:56.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:56.637 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:42:56.637 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:42:56.669 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:42:56.670 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:42:56.707 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:42:56.707 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:42:56.721 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:56.741 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:42:56.741 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:42:56.743 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:42:56.743 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:42:56.743 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T14:42:56.743 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:42:56.759 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:56.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:56.861 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:56.899 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:56.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:57.001 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:57.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.043 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:57.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:57.144 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:57.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.190 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:57.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:57.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:56 smithi204 ceph-mon[87071]: pgmap v369: 99 pgs: 99 active+clean; 2.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 966 op/s 2024-09-13T14:42:57.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:56 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2482734031' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:57.291 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:57.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:56 smithi096 ceph-mon[98901]: pgmap v369: 99 pgs: 99 active+clean; 2.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 966 op/s 2024-09-13T14:42:57.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:56 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2482734031' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:57.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:56 smithi096 ceph-mon[102928]: pgmap v369: 99 pgs: 99 active+clean; 2.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 966 op/s 2024-09-13T14:42:57.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:56 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2482734031' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:57.326 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:57.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:57.428 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:57.466 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:57.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:57.568 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:57.605 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:57.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:57.669 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:42:57.707 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:57.733 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:57.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:57.835 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:57.862 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:57.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:57.963 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:58.026 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:58.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:58.130 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:58.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.157 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:58.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:58.258 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:58.264 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:42:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:42:57] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.51.0" 2024-09-13T14:42:58.301 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:58.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:58.403 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:58.434 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:58.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:58.536 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:58.583 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:58.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:58.685 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:58.710 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:58.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:58.812 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:58.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.854 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:58.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:58.955 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:58.988 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:58.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:59.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:42:58 smithi096 ceph-mon[98901]: pgmap v370: 99 pgs: 99 active+clean; 2.4 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 976 op/s 2024-09-13T14:42:59.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:42:58 smithi096 ceph-mon[102928]: pgmap v370: 99 pgs: 99 active+clean; 2.4 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 976 op/s 2024-09-13T14:42:59.091 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:59.123 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:59.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:59.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:42:58 smithi204 ceph-mon[87071]: pgmap v370: 99 pgs: 99 active+clean; 2.4 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 976 op/s 2024-09-13T14:42:59.224 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:59.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.257 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:59.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:59.358 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:59.391 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:59.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:59.492 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:59.536 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:59.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:59.638 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:59.669 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:59.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:59.742 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:59.771 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:59.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.840 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:42:59.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:59.855 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:59.941 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:59.967 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:42:59.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:00.069 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:00.109 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:00.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:00.211 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:00.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.238 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:00.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:00.339 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:00.374 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:00.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:00.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:43:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:43:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:00.476 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:00.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.506 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:00.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:00.607 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:00.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.646 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:00.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:00.718 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:43:00.747 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:00.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.805 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:00.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:00.906 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:00.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.945 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:00.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:00.953 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:00 smithi096 ceph-mon[98901]: pgmap v371: 99 pgs: 99 active+clean; 2.3 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 969 op/s 2024-09-13T14:43:00.953 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:00 smithi096 ceph-mon[102928]: pgmap v371: 99 pgs: 99 active+clean; 2.3 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 969 op/s 2024-09-13T14:43:01.046 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:01.105 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:01.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:01.207 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:01.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:00 smithi204 ceph-mon[87071]: pgmap v371: 99 pgs: 99 active+clean; 2.3 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 969 op/s 2024-09-13T14:43:01.235 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:01.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:01.337 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:01.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.374 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:01.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:01.475 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:01.507 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:01.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:01.609 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:01.647 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:01.749 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:01.785 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:01.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:01.860 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:43:01.887 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:01.929 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:01.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:02.031 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:02.055 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:02.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:02.115 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:01 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2161751757' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:43:02.116 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:01 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:43:02.116 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:01 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2161751757' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:43:02.116 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:01 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:43:02.157 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:02.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.208 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:02.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:02.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:01 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2161751757' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:43:02.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:01 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:43:02.309 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:02.342 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:02.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:02.444 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:02.479 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:02.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:02.581 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:02.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.608 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:02.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:02.709 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:02.744 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:02.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:02.846 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:02.879 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:02.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:02.980 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:03.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.015 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:03.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:03.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:02 smithi096 ceph-mon[98901]: pgmap v372: 99 pgs: 99 active+clean; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 969 op/s 2024-09-13T14:43:03.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:02 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:03.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:02 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:43:03.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:02 smithi096 ceph-mon[98901]: osdmap e139: 8 total, 8 up, 7 in 2024-09-13T14:43:03.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:02 smithi096 ceph-mon[102928]: pgmap v372: 99 pgs: 99 active+clean; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 969 op/s 2024-09-13T14:43:03.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:02 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:03.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:02 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:43:03.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:02 smithi096 ceph-mon[102928]: osdmap e139: 8 total, 8 up, 7 in 2024-09-13T14:43:03.117 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:03.155 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:03.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:03.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:02 smithi204 ceph-mon[87071]: pgmap v372: 99 pgs: 99 active+clean; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 969 op/s 2024-09-13T14:43:03.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:02 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:03.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:02 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:43:03.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:02 smithi204 ceph-mon[87071]: osdmap e139: 8 total, 8 up, 7 in 2024-09-13T14:43:03.257 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:03.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.285 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:03.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:03.386 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:03.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.417 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:03.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:03.519 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:03.548 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:03.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:03.650 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:03.692 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:03.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:03.794 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:03.827 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:03.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:03.929 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:03.958 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:03.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:04.060 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:04.098 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:04.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:04.200 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:04.249 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:04.253 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:04.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:04.272 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:43:04.273 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:43:04.355 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:04.379 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:04.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:04.481 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:04.520 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:04.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:04.622 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:04.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.669 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:04.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:04.771 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:04.812 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:04.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:04.914 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:04.941 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:04.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:05.046 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:05.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:04 smithi096 ceph-mon[98901]: pgmap v374: 99 pgs: 99 active+clean; 2.1 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 981 op/s 2024-09-13T14:43:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:04 smithi096 ceph-mon[98901]: from='client.35244 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:05.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:04 smithi096 ceph-mon[102928]: pgmap v374: 99 pgs: 99 active+clean; 2.1 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 981 op/s 2024-09-13T14:43:05.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:04 smithi096 ceph-mon[102928]: from='client.35244 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:05.079 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:05.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:05.182 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:05.213 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:05.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:05.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:04 smithi204 ceph-mon[87071]: pgmap v374: 99 pgs: 99 active+clean; 2.1 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 981 op/s 2024-09-13T14:43:05.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:04 smithi204 ceph-mon[87071]: from='client.35244 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:05.316 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:05.323 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":374,"stamp":"2024-09-13T14:43:03.705210+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2234193893,"num_objects":34047,"num_object_clones":0,"num_object_copies":102141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34047,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":227276,"num_write_kb":8344976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":4203971,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119701,"ondisk_log_size":119701,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7966676,"kb_used_data":6563224,"kb_used_omap":34,"kb_used_meta":1403229,"kb_avail":648249388,"statfs":{"total":671965249536,"available":663807373312,"internally_reserved":0,"allocated":6720741376,"data_stored":6715013105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35126,"internal_metadata":1436907210},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,7,10,7,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-643432448,"num_objects":-9818,"num_object_clones":0,"num_object_copies":-29454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9818,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003342"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":141,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.840034+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:42:51.840034+0000","last_peered":"2024-09-13T14:42:51.840034+0000","last_clean":"2024-09-13T14:42:51.840034+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:42:51.840034+0000","last_undegraded":"2024-09-13T14:42:51.840034+0000","last_fullsized":"2024-09-13T14:42:51.840034+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":104,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.436801+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:42:37.436801+0000","last_peered":"2024-09-13T14:42:37.436801+0000","last_clean":"2024-09-13T14:42:37.436801+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:42:37.436801+0000","last_undegraded":"2024-09-13T14:42:37.436801+0000","last_fullsized":"2024-09-13T14:42:37.436801+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.440640+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:42:37.440640+0000","last_peered":"2024-09-13T14:42:37.440640+0000","last_clean":"2024-09-13T14:42:37.440640+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:42:37.440640+0000","last_undegraded":"2024-09-13T14:42:37.440640+0000","last_fullsized":"2024-09-13T14:42:37.440640+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":136,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.870059+0000","last_change":"2024-09-13T14:38:13.977954+0000","last_active":"2024-09-13T14:43:01.870059+0000","last_peered":"2024-09-13T14:43:01.870059+0000","last_clean":"2024-09-13T14:43:01.870059+0000","last_became_active":"2024-09-13T14:38:13.977619+0000","last_became_peered":"2024-09-13T14:38:13.977619+0000","last_unstale":"2024-09-13T14:43:01.870059+0000","last_undegraded":"2024-09-13T14:43:01.870059+0000","last_fullsized":"2024-09-13T14:43:01.870059+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:58:30.259967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":137,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.868072+0000","last_change":"2024-09-13T14:38:13.978232+0000","last_active":"2024-09-13T14:43:01.868072+0000","last_peered":"2024-09-13T14:43:01.868072+0000","last_clean":"2024-09-13T14:43:01.868072+0000","last_became_active":"2024-09-13T14:38:13.977921+0000","last_became_peered":"2024-09-13T14:38:13.977921+0000","last_unstale":"2024-09-13T14:43:01.868072+0000","last_undegraded":"2024-09-13T14:43:01.868072+0000","last_fullsized":"2024-09-13T14:43:01.868072+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:10.127626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":95,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.436676+0000","last_change":"2024-09-13T14:39:42.765301+0000","last_active":"2024-09-13T14:42:37.436676+0000","last_peered":"2024-09-13T14:42:37.436676+0000","last_clean":"2024-09-13T14:42:37.436676+0000","last_became_active":"2024-09-13T14:39:42.764977+0000","last_became_peered":"2024-09-13T14:39:42.764977+0000","last_unstale":"2024-09-13T14:42:37.436676+0000","last_undegraded":"2024-09-13T14:42:37.436676+0000","last_fullsized":"2024-09-13T14:42:37.436676+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:25.223972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":122,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.868311+0000","last_change":"2024-09-13T14:37:40.925691+0000","last_active":"2024-09-13T14:43:01.868311+0000","last_peered":"2024-09-13T14:43:01.868311+0000","last_clean":"2024-09-13T14:43:01.868311+0000","last_became_active":"2024-09-13T14:37:40.925360+0000","last_became_peered":"2024-09-13T14:37:40.925360+0000","last_unstale":"2024-09-13T14:43:01.868311+0000","last_undegraded":"2024-09-13T14:43:01.868311+0000","last_fullsized":"2024-09-13T14:43:01.868311+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:07.337967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":97,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.867071+0000","last_change":"2024-09-13T14:39:42.781872+0000","last_active":"2024-09-13T14:43:01.867071+0000","last_peered":"2024-09-13T14:43:01.867071+0000","last_clean":"2024-09-13T14:43:01.867071+0000","last_became_active":"2024-09-13T14:39:42.770254+0000","last_became_peered":"2024-09-13T14:39:42.770254+0000","last_unstale":"2024-09-13T14:43:01.867071+0000","last_undegraded":"2024-09-13T14:43:01.867071+0000","last_fullsized":"2024-09-13T14:43:01.867071+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:31.211081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.9","version":"138'14038","reported_seq":14077,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.675426+0000","last_change":"2024-09-13T14:39:42.764535+0000","last_active":"2024-09-13T14:43:01.675426+0000","last_peered":"2024-09-13T14:43:01.675426+0000","last_clean":"2024-09-13T14:43:01.675426+0000","last_became_active":"2024-09-13T14:39:42.764101+0000","last_became_peered":"2024-09-13T14:39:42.764101+0000","last_unstale":"2024-09-13T14:43:01.675426+0000","last_undegraded":"2024-09-13T14:43:01.675426+0000","last_fullsized":"2024-09-13T14:43:01.675426+0000","mapping_epoch":107,"log_start":"122'6100","ondisk_log_start":"122'6100","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":7938,"log_dups_size":0,"ondisk_log_size":7938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:22.498969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139853824,"num_objects":2134,"num_object_clones":0,"num_object_copies":6402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14038,"num_write_kb":517504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":125,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.838502+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:42:51.838502+0000","last_peered":"2024-09-13T14:42:51.838502+0000","last_clean":"2024-09-13T14:42:51.838502+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:42:51.838502+0000","last_undegraded":"2024-09-13T14:42:51.838502+0000","last_fullsized":"2024-09-13T14:42:51.838502+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":95,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.440155+0000","last_change":"2024-09-13T14:39:42.764876+0000","last_active":"2024-09-13T14:42:37.440155+0000","last_peered":"2024-09-13T14:42:37.440155+0000","last_clean":"2024-09-13T14:42:37.440155+0000","last_became_active":"2024-09-13T14:39:42.764082+0000","last_became_peered":"2024-09-13T14:39:42.764082+0000","last_unstale":"2024-09-13T14:42:37.440155+0000","last_undegraded":"2024-09-13T14:42:37.440155+0000","last_fullsized":"2024-09-13T14:42:37.440155+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:10:07.458952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"138'13760","reported_seq":13833,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:59.140141+0000","last_change":"2024-09-13T14:39:42.767715+0000","last_active":"2024-09-13T14:42:59.140141+0000","last_peered":"2024-09-13T14:42:59.140141+0000","last_clean":"2024-09-13T14:42:59.140141+0000","last_became_active":"2024-09-13T14:39:42.766678+0000","last_became_peered":"2024-09-13T14:39:42.766678+0000","last_unstale":"2024-09-13T14:42:59.140141+0000","last_undegraded":"2024-09-13T14:42:59.140141+0000","last_fullsized":"2024-09-13T14:42:59.140141+0000","mapping_epoch":107,"log_start":"121'5600","ondisk_log_start":"121'5600","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":8160,"log_dups_size":0,"ondisk_log_size":8160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:30:38.645181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146800640,"num_objects":2240,"num_object_clones":0,"num_object_copies":6720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13760,"num_write_kb":512000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":104,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.437497+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:42:37.437497+0000","last_peered":"2024-09-13T14:42:37.437497+0000","last_clean":"2024-09-13T14:42:37.437497+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:42:37.437497+0000","last_undegraded":"2024-09-13T14:42:37.437497+0000","last_fullsized":"2024-09-13T14:42:37.437497+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":129,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.443154+0000","last_change":"2024-09-13T14:38:13.979677+0000","last_active":"2024-09-13T14:42:37.443154+0000","last_peered":"2024-09-13T14:42:37.443154+0000","last_clean":"2024-09-13T14:42:37.443154+0000","last_became_active":"2024-09-13T14:38:13.979323+0000","last_became_peered":"2024-09-13T14:38:13.979323+0000","last_unstale":"2024-09-13T14:42:37.443154+0000","last_undegraded":"2024-09-13T14:42:37.443154+0000","last_fullsized":"2024-09-13T14:42:37.443154+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:29:03.491254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"138'13889","reported_seq":13963,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:59.158368+0000","last_change":"2024-09-13T14:39:42.766942+0000","last_active":"2024-09-13T14:42:59.158368+0000","last_peered":"2024-09-13T14:42:59.158368+0000","last_clean":"2024-09-13T14:42:59.158368+0000","last_became_active":"2024-09-13T14:39:42.766647+0000","last_became_peered":"2024-09-13T14:39:42.766647+0000","last_unstale":"2024-09-13T14:42:59.158368+0000","last_undegraded":"2024-09-13T14:42:59.158368+0000","last_fullsized":"2024-09-13T14:42:59.158368+0000","mapping_epoch":107,"log_start":"121'5700","ondisk_log_start":"121'5700","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":8189,"log_dups_size":0,"ondisk_log_size":8189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:16.267901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144113664,"num_objects":2199,"num_object_clones":0,"num_object_copies":6597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13889,"num_write_kb":514816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":100,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.437346+0000","last_change":"2024-09-13T14:39:42.774547+0000","last_active":"2024-09-13T14:42:37.437346+0000","last_peered":"2024-09-13T14:42:37.437346+0000","last_clean":"2024-09-13T14:42:37.437346+0000","last_became_active":"2024-09-13T14:39:42.766006+0000","last_became_peered":"2024-09-13T14:39:42.766006+0000","last_unstale":"2024-09-13T14:42:37.437346+0000","last_undegraded":"2024-09-13T14:42:37.437346+0000","last_fullsized":"2024-09-13T14:42:37.437346+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:06:29.632163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":126,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.865104+0000","last_change":"2024-09-13T14:39:42.765763+0000","last_active":"2024-09-13T14:43:01.865104+0000","last_peered":"2024-09-13T14:43:01.865104+0000","last_clean":"2024-09-13T14:43:01.865104+0000","last_became_active":"2024-09-13T14:39:42.765441+0000","last_became_peered":"2024-09-13T14:39:42.765441+0000","last_unstale":"2024-09-13T14:43:01.865104+0000","last_undegraded":"2024-09-13T14:43:01.865104+0000","last_fullsized":"2024-09-13T14:43:01.865104+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:06.483638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.a","version":"138'14373","reported_seq":14447,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:59.188618+0000","last_change":"2024-09-13T14:39:42.763571+0000","last_active":"2024-09-13T14:42:59.188618+0000","last_peered":"2024-09-13T14:42:59.188618+0000","last_clean":"2024-09-13T14:42:59.188618+0000","last_became_active":"2024-09-13T14:39:42.763270+0000","last_became_peered":"2024-09-13T14:39:42.763270+0000","last_unstale":"2024-09-13T14:42:59.188618+0000","last_undegraded":"2024-09-13T14:42:59.188618+0000","last_fullsized":"2024-09-13T14:42:59.188618+0000","mapping_epoch":107,"log_start":"121'6200","ondisk_log_start":"121'6200","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":8173,"log_dups_size":0,"ondisk_log_size":8173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:38:09.281466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151715840,"num_objects":2315,"num_object_clones":0,"num_object_copies":6945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14373,"num_write_kb":534016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":113,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.440556+0000","last_change":"2024-09-13T14:39:42.765558+0000","last_active":"2024-09-13T14:42:37.440556+0000","last_peered":"2024-09-13T14:42:37.440556+0000","last_clean":"2024-09-13T14:42:37.440556+0000","last_became_active":"2024-09-13T14:39:42.765246+0000","last_became_peered":"2024-09-13T14:39:42.765246+0000","last_unstale":"2024-09-13T14:42:37.440556+0000","last_undegraded":"2024-09-13T14:42:37.440556+0000","last_fullsized":"2024-09-13T14:42:37.440556+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:08.640610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":137,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.869100+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:43:01.869100+0000","last_peered":"2024-09-13T14:43:01.869100+0000","last_clean":"2024-09-13T14:43:01.869100+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:43:01.869100+0000","last_undegraded":"2024-09-13T14:43:01.869100+0000","last_fullsized":"2024-09-13T14:43:01.869100+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":265,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.436581+0000","last_change":"2024-09-13T14:40:32.590204+0000","last_active":"2024-09-13T14:42:37.436581+0000","last_peered":"2024-09-13T14:42:37.436581+0000","last_clean":"2024-09-13T14:42:37.436581+0000","last_became_active":"2024-09-13T14:40:22.949830+0000","last_became_peered":"2024-09-13T14:40:22.949830+0000","last_unstale":"2024-09-13T14:42:37.436581+0000","last_undegraded":"2024-09-13T14:42:37.436581+0000","last_fullsized":"2024-09-13T14:42:37.436581+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:32.472827+0000","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.d","version":"138'14097","reported_seq":14172,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:59.163979+0000","last_change":"2024-09-13T14:39:42.759038+0000","last_active":"2024-09-13T14:42:59.163979+0000","last_peered":"2024-09-13T14:42:59.163979+0000","last_clean":"2024-09-13T14:42:59.163979+0000","last_became_active":"2024-09-13T14:39:42.758753+0000","last_became_peered":"2024-09-13T14:39:42.758753+0000","last_unstale":"2024-09-13T14:42:59.163979+0000","last_undegraded":"2024-09-13T14:42:59.163979+0000","last_fullsized":"2024-09-13T14:42:59.163979+0000","mapping_epoch":107,"log_start":"121'5900","ondisk_log_start":"121'5900","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":8197,"log_dups_size":0,"ondisk_log_size":8197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:14:07.557396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148242456,"num_objects":2263,"num_object_clones":0,"num_object_copies":6789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2263,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14097,"num_write_kb":523457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":138,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.839635+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:42:51.839635+0000","last_peered":"2024-09-13T14:42:51.839635+0000","last_clean":"2024-09-13T14:42:51.839635+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:42:51.839635+0000","last_undegraded":"2024-09-13T14:42:51.839635+0000","last_fullsized":"2024-09-13T14:42:51.839635+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":112,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.443678+0000","last_change":"2024-09-13T14:39:42.775584+0000","last_active":"2024-09-13T14:42:37.443678+0000","last_peered":"2024-09-13T14:42:37.443678+0000","last_clean":"2024-09-13T14:42:37.443678+0000","last_became_active":"2024-09-13T14:39:42.775254+0000","last_became_peered":"2024-09-13T14:39:42.775254+0000","last_unstale":"2024-09-13T14:42:37.443678+0000","last_undegraded":"2024-09-13T14:42:37.443678+0000","last_fullsized":"2024-09-13T14:42:37.443678+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:44.607098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":46,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.439989+0000","last_change":"2024-09-13T14:40:27.868402+0000","last_active":"2024-09-13T14:42:37.439989+0000","last_peered":"2024-09-13T14:42:37.439989+0000","last_clean":"2024-09-13T14:42:37.439989+0000","last_became_active":"2024-09-13T14:40:22.966301+0000","last_became_peered":"2024-09-13T14:40:22.966301+0000","last_unstale":"2024-09-13T14:42:37.439989+0000","last_undegraded":"2024-09-13T14:42:37.439989+0000","last_fullsized":"2024-09-13T14:42:37.439989+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:18.660186+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.c","version":"139'14235","reported_seq":14311,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:02.089310+0000","last_change":"2024-09-13T14:39:42.772481+0000","last_active":"2024-09-13T14:43:02.089310+0000","last_peered":"2024-09-13T14:43:02.089310+0000","last_clean":"2024-09-13T14:43:02.089310+0000","last_became_active":"2024-09-13T14:39:42.772211+0000","last_became_peered":"2024-09-13T14:39:42.772211+0000","last_unstale":"2024-09-13T14:43:02.089310+0000","last_undegraded":"2024-09-13T14:43:02.089310+0000","last_fullsized":"2024-09-13T14:43:02.089310+0000","mapping_epoch":107,"log_start":"124'7400","ondisk_log_start":"124'7400","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6835,"log_dups_size":0,"ondisk_log_size":6835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:30:16.860807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133758976,"num_objects":2041,"num_object_clones":0,"num_object_copies":6123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14235,"num_write_kb":520832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":94,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.443291+0000","last_change":"2024-09-13T14:39:42.776751+0000","last_active":"2024-09-13T14:42:37.443291+0000","last_peered":"2024-09-13T14:42:37.443291+0000","last_clean":"2024-09-13T14:42:37.443291+0000","last_became_active":"2024-09-13T14:39:42.774855+0000","last_became_peered":"2024-09-13T14:39:42.774855+0000","last_unstale":"2024-09-13T14:42:37.443291+0000","last_undegraded":"2024-09-13T14:42:37.443291+0000","last_fullsized":"2024-09-13T14:42:37.443291+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:11.022013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":119,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.443713+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:42:37.443713+0000","last_peered":"2024-09-13T14:42:37.443713+0000","last_clean":"2024-09-13T14:42:37.443713+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:42:37.443713+0000","last_undegraded":"2024-09-13T14:42:37.443713+0000","last_fullsized":"2024-09-13T14:42:37.443713+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"138'14242","reported_seq":14281,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.684648+0000","last_change":"2024-09-13T14:39:42.764380+0000","last_active":"2024-09-13T14:43:01.684648+0000","last_peered":"2024-09-13T14:43:01.684648+0000","last_clean":"2024-09-13T14:43:01.684648+0000","last_became_active":"2024-09-13T14:39:42.764127+0000","last_became_peered":"2024-09-13T14:39:42.764127+0000","last_unstale":"2024-09-13T14:43:01.684648+0000","last_undegraded":"2024-09-13T14:43:01.684648+0000","last_fullsized":"2024-09-13T14:43:01.684648+0000","mapping_epoch":107,"log_start":"123'6900","ondisk_log_start":"123'6900","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":7342,"log_dups_size":0,"ondisk_log_size":7342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:37:32.713989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139984896,"num_objects":2136,"num_object_clones":0,"num_object_copies":6408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14242,"num_write_kb":524096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":123,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.867248+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:43:01.867248+0000","last_peered":"2024-09-13T14:43:01.867248+0000","last_clean":"2024-09-13T14:43:01.867248+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:43:01.867248+0000","last_undegraded":"2024-09-13T14:43:01.867248+0000","last_fullsized":"2024-09-13T14:43:01.867248+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":112,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.443585+0000","last_change":"2024-09-13T14:39:42.776187+0000","last_active":"2024-09-13T14:42:37.443585+0000","last_peered":"2024-09-13T14:42:37.443585+0000","last_clean":"2024-09-13T14:42:37.443585+0000","last_became_active":"2024-09-13T14:39:42.775751+0000","last_became_peered":"2024-09-13T14:39:42.775751+0000","last_unstale":"2024-09-13T14:42:37.443585+0000","last_undegraded":"2024-09-13T14:42:37.443585+0000","last_fullsized":"2024-09-13T14:42:37.443585+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:35.155717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":39,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.440092+0000","last_change":"2024-09-13T14:40:25.861254+0000","last_active":"2024-09-13T14:42:37.440092+0000","last_peered":"2024-09-13T14:42:37.440092+0000","last_clean":"2024-09-13T14:42:37.440092+0000","last_became_active":"2024-09-13T14:40:22.957614+0000","last_became_peered":"2024-09-13T14:40:22.957614+0000","last_unstale":"2024-09-13T14:42:37.440092+0000","last_undegraded":"2024-09-13T14:42:37.440092+0000","last_fullsized":"2024-09-13T14:42:37.440092+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:53.388190+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.e","version":"138'14511","reported_seq":14550,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.690325+0000","last_change":"2024-09-13T14:39:42.772007+0000","last_active":"2024-09-13T14:43:01.690325+0000","last_peered":"2024-09-13T14:43:01.690325+0000","last_clean":"2024-09-13T14:43:01.690325+0000","last_became_active":"2024-09-13T14:39:42.771672+0000","last_became_peered":"2024-09-13T14:39:42.771672+0000","last_unstale":"2024-09-13T14:43:01.690325+0000","last_undegraded":"2024-09-13T14:43:01.690325+0000","last_fullsized":"2024-09-13T14:43:01.690325+0000","mapping_epoch":107,"log_start":"124'7600","ondisk_log_start":"124'7600","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6911,"log_dups_size":0,"ondisk_log_size":6911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:26:12.134635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137428992,"num_objects":2097,"num_object_clones":0,"num_object_copies":6291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14511,"num_write_kb":531456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":142,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.839798+0000","last_change":"2024-09-13T14:38:13.981576+0000","last_active":"2024-09-13T14:42:51.839798+0000","last_peered":"2024-09-13T14:42:51.839798+0000","last_clean":"2024-09-13T14:42:51.839798+0000","last_became_active":"2024-09-13T14:38:13.981262+0000","last_became_peered":"2024-09-13T14:38:13.981262+0000","last_unstale":"2024-09-13T14:42:51.839798+0000","last_undegraded":"2024-09-13T14:42:51.839798+0000","last_fullsized":"2024-09-13T14:42:51.839798+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:20.284250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":124,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.866071+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:43:01.866071+0000","last_peered":"2024-09-13T14:43:01.866071+0000","last_clean":"2024-09-13T14:43:01.866071+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:43:01.866071+0000","last_undegraded":"2024-09-13T14:43:01.866071+0000","last_fullsized":"2024-09-13T14:43:01.866071+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.4","version":"138'14122","reported_seq":14161,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.624747+0000","last_change":"2024-09-13T14:39:42.765514+0000","last_active":"2024-09-13T14:43:01.624747+0000","last_peered":"2024-09-13T14:43:01.624747+0000","last_clean":"2024-09-13T14:43:01.624747+0000","last_became_active":"2024-09-13T14:39:42.765209+0000","last_became_peered":"2024-09-13T14:39:42.765209+0000","last_unstale":"2024-09-13T14:43:01.624747+0000","last_undegraded":"2024-09-13T14:43:01.624747+0000","last_fullsized":"2024-09-13T14:43:01.624747+0000","mapping_epoch":107,"log_start":"122'6200","ondisk_log_start":"122'6200","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":7922,"log_dups_size":0,"ondisk_log_size":7922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:59.785672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136839168,"num_objects":2088,"num_object_clones":0,"num_object_copies":6264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14122,"num_write_kb":518720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":256,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.436459+0000","last_change":"2024-09-13T14:40:22.639864+0000","last_active":"2024-09-13T14:42:37.436459+0000","last_peered":"2024-09-13T14:42:37.436459+0000","last_clean":"2024-09-13T14:42:37.436459+0000","last_became_active":"2024-09-13T14:40:20.926503+0000","last_became_peered":"2024-09-13T14:40:20.926503+0000","last_unstale":"2024-09-13T14:42:37.436459+0000","last_undegraded":"2024-09-13T14:42:37.436459+0000","last_fullsized":"2024-09-13T14:42:37.436459+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:22.639760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:22.639760+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:08.523563+0000","scrub_duration":0.0051120610000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":665,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.440517+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:42:37.440517+0000","last_peered":"2024-09-13T14:42:37.440517+0000","last_clean":"2024-09-13T14:42:37.440517+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:42:37.440517+0000","last_undegraded":"2024-09-13T14:42:37.440517+0000","last_fullsized":"2024-09-13T14:42:37.440517+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":99,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.842823+0000","last_change":"2024-09-13T14:39:42.768341+0000","last_active":"2024-09-13T14:42:51.842823+0000","last_peered":"2024-09-13T14:42:51.842823+0000","last_clean":"2024-09-13T14:42:51.842823+0000","last_became_active":"2024-09-13T14:39:42.768034+0000","last_became_peered":"2024-09-13T14:39:42.768034+0000","last_unstale":"2024-09-13T14:42:51.842823+0000","last_undegraded":"2024-09-13T14:42:51.842823+0000","last_fullsized":"2024-09-13T14:42:51.842823+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:28:55.502108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":44,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.443656+0000","last_change":"2024-09-13T14:39:42.772096+0000","last_active":"2024-09-13T14:42:37.443656+0000","last_peered":"2024-09-13T14:42:37.443656+0000","last_clean":"2024-09-13T14:42:37.443656+0000","last_became_active":"2024-09-13T14:39:42.771762+0000","last_became_peered":"2024-09-13T14:39:42.771762+0000","last_unstale":"2024-09-13T14:42:37.443656+0000","last_undegraded":"2024-09-13T14:42:37.443656+0000","last_fullsized":"2024-09-13T14:42:37.443656+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:16.172571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":253,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.865171+0000","last_change":"2024-09-13T14:40:23.784403+0000","last_active":"2024-09-13T14:43:01.865171+0000","last_peered":"2024-09-13T14:43:01.865171+0000","last_clean":"2024-09-13T14:43:01.865171+0000","last_became_active":"2024-09-13T14:40:22.951246+0000","last_became_peered":"2024-09-13T14:40:22.951246+0000","last_unstale":"2024-09-13T14:43:01.865171+0000","last_undegraded":"2024-09-13T14:43:01.865171+0000","last_fullsized":"2024-09-13T14:43:01.865171+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.784263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.784263+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:48:09.437261+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.7","version":"138'13994","reported_seq":14033,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.705010+0000","last_change":"2024-09-13T14:39:42.773740+0000","last_active":"2024-09-13T14:43:01.705010+0000","last_peered":"2024-09-13T14:43:01.705010+0000","last_clean":"2024-09-13T14:43:01.705010+0000","last_became_active":"2024-09-13T14:39:42.773444+0000","last_became_peered":"2024-09-13T14:39:42.773444+0000","last_unstale":"2024-09-13T14:43:01.705010+0000","last_undegraded":"2024-09-13T14:43:01.705010+0000","last_fullsized":"2024-09-13T14:43:01.705010+0000","mapping_epoch":107,"log_start":"122'6600","ondisk_log_start":"122'6600","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":7394,"log_dups_size":0,"ondisk_log_size":7394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:09:16.690630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137363456,"num_objects":2096,"num_object_clones":0,"num_object_copies":6288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13994,"num_write_kb":514880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":349,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.436432+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:42:37.436432+0000","last_peered":"2024-09-13T14:42:37.436432+0000","last_clean":"2024-09-13T14:42:37.436432+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:42:37.436432+0000","last_undegraded":"2024-09-13T14:42:37.436432+0000","last_fullsized":"2024-09-13T14:42:37.436432+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":144,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.867333+0000","last_change":"2024-09-13T14:38:13.977962+0000","last_active":"2024-09-13T14:43:01.867333+0000","last_peered":"2024-09-13T14:43:01.867333+0000","last_clean":"2024-09-13T14:43:01.867333+0000","last_became_active":"2024-09-13T14:38:13.977638+0000","last_became_peered":"2024-09-13T14:38:13.977638+0000","last_unstale":"2024-09-13T14:43:01.867333+0000","last_undegraded":"2024-09-13T14:43:01.867333+0000","last_fullsized":"2024-09-13T14:43:01.867333+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:53:16.902157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":148,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.868198+0000","last_change":"2024-09-13T14:37:40.927444+0000","last_active":"2024-09-13T14:43:01.868198+0000","last_peered":"2024-09-13T14:43:01.868198+0000","last_clean":"2024-09-13T14:43:01.868198+0000","last_became_active":"2024-09-13T14:37:40.927140+0000","last_became_peered":"2024-09-13T14:37:40.927140+0000","last_unstale":"2024-09-13T14:43:01.868198+0000","last_undegraded":"2024-09-13T14:43:01.868198+0000","last_fullsized":"2024-09-13T14:43:01.868198+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:32:54.207568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":265,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.436422+0000","last_change":"2024-09-13T14:40:28.602999+0000","last_active":"2024-09-13T14:42:37.436422+0000","last_peered":"2024-09-13T14:42:37.436422+0000","last_clean":"2024-09-13T14:42:37.436422+0000","last_became_active":"2024-09-13T14:40:22.949002+0000","last_became_peered":"2024-09-13T14:40:22.949002+0000","last_unstale":"2024-09-13T14:42:37.436422+0000","last_undegraded":"2024-09-13T14:42:37.436422+0000","last_fullsized":"2024-09-13T14:42:37.436422+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:28.602942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:28.602942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:29:46.037596+0000","scrub_duration":0.00945176,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.6","version":"138'13901","reported_seq":13975,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:59.158050+0000","last_change":"2024-09-13T14:39:42.759808+0000","last_active":"2024-09-13T14:42:59.158050+0000","last_peered":"2024-09-13T14:42:59.158050+0000","last_clean":"2024-09-13T14:42:59.158050+0000","last_became_active":"2024-09-13T14:39:42.759199+0000","last_became_peered":"2024-09-13T14:39:42.759199+0000","last_unstale":"2024-09-13T14:42:59.158050+0000","last_undegraded":"2024-09-13T14:42:59.158050+0000","last_fullsized":"2024-09-13T14:42:59.158050+0000","mapping_epoch":107,"log_start":"121'5700","ondisk_log_start":"121'5700","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":8201,"log_dups_size":0,"ondisk_log_size":8201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:55:55.573511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145162240,"num_objects":2215,"num_object_clones":0,"num_object_copies":6645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13901,"num_write_kb":515712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":546,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.834563+0000","last_change":"2024-09-13T14:38:13.978679+0000","last_active":"2024-09-13T14:42:51.834563+0000","last_peered":"2024-09-13T14:42:51.834563+0000","last_clean":"2024-09-13T14:42:51.834563+0000","last_became_active":"2024-09-13T14:38:13.978171+0000","last_became_peered":"2024-09-13T14:38:13.978171+0000","last_unstale":"2024-09-13T14:42:51.834563+0000","last_undegraded":"2024-09-13T14:42:51.834563+0000","last_fullsized":"2024-09-13T14:42:51.834563+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:57.691198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":131,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.865250+0000","last_change":"2024-09-13T14:39:42.769691+0000","last_active":"2024-09-13T14:43:01.865250+0000","last_peered":"2024-09-13T14:43:01.865250+0000","last_clean":"2024-09-13T14:43:01.865250+0000","last_became_active":"2024-09-13T14:39:42.769266+0000","last_became_peered":"2024-09-13T14:39:42.769266+0000","last_unstale":"2024-09-13T14:43:01.865250+0000","last_undegraded":"2024-09-13T14:43:01.865250+0000","last_fullsized":"2024-09-13T14:43:01.865250+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:11.637732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":157,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.865933+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:43:01.865933+0000","last_peered":"2024-09-13T14:43:01.865933+0000","last_clean":"2024-09-13T14:43:01.865933+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:43:01.865933+0000","last_undegraded":"2024-09-13T14:43:01.865933+0000","last_fullsized":"2024-09-13T14:43:01.865933+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":39,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.440129+0000","last_change":"2024-09-13T14:40:26.896691+0000","last_active":"2024-09-13T14:42:37.440129+0000","last_peered":"2024-09-13T14:42:37.440129+0000","last_clean":"2024-09-13T14:42:37.440129+0000","last_became_active":"2024-09-13T14:40:22.949772+0000","last_became_peered":"2024-09-13T14:40:22.949772+0000","last_unstale":"2024-09-13T14:42:37.440129+0000","last_undegraded":"2024-09-13T14:42:37.440129+0000","last_fullsized":"2024-09-13T14:42:37.440129+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:26.896613+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:26.896613+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:01.615433+0000","scrub_duration":0.004120828,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.5","version":"139'14159","reported_seq":14235,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:03.409063+0000","last_change":"2024-09-13T14:39:42.776210+0000","last_active":"2024-09-13T14:43:03.409063+0000","last_peered":"2024-09-13T14:43:03.409063+0000","last_clean":"2024-09-13T14:43:03.409063+0000","last_became_active":"2024-09-13T14:39:42.775823+0000","last_became_peered":"2024-09-13T14:39:42.775823+0000","last_unstale":"2024-09-13T14:43:03.409063+0000","last_undegraded":"2024-09-13T14:43:03.409063+0000","last_fullsized":"2024-09-13T14:43:03.409063+0000","mapping_epoch":107,"log_start":"125'8000","ondisk_log_start":"125'8000","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6159,"log_dups_size":0,"ondisk_log_size":6159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:39.020681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130088960,"num_objects":1985,"num_object_clones":0,"num_object_copies":5955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14159,"num_write_kb":516608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":729,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.440292+0000","last_change":"2024-09-13T14:39:42.805055+0000","last_active":"2024-09-13T14:42:37.440292+0000","last_peered":"2024-09-13T14:42:37.440292+0000","last_clean":"2024-09-13T14:42:37.440292+0000","last_became_active":"2024-09-13T14:39:42.762456+0000","last_became_peered":"2024-09-13T14:39:42.762456+0000","last_unstale":"2024-09-13T14:42:37.440292+0000","last_undegraded":"2024-09-13T14:42:37.440292+0000","last_fullsized":"2024-09-13T14:42:37.440292+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:54:40.839991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":113,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.443385+0000","last_change":"2024-09-13T14:39:42.775159+0000","last_active":"2024-09-13T14:42:37.443385+0000","last_peered":"2024-09-13T14:42:37.443385+0000","last_clean":"2024-09-13T14:42:37.443385+0000","last_became_active":"2024-09-13T14:39:42.774841+0000","last_became_peered":"2024-09-13T14:39:42.774841+0000","last_unstale":"2024-09-13T14:42:37.443385+0000","last_undegraded":"2024-09-13T14:42:37.443385+0000","last_fullsized":"2024-09-13T14:42:37.443385+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:35:45.579397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":135,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.865827+0000","last_change":"2024-09-13T14:38:13.981294+0000","last_active":"2024-09-13T14:43:01.865827+0000","last_peered":"2024-09-13T14:43:01.865827+0000","last_clean":"2024-09-13T14:43:01.865827+0000","last_became_active":"2024-09-13T14:38:13.980975+0000","last_became_peered":"2024-09-13T14:38:13.980975+0000","last_unstale":"2024-09-13T14:43:01.865827+0000","last_undegraded":"2024-09-13T14:43:01.865827+0000","last_fullsized":"2024-09-13T14:43:01.865827+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:17.545179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":60,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.840357+0000","last_change":"2024-09-13T14:40:23.093425+0000","last_active":"2024-09-13T14:42:51.840357+0000","last_peered":"2024-09-13T14:42:51.840357+0000","last_clean":"2024-09-13T14:42:51.840357+0000","last_became_active":"2024-09-13T14:40:22.947358+0000","last_became_peered":"2024-09-13T14:40:22.947358+0000","last_unstale":"2024-09-13T14:42:51.840357+0000","last_undegraded":"2024-09-13T14:42:51.840357+0000","last_fullsized":"2024-09-13T14:42:51.840357+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.093356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.093356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:38:28.248105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.3","version":"139'14111","reported_seq":14187,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:03.426634+0000","last_change":"2024-09-13T14:39:42.770363+0000","last_active":"2024-09-13T14:43:03.426634+0000","last_peered":"2024-09-13T14:43:03.426634+0000","last_clean":"2024-09-13T14:43:03.426634+0000","last_became_active":"2024-09-13T14:39:42.770092+0000","last_became_peered":"2024-09-13T14:39:42.770092+0000","last_unstale":"2024-09-13T14:43:03.426634+0000","last_undegraded":"2024-09-13T14:43:03.426634+0000","last_fullsized":"2024-09-13T14:43:03.426634+0000","mapping_epoch":107,"log_start":"125'7900","ondisk_log_start":"125'7900","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6211,"log_dups_size":0,"ondisk_log_size":6211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:20:20.375252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127205376,"num_objects":1941,"num_object_clones":0,"num_object_copies":5823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14111,"num_write_kb":513664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.440778+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:42:37.440778+0000","last_peered":"2024-09-13T14:42:37.440778+0000","last_clean":"2024-09-13T14:42:37.440778+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:42:37.440778+0000","last_undegraded":"2024-09-13T14:42:37.440778+0000","last_fullsized":"2024-09-13T14:42:37.440778+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":641,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.865908+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:43:01.865908+0000","last_peered":"2024-09-13T14:43:01.865908+0000","last_clean":"2024-09-13T14:43:01.865908+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:43:01.865908+0000","last_undegraded":"2024-09-13T14:43:01.865908+0000","last_fullsized":"2024-09-13T14:43:01.865908+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":113,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.865384+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:43:01.865384+0000","last_peered":"2024-09-13T14:43:01.865384+0000","last_clean":"2024-09-13T14:43:01.865384+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:43:01.865384+0000","last_undegraded":"2024-09-13T14:43:01.865384+0000","last_fullsized":"2024-09-13T14:43:01.865384+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":208,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.867785+0000","last_change":"2024-09-13T14:40:23.334441+0000","last_active":"2024-09-13T14:43:01.867785+0000","last_peered":"2024-09-13T14:43:01.867785+0000","last_clean":"2024-09-13T14:43:01.867785+0000","last_became_active":"2024-09-13T14:40:22.952908+0000","last_became_peered":"2024-09-13T14:40:22.952908+0000","last_unstale":"2024-09-13T14:43:01.867785+0000","last_undegraded":"2024-09-13T14:43:01.867785+0000","last_fullsized":"2024-09-13T14:43:01.867785+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.334379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.334379+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:58:19.770506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"139'14369","reported_seq":14445,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:03.423806+0000","last_change":"2024-09-13T14:39:42.770399+0000","last_active":"2024-09-13T14:43:03.423806+0000","last_peered":"2024-09-13T14:43:03.423806+0000","last_clean":"2024-09-13T14:43:03.423806+0000","last_became_active":"2024-09-13T14:39:42.770074+0000","last_became_peered":"2024-09-13T14:39:42.770074+0000","last_unstale":"2024-09-13T14:43:03.423806+0000","last_undegraded":"2024-09-13T14:43:03.423806+0000","last_fullsized":"2024-09-13T14:43:03.423806+0000","mapping_epoch":107,"log_start":"126'8200","ondisk_log_start":"126'8200","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6169,"log_dups_size":0,"ondisk_log_size":6169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:28:03.030813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128909312,"num_objects":1967,"num_object_clones":0,"num_object_copies":5901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14369,"num_write_kb":522752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":124,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.865860+0000","last_change":"2024-09-13T14:37:40.921861+0000","last_active":"2024-09-13T14:43:01.865860+0000","last_peered":"2024-09-13T14:43:01.865860+0000","last_clean":"2024-09-13T14:43:01.865860+0000","last_became_active":"2024-09-13T14:37:40.921587+0000","last_became_peered":"2024-09-13T14:37:40.921587+0000","last_unstale":"2024-09-13T14:43:01.865860+0000","last_undegraded":"2024-09-13T14:43:01.865860+0000","last_fullsized":"2024-09-13T14:43:01.865860+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:24.619352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":351,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.437431+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:42:37.437431+0000","last_peered":"2024-09-13T14:42:37.437431+0000","last_clean":"2024-09-13T14:42:37.437431+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:42:37.437431+0000","last_undegraded":"2024-09-13T14:42:37.437431+0000","last_fullsized":"2024-09-13T14:42:37.437431+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":126,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.868158+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:43:01.868158+0000","last_peered":"2024-09-13T14:43:01.868158+0000","last_clean":"2024-09-13T14:43:01.868158+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:43:01.868158+0000","last_undegraded":"2024-09-13T14:43:01.868158+0000","last_fullsized":"2024-09-13T14:43:01.868158+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":39,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.443656+0000","last_change":"2024-09-13T14:40:30.500108+0000","last_active":"2024-09-13T14:42:37.443656+0000","last_peered":"2024-09-13T14:42:37.443656+0000","last_clean":"2024-09-13T14:42:37.443656+0000","last_became_active":"2024-09-13T14:40:22.949882+0000","last_became_peered":"2024-09-13T14:40:22.949882+0000","last_unstale":"2024-09-13T14:42:37.443656+0000","last_undegraded":"2024-09-13T14:42:37.443656+0000","last_fullsized":"2024-09-13T14:42:37.443656+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:40:30.500046+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:22:58.480202+0000","scrub_duration":0.00248832,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1","version":"138'14267","reported_seq":14306,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.700717+0000","last_change":"2024-09-13T14:39:42.772158+0000","last_active":"2024-09-13T14:43:01.700717+0000","last_peered":"2024-09-13T14:43:01.700717+0000","last_clean":"2024-09-13T14:43:01.700717+0000","last_became_active":"2024-09-13T14:39:42.771189+0000","last_became_peered":"2024-09-13T14:39:42.771189+0000","last_unstale":"2024-09-13T14:43:01.700717+0000","last_undegraded":"2024-09-13T14:43:01.700717+0000","last_fullsized":"2024-09-13T14:43:01.700717+0000","mapping_epoch":107,"log_start":"124'7400","ondisk_log_start":"124'7400","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6867,"log_dups_size":0,"ondisk_log_size":6867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:58:08.882288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148045824,"num_objects":2259,"num_object_clones":0,"num_object_copies":6777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14267,"num_write_kb":528832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":138,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.840443+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:42:51.840443+0000","last_peered":"2024-09-13T14:42:51.840443+0000","last_clean":"2024-09-13T14:42:51.840443+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:42:51.840443+0000","last_undegraded":"2024-09-13T14:42:51.840443+0000","last_fullsized":"2024-09-13T14:42:51.840443+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":462,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.866304+0000","last_change":"2024-09-13T14:39:44.990216+0000","last_active":"2024-09-13T14:43:01.866304+0000","last_peered":"2024-09-13T14:43:01.866304+0000","last_clean":"2024-09-13T14:43:01.866304+0000","last_became_active":"2024-09-13T14:39:44.989433+0000","last_became_peered":"2024-09-13T14:39:44.989433+0000","last_unstale":"2024-09-13T14:43:01.866304+0000","last_undegraded":"2024-09-13T14:43:01.866304+0000","last_fullsized":"2024-09-13T14:43:01.866304+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:00.426042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":135,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.839971+0000","last_change":"2024-09-13T14:38:13.979255+0000","last_active":"2024-09-13T14:42:51.839971+0000","last_peered":"2024-09-13T14:42:51.839971+0000","last_clean":"2024-09-13T14:42:51.839971+0000","last_became_active":"2024-09-13T14:38:13.978868+0000","last_became_peered":"2024-09-13T14:38:13.978868+0000","last_unstale":"2024-09-13T14:42:51.839971+0000","last_undegraded":"2024-09-13T14:42:51.839971+0000","last_fullsized":"2024-09-13T14:42:51.839971+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:15:26.003538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":65,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.839944+0000","last_change":"2024-09-13T14:40:24.124750+0000","last_active":"2024-09-13T14:42:51.839944+0000","last_peered":"2024-09-13T14:42:51.839944+0000","last_clean":"2024-09-13T14:42:51.839944+0000","last_became_active":"2024-09-13T14:40:22.958820+0000","last_became_peered":"2024-09-13T14:40:22.958820+0000","last_unstale":"2024-09-13T14:42:51.839944+0000","last_undegraded":"2024-09-13T14:42:51.839944+0000","last_fullsized":"2024-09-13T14:42:51.839944+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:24.124531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:24.124531+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:36:35.757492+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"138'14301","reported_seq":14375,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.730691+0000","last_change":"2024-09-13T14:39:42.770912+0000","last_active":"2024-09-13T14:43:01.730691+0000","last_peered":"2024-09-13T14:43:01.730691+0000","last_clean":"2024-09-13T14:43:01.730691+0000","last_became_active":"2024-09-13T14:39:42.770646+0000","last_became_peered":"2024-09-13T14:39:42.770646+0000","last_unstale":"2024-09-13T14:43:01.730691+0000","last_undegraded":"2024-09-13T14:43:01.730691+0000","last_fullsized":"2024-09-13T14:43:01.730691+0000","mapping_epoch":107,"log_start":"124'7400","ondisk_log_start":"124'7400","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:41.740366+0000","last_clean_scrub_stamp":"2024-09-13T14:39:41.740366+0000","objects_scrubbed":0,"log_size":6901,"log_dups_size":0,"ondisk_log_size":6901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:54:13.100820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133890048,"num_objects":2043,"num_object_clones":0,"num_object_copies":6129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14301,"num_write_kb":523008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":148,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.867936+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:43:01.867936+0000","last_peered":"2024-09-13T14:43:01.867936+0000","last_clean":"2024-09-13T14:43:01.867936+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:43:01.867936+0000","last_undegraded":"2024-09-13T14:43:01.867936+0000","last_fullsized":"2024-09-13T14:43:01.867936+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1055,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.437358+0000","last_change":"2024-09-13T14:39:44.990389+0000","last_active":"2024-09-13T14:42:37.437358+0000","last_peered":"2024-09-13T14:42:37.437358+0000","last_clean":"2024-09-13T14:42:37.437358+0000","last_became_active":"2024-09-13T14:39:44.989805+0000","last_became_peered":"2024-09-13T14:39:44.989805+0000","last_unstale":"2024-09-13T14:42:37.437358+0000","last_undegraded":"2024-09-13T14:42:37.437358+0000","last_fullsized":"2024-09-13T14:42:37.437358+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:15.392445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":95,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.443076+0000","last_change":"2024-09-13T14:39:42.769954+0000","last_active":"2024-09-13T14:42:37.443076+0000","last_peered":"2024-09-13T14:42:37.443076+0000","last_clean":"2024-09-13T14:42:37.443076+0000","last_became_active":"2024-09-13T14:39:42.769635+0000","last_became_peered":"2024-09-13T14:39:42.769635+0000","last_unstale":"2024-09-13T14:42:37.443076+0000","last_undegraded":"2024-09-13T14:42:37.443076+0000","last_fullsized":"2024-09-13T14:42:37.443076+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:51.588071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":112,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.443366+0000","last_change":"2024-09-13T14:39:42.775357+0000","last_active":"2024-09-13T14:42:37.443366+0000","last_peered":"2024-09-13T14:42:37.443366+0000","last_clean":"2024-09-13T14:42:37.443366+0000","last_became_active":"2024-09-13T14:39:42.775041+0000","last_became_peered":"2024-09-13T14:39:42.775041+0000","last_unstale":"2024-09-13T14:42:37.443366+0000","last_undegraded":"2024-09-13T14:42:37.443366+0000","last_fullsized":"2024-09-13T14:42:37.443366+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:44.768910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":139,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.867664+0000","last_change":"2024-09-13T14:39:42.787151+0000","last_active":"2024-09-13T14:43:01.867664+0000","last_peered":"2024-09-13T14:43:01.867664+0000","last_clean":"2024-09-13T14:43:01.867664+0000","last_became_active":"2024-09-13T14:39:42.771712+0000","last_became_peered":"2024-09-13T14:39:42.771712+0000","last_unstale":"2024-09-13T14:43:01.867664+0000","last_undegraded":"2024-09-13T14:43:01.867664+0000","last_fullsized":"2024-09-13T14:43:01.867664+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:54.216663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":131,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.865255+0000","last_change":"2024-09-13T14:39:42.772974+0000","last_active":"2024-09-13T14:43:01.865255+0000","last_peered":"2024-09-13T14:43:01.865255+0000","last_clean":"2024-09-13T14:43:01.865255+0000","last_became_active":"2024-09-13T14:39:42.771449+0000","last_became_peered":"2024-09-13T14:39:42.771449+0000","last_unstale":"2024-09-13T14:43:01.865255+0000","last_undegraded":"2024-09-13T14:43:01.865255+0000","last_fullsized":"2024-09-13T14:43:01.865255+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:46.441079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":118,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.437278+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:42:37.437278+0000","last_peered":"2024-09-13T14:42:37.437278+0000","last_clean":"2024-09-13T14:42:37.437278+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:42:37.437278+0000","last_undegraded":"2024-09-13T14:42:37.437278+0000","last_fullsized":"2024-09-13T14:42:37.437278+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.440878+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:42:37.440878+0000","last_peered":"2024-09-13T14:42:37.440878+0000","last_clean":"2024-09-13T14:42:37.440878+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:42:37.440878+0000","last_undegraded":"2024-09-13T14:42:37.440878+0000","last_fullsized":"2024-09-13T14:42:37.440878+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":99,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.839902+0000","last_change":"2024-09-13T14:39:42.767764+0000","last_active":"2024-09-13T14:42:51.839902+0000","last_peered":"2024-09-13T14:42:51.839902+0000","last_clean":"2024-09-13T14:42:51.839902+0000","last_became_active":"2024-09-13T14:39:42.767450+0000","last_became_peered":"2024-09-13T14:39:42.767450+0000","last_unstale":"2024-09-13T14:42:51.839902+0000","last_undegraded":"2024-09-13T14:42:51.839902+0000","last_fullsized":"2024-09-13T14:42:51.839902+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:47.729382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":136,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.868690+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:43:01.868690+0000","last_peered":"2024-09-13T14:43:01.868690+0000","last_clean":"2024-09-13T14:43:01.868690+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:43:01.868690+0000","last_undegraded":"2024-09-13T14:43:01.868690+0000","last_fullsized":"2024-09-13T14:43:01.868690+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":148,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.840051+0000","last_change":"2024-09-13T14:39:42.772266+0000","last_active":"2024-09-13T14:42:51.840051+0000","last_peered":"2024-09-13T14:42:51.840051+0000","last_clean":"2024-09-13T14:42:51.840051+0000","last_became_active":"2024-09-13T14:39:42.771880+0000","last_became_peered":"2024-09-13T14:39:42.771880+0000","last_unstale":"2024-09-13T14:42:51.840051+0000","last_undegraded":"2024-09-13T14:42:51.840051+0000","last_fullsized":"2024-09-13T14:42:51.840051+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:41:01.160105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":93,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.445381+0000","last_change":"2024-09-13T14:39:42.773890+0000","last_active":"2024-09-13T14:42:37.445381+0000","last_peered":"2024-09-13T14:42:37.445381+0000","last_clean":"2024-09-13T14:42:37.445381+0000","last_became_active":"2024-09-13T14:39:42.773523+0000","last_became_peered":"2024-09-13T14:39:42.773523+0000","last_unstale":"2024-09-13T14:42:37.445381+0000","last_undegraded":"2024-09-13T14:42:37.445381+0000","last_fullsized":"2024-09-13T14:42:37.445381+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:13.156261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":159,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.839925+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:42:51.839925+0000","last_peered":"2024-09-13T14:42:51.839925+0000","last_clean":"2024-09-13T14:42:51.839925+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:42:51.839925+0000","last_undegraded":"2024-09-13T14:42:51.839925+0000","last_fullsized":"2024-09-13T14:42:51.839925+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":137,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.868120+0000","last_change":"2024-09-13T14:38:13.979604+0000","last_active":"2024-09-13T14:43:01.868120+0000","last_peered":"2024-09-13T14:43:01.868120+0000","last_clean":"2024-09-13T14:43:01.868120+0000","last_became_active":"2024-09-13T14:38:13.979288+0000","last_became_peered":"2024-09-13T14:38:13.979288+0000","last_unstale":"2024-09-13T14:43:01.868120+0000","last_undegraded":"2024-09-13T14:43:01.868120+0000","last_fullsized":"2024-09-13T14:43:01.868120+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:21.031527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":124,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.866114+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:43:01.866114+0000","last_peered":"2024-09-13T14:43:01.866114+0000","last_clean":"2024-09-13T14:43:01.866114+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:43:01.866114+0000","last_undegraded":"2024-09-13T14:43:01.866114+0000","last_fullsized":"2024-09-13T14:43:01.866114+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":154,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.840261+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:42:51.840261+0000","last_peered":"2024-09-13T14:42:51.840261+0000","last_clean":"2024-09-13T14:42:51.840261+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:42:51.840261+0000","last_undegraded":"2024-09-13T14:42:51.840261+0000","last_fullsized":"2024-09-13T14:42:51.840261+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":121,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.443480+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:42:37.443480+0000","last_peered":"2024-09-13T14:42:37.443480+0000","last_clean":"2024-09-13T14:42:37.443480+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:42:37.443480+0000","last_undegraded":"2024-09-13T14:42:37.443480+0000","last_fullsized":"2024-09-13T14:42:37.443480+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":161,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.834922+0000","last_change":"2024-09-13T14:38:13.976950+0000","last_active":"2024-09-13T14:42:51.834922+0000","last_peered":"2024-09-13T14:42:51.834922+0000","last_clean":"2024-09-13T14:42:51.834922+0000","last_became_active":"2024-09-13T14:38:13.976639+0000","last_became_peered":"2024-09-13T14:38:13.976639+0000","last_unstale":"2024-09-13T14:42:51.834922+0000","last_undegraded":"2024-09-13T14:42:51.834922+0000","last_fullsized":"2024-09-13T14:42:51.834922+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:24.126890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":128,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.868066+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:43:01.868066+0000","last_peered":"2024-09-13T14:43:01.868066+0000","last_clean":"2024-09-13T14:43:01.868066+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:43:01.868066+0000","last_undegraded":"2024-09-13T14:43:01.868066+0000","last_fullsized":"2024-09-13T14:43:01.868066+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":119,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.443312+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:42:37.443312+0000","last_peered":"2024-09-13T14:42:37.443312+0000","last_clean":"2024-09-13T14:42:37.443312+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:42:37.443312+0000","last_undegraded":"2024-09-13T14:42:37.443312+0000","last_fullsized":"2024-09-13T14:42:37.443312+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":105,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.440472+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:42:37.440472+0000","last_peered":"2024-09-13T14:42:37.440472+0000","last_clean":"2024-09-13T14:42:37.440472+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:42:37.440472+0000","last_undegraded":"2024-09-13T14:42:37.440472+0000","last_fullsized":"2024-09-13T14:42:37.440472+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":119,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.443404+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:42:37.443404+0000","last_peered":"2024-09-13T14:42:37.443404+0000","last_clean":"2024-09-13T14:42:37.443404+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:42:37.443404+0000","last_undegraded":"2024-09-13T14:42:37.443404+0000","last_fullsized":"2024-09-13T14:42:37.443404+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":130,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.440713+0000","last_change":"2024-09-13T14:39:42.778413+0000","last_active":"2024-09-13T14:42:37.440713+0000","last_peered":"2024-09-13T14:42:37.440713+0000","last_clean":"2024-09-13T14:42:37.440713+0000","last_became_active":"2024-09-13T14:39:42.766624+0000","last_became_peered":"2024-09-13T14:39:42.766624+0000","last_unstale":"2024-09-13T14:42:37.440713+0000","last_undegraded":"2024-09-13T14:42:37.440713+0000","last_fullsized":"2024-09-13T14:42:37.440713+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:13.435604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.440258+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:42:37.440258+0000","last_peered":"2024-09-13T14:42:37.440258+0000","last_clean":"2024-09-13T14:42:37.440258+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:42:37.440258+0000","last_undegraded":"2024-09-13T14:42:37.440258+0000","last_fullsized":"2024-09-13T14:42:37.440258+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":154,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-09-13T14:42:51.840331+0000","last_change":"2024-09-13T14:39:42.767451+0000","last_active":"2024-09-13T14:42:51.840331+0000","last_peered":"2024-09-13T14:42:51.840331+0000","last_clean":"2024-09-13T14:42:51.840331+0000","last_became_active":"2024-09-13T14:39:42.767086+0000","last_became_peered":"2024-09-13T14:39:42.767086+0000","last_unstale":"2024-09-13T14:42:51.840331+0000","last_undegraded":"2024-09-13T14:42:51.840331+0000","last_fullsized":"2024-09-13T14:42:51.840331+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:02.674069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":105,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:42:37.443017+0000","last_change":"2024-09-13T14:39:42.768653+0000","last_active":"2024-09-13T14:42:37.443017+0000","last_peered":"2024-09-13T14:42:37.443017+0000","last_clean":"2024-09-13T14:42:37.443017+0000","last_became_active":"2024-09-13T14:39:42.768333+0000","last_became_peered":"2024-09-13T14:39:42.768333+0000","last_unstale":"2024-09-13T14:42:37.443017+0000","last_undegraded":"2024-09-13T14:42:37.443017+0000","last_fullsized":"2024-09-13T14:42:37.443017+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:25.655793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":140,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-09-13T14:43:01.868456+0000","last_change":"2024-09-13T14:37:40.924860+0000","last_active":"2024-09-13T14:43:01.868456+0000","last_peered":"2024-09-13T14:43:01.868456+0000","last_clean":"2024-09-13T14:43:01.868456+0000","last_became_active":"2024-09-13T14:37:40.924522+0000","last_became_peered":"2024-09-13T14:37:40.924522+0000","last_unstale":"2024-09-13T14:43:01.868456+0000","last_undegraded":"2024-09-13T14:43:01.868456+0000","last_fullsized":"2024-09-13T14:43:01.868456+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:10.340511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":2229403672,"num_objects":34019,"num_object_clones":0,"num_object_copies":102057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34019,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":226369,"num_write_kb":8332353,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6618296320,"data_stored":6618284104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117569,"ondisk_log_size":117569,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":9648,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397709,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":999012,"kb_used_data":689180,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92746140,"statfs":{"total":95995035648,"available":94972047360,"internally_reserved":0,"allocated":705720320,"data_stored":704895546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":184683593938,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790167,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1517848,"kb_used_data":1220496,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":92227304,"statfs":{"total":95995035648,"available":94440759296,"internally_reserved":0,"allocated":1249787904,"data_stored":1248971431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986396,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1399124,"kb_used_data":1093388,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":92346028,"statfs":{"total":95995035648,"available":94562332672,"internally_reserved":0,"allocated":1119629312,"data_stored":1118816098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598844,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":642508,"kb_used_data":556612,"kb_used_omap":6,"kb_used_meta":85817,"kb_avail":93102644,"statfs":{"total":95995035648,"available":95337107456,"internally_reserved":0,"allocated":569970688,"data_stored":569165918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":87877600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762370,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1323492,"kb_used_data":1179356,"kb_used_omap":8,"kb_used_meta":144119,"kb_avail":92421660,"statfs":{"total":95995035648,"available":94639779840,"internally_reserved":0,"allocated":1207660544,"data_stored":1206836522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":147578314},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,4,2,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":94,"seq":403726925897,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905056,"kb_used_data":797012,"kb_used_omap":7,"kb_used_meta":108024,"kb_avail":92840096,"statfs":{"total":95995035648,"available":95068258304,"internally_reserved":0,"allocated":816140288,"data_stored":815327354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":110617289},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122137,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1179636,"kb_used_data":1027180,"kb_used_omap":7,"kb_used_meta":152440,"kb_avail":92565516,"statfs":{"total":95995035648,"available":94787088384,"internally_reserved":0,"allocated":1051832320,"data_stored":1051000236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":156099151},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1032454144,"data_stored":1032454144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":809762816,"data_stored":809762816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1179062272,"data_stored":1179058200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":556400640,"data_stored":556400640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1111883776,"data_stored":1111883776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1230442496,"data_stored":1230438424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":698290176,"data_stored":698286104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:43:05.328 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 11 2024-09-13T14:43:05.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.356 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:05.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:05.458 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:05.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.507 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:05.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:05.517 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:43:05.551 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:43:05.551 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:43:05.551 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:43:05.551 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:43:05.551 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:43:05.608 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:05.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.657 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:05.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:05.758 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:05.807 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:05.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:05.909 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:05.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.944 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:05.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:06.045 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:06.084 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:06.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:06.175 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:05 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:06.176 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:05 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:06.185 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:06.211 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:06.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:06.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:05 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:06.314 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:06.336 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:06.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:06.437 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:06.471 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:06.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:06.573 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:06.624 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:43:06.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.639 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:06.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:06.740 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:06.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.781 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:06.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:06.882 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:06.910 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:06.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:07.012 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:07.039 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:07.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:06 smithi096 ceph-mon[98901]: pgmap v375: 99 pgs: 99 active+clean; 2.0 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 986 op/s 2024-09-13T14:43:07.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:06 smithi096 ceph-mon[102928]: pgmap v375: 99 pgs: 99 active+clean; 2.0 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 986 op/s 2024-09-13T14:43:07.141 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:07.170 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:07.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:07.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:06 smithi204 ceph-mon[87071]: pgmap v375: 99 pgs: 99 active+clean; 2.0 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 986 op/s 2024-09-13T14:43:07.272 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:07.298 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:07.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:07.400 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:07.425 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:07.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:07.527 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:07.559 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:07.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:07.661 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:07.706 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:07.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:07.808 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:07.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.841 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:07.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:07.943 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:07.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.984 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:07.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:08.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:43:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:43:07] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.51.0" 2024-09-13T14:43:08.085 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:08.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.134 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:08.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:08.214 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:08.234 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:08.235 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:08.237 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:08.259 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:08.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:08.361 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:08.391 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:08.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:08.493 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:08.527 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:08.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:08.629 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:08.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.667 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:08.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:08.769 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:08.838 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:08.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:08.940 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:08.967 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:08.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:09.066 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:43:09.066 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:43:09.067 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:08 smithi096 ceph-mon[98901]: pgmap v376: 99 pgs: 99 active+clean; 1.8 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 966 op/s 2024-09-13T14:43:09.067 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:08 smithi096 ceph-mon[102928]: pgmap v376: 99 pgs: 99 active+clean; 1.8 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 966 op/s 2024-09-13T14:43:09.069 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:09.110 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:43:09.110 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:43:09.113 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:09.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:09.142 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:43:09.142 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:43:09.179 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:43:09.180 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:43:09.215 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:09.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:08 smithi204 ceph-mon[87071]: pgmap v376: 99 pgs: 99 active+clean; 1.8 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 966 op/s 2024-09-13T14:43:09.218 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:43:09.219 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:43:09.249 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:09.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:09.251 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:43:09.252 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:43:09.283 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:43:09.283 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:43:09.315 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:43:09.316 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:43:09.350 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:09.376 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:09.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:09.478 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:09.511 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:09.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:09.612 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:09.646 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:09.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:09.748 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:09.782 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:09.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:09.884 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:09.914 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:09.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:10.016 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:10.041 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:10.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:10.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:09 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4202255329' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:10.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:09 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4202255329' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:10.143 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:10.167 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:10.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:10.213 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:09 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4202255329' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:10.269 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:10.305 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:10.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:10.407 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:10.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.454 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:10.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:10.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:43:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:43:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:10.556 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:10.581 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:10.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:10.683 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:10.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.718 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:10.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:10.818 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:10.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.846 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:10.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:10.947 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:10.987 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:10.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:11.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:10 smithi096 ceph-mon[98901]: pgmap v377: 99 pgs: 99 active+clean; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 952 op/s 2024-09-13T14:43:11.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:10 smithi096 ceph-mon[102928]: pgmap v377: 99 pgs: 99 active+clean; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 952 op/s 2024-09-13T14:43:11.089 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:11.124 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:11.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:11.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:10 smithi204 ceph-mon[87071]: pgmap v377: 99 pgs: 99 active+clean; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 952 op/s 2024-09-13T14:43:11.225 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:11.260 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:11.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:11.362 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:11.391 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:11.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:11.493 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:11.521 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:11.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:11.622 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:11.653 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:11.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:11.754 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:11.786 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:11.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:11.888 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:11.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.926 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:11.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:12.027 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:12.059 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:12.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:12.158 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:43:12.161 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:12.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.243 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:12.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:12.344 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:12.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.380 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:12.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:12.481 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:12.524 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:12.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:12.626 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:12.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.660 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:12.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:12.761 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:12.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.792 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:12.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:12.893 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:12.933 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:12.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:13.036 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:13.062 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:13.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:13.164 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:13.199 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:13.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:13.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:12 smithi204 ceph-mon[87071]: pgmap v378: 99 pgs: 99 active+clean; 1.7 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 960 op/s 2024-09-13T14:43:13.302 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:13.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:12 smithi096 ceph-mon[102928]: pgmap v378: 99 pgs: 99 active+clean; 1.7 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 960 op/s 2024-09-13T14:43:13.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:12 smithi096 ceph-mon[98901]: pgmap v378: 99 pgs: 99 active+clean; 1.7 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 960 op/s 2024-09-13T14:43:13.332 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:13.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:13.435 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:13.462 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:13.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:13.564 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:13.596 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:13.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:13.698 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:13.727 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:13.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:13.829 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:13.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.856 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:13.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:13.958 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:14.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.020 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:14.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:14.121 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:14.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.154 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:14.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:14.255 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:14.286 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:14.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:14.389 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:14.428 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:14.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:14.530 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:14.563 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:14.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:14.665 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:14.696 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:14.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:14.798 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:14.837 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:14.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:14.939 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:14.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.974 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:14.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:15.057 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:14 smithi096 ceph-mon[98901]: pgmap v379: 99 pgs: 99 active+clean; 1.5 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 913 op/s 2024-09-13T14:43:15.057 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:14 smithi096 ceph-mon[102928]: pgmap v379: 99 pgs: 99 active+clean; 1.5 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 913 op/s 2024-09-13T14:43:15.075 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:15.108 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:15.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:15.210 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:15.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:14 smithi204 ceph-mon[87071]: pgmap v379: 99 pgs: 99 active+clean; 1.5 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 913 op/s 2024-09-13T14:43:15.260 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:15.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:15.362 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:15.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.401 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:15.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:15.502 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:15.538 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:15.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:15.640 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:15.669 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:15.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:15.771 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:15.778 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:43:15.779 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:43:15.795 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:15.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:15.897 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:15.934 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:15.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:16.036 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:16.072 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:16.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:16.173 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:16.207 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:16.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:16.309 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:16.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.343 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:16.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:16.444 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:16.485 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:16.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:16.586 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:16.609 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:16.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:16.711 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:16.740 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:16.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:16.842 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:16.868 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:16.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:16.927 INFO:teuthology.orchestra.run.smithi096.stderr:set pool 1 pgp_num to 11 2024-09-13T14:43:16.970 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:17.003 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:17.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:17.105 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:17.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.141 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:17.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:16 smithi204 ceph-mon[87071]: pgmap v380: 99 pgs: 99 active+clean; 1.4 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 926 op/s 2024-09-13T14:43:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:16 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3582120471' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:43:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:16 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:43:17.242 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:17.243 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:16 smithi096 ceph-mon[98901]: pgmap v380: 99 pgs: 99 active+clean; 1.4 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 926 op/s 2024-09-13T14:43:17.243 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:16 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3582120471' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:43:17.244 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:16 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:43:17.244 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:16 smithi096 ceph-mon[102928]: pgmap v380: 99 pgs: 99 active+clean; 1.4 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 926 op/s 2024-09-13T14:43:17.244 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:16 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3582120471' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:43:17.244 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:16 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:43:17.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.275 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:17.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:17.376 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:17.413 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:17.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:17.515 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:17.548 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:17.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:17.650 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:17.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.686 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:17.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:17.787 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:17.812 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:17.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:17.913 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:17.940 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:17.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:18.043 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:18.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.089 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:18.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:18.190 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:18.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:17 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:43:18.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:17 smithi204 ceph-mon[87071]: osdmap e140: 8 total, 8 up, 7 in 2024-09-13T14:43:18.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.219 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:18.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:18.320 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:18.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:17 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:43:18.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:17 smithi096 ceph-mon[98901]: osdmap e140: 8 total, 8 up, 7 in 2024-09-13T14:43:18.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:17 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:43:18.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:17 smithi096 ceph-mon[102928]: osdmap e140: 8 total, 8 up, 7 in 2024-09-13T14:43:18.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:43:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:43:17] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.51.0" 2024-09-13T14:43:18.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.357 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:18.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:18.458 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:18.493 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:18.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:18.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:18.628 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:18.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:18.730 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:18.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.760 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:18.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:18.860 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:18.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.894 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:18.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:18.995 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:19.028 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:19.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:19.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:18 smithi096 ceph-mon[98901]: pgmap v382: 99 pgs: 99 active+clean; 1.3 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:43:19.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:18 smithi096 ceph-mon[102928]: pgmap v382: 99 pgs: 99 active+clean; 1.3 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:43:19.130 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:19.161 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:19.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:19.225 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:18 smithi204 ceph-mon[87071]: pgmap v382: 99 pgs: 99 active+clean; 1.3 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:43:19.263 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:19.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.288 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:19.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:19.389 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:19.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.421 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:19.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:19.522 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:19.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.546 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:19.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:19.648 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:19.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.690 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:19.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:19.791 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:19.827 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:19.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:19.928 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:43:19.929 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:19.953 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:19.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:19.983 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:20.056 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:20.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.093 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:20.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:20.195 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:20.224 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:20.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:20.325 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:20.361 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:20.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:20.463 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:20.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:43:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:43:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:20.499 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:20.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:20.601 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:20.626 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:20.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:20.728 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:20.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.772 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:20.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:20.873 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:20.899 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:21.001 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:21.025 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:21.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:21.127 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:21.143 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:21 smithi096 ceph-mon[98901]: pgmap v383: 99 pgs: 99 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 979 op/s 2024-09-13T14:43:21.143 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:21.144 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:21 smithi096 ceph-mon[102928]: pgmap v383: 99 pgs: 99 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 979 op/s 2024-09-13T14:43:21.144 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:21.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.183 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:21.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:21.284 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:21.312 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:21.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:21.414 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:21.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.450 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:21.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:21.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:21 smithi204 ceph-mon[87071]: pgmap v383: 99 pgs: 99 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 979 op/s 2024-09-13T14:43:21.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:21.552 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:21.577 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:21.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:21.678 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:21.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.720 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:21.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:21.812 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:21.822 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:21.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.889 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:21.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:21.990 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:22.015 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:22.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:22.049 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:43:22.117 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:22.147 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:22.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:22.249 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:22.274 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:22.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:22.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:22 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4253210828' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:22.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:22 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4253210828' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:22.376 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:22.400 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:22.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:22.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:22 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4253210828' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:22.502 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:22.532 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:22.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:22.634 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:22.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.661 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:22.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:22.762 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:22.808 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:22.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:22.910 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:22.934 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:22.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:23.036 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:23.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.071 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:23.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:23.172 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:23.206 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:23.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:23.308 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:23.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:23 smithi096 ceph-mon[98901]: pgmap v384: 99 pgs: 99 active+clean; 1.1 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 969 op/s 2024-09-13T14:43:23.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:23 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:43:23.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:23 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:43:23.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:23 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4253210828' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:23.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:23 smithi096 ceph-mon[98901]: osdmap e141: 8 total, 8 up, 7 in 2024-09-13T14:43:23.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:23 smithi096 ceph-mon[102928]: pgmap v384: 99 pgs: 99 active+clean; 1.1 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 969 op/s 2024-09-13T14:43:23.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:23 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:43:23.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:23 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:43:23.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:23 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4253210828' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:23.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:23 smithi096 ceph-mon[102928]: osdmap e141: 8 total, 8 up, 7 in 2024-09-13T14:43:23.334 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:23.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:23.436 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:23.464 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:23.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:23.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:23 smithi204 ceph-mon[87071]: pgmap v384: 99 pgs: 99 active+clean; 1.1 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 969 op/s 2024-09-13T14:43:23.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:23 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:43:23.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:23 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:43:23.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:23 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4253210828' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:23.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:23 smithi204 ceph-mon[87071]: osdmap e141: 8 total, 8 up, 7 in 2024-09-13T14:43:23.566 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:23.605 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:23.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:23.707 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:23.737 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:23.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:23.801 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:43:23.801 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:43:23.817 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:43:23.818 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:43:23.818 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-09-13T14:43:23.818 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:43:23.818 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:43:23.819 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-09-13T14:43:23.838 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:43:23.838 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:43:23.839 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:23.864 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:23.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:23.881 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:43:23.882 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:43:23.922 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:43:23.922 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:43:23.958 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:43:23.959 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:43:23.966 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:23.992 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:43:23.992 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:43:24.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.001 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:24.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:24.021 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:43:24.021 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:43:24.056 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:43:24.056 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:43:24.103 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:24.128 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:24.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:24.229 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:24.261 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:24.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:24.262 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:24 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2332565723' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:24.262 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:24 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2332565723' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:24.363 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:24.390 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:24.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:24.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:24 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2332565723' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:24.491 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:24.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.535 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:24.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:24.636 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:24.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.671 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:24.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:24.772 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:24.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.809 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:24.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:24.910 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:24.946 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:24.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:25.048 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:25.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.075 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:25.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:25.124 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:25.176 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:25.204 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:25.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:25.306 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:25.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:25 smithi096 ceph-mon[98901]: pgmap v386: 99 pgs: 99 active+clean; 938 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 996 op/s 2024-09-13T14:43:25.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:25 smithi096 ceph-mon[102928]: pgmap v386: 99 pgs: 99 active+clean; 938 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 996 op/s 2024-09-13T14:43:25.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.337 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:25.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:25.438 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:25.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:25 smithi204 ceph-mon[87071]: pgmap v386: 99 pgs: 99 active+clean; 938 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 996 op/s 2024-09-13T14:43:25.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.469 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:25.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:25.570 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:25.603 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:25.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:25.705 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:25.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.746 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:25.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:25.847 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:25.875 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:25.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:25.977 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:26.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.012 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:26.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:26.113 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:26.138 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:26.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:26.240 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:26.272 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:26.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:26.312 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:43:26.373 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:26.402 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:26.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:26.504 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:26.536 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:26.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:26.638 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:26.670 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:26.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:26.773 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:26.804 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:26.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:26.906 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:26.942 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:26.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:27.044 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:27.077 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:27.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:27.092 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:43:27.179 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:27.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.206 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:27.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:27.307 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:27.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:27 smithi096 ceph-mon[98901]: pgmap v387: 99 pgs: 99 active+clean; 850 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 826 op/s 2024-09-13T14:43:27.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:27 smithi096 ceph-mon[102928]: pgmap v387: 99 pgs: 99 active+clean; 850 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 826 op/s 2024-09-13T14:43:27.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.355 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:27.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:27.456 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:27.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:27 smithi204 ceph-mon[87071]: pgmap v387: 99 pgs: 99 active+clean; 850 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 826 op/s 2024-09-13T14:43:27.491 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:27.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:27.593 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:27.624 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:27.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:27.726 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:27.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.778 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:27.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:27.879 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:27.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.902 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:27.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:28.003 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:28.039 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:28.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:28.141 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:28.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.207 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:28.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:28.308 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:28.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:43:28.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:28.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:28 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:28.325 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:43:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:43:27] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-13T14:43:28.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:43:28.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:28.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:28 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:28.338 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:28.340 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:28.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:28.442 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:28.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:43:28.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:28.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:28 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:28.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.513 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:28.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:28.614 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:28.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.658 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:28.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:28.759 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:28.811 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:28.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:28.913 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:28.954 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:28.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:28.980 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:43:28.980 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:43:28.980 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.980 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:252634 5.b 5:d7a4a3a6:::benchmark_data_smithi096_138841_object122438:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-09-13T14:43:28.980 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:43:28.956369+0000", 2024-09-13T14:43:28.980 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.021710588999999999, 2024-09-13T14:43:28.980 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.021740028000000002, 2024-09-13T14:43:28.980 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:43:28.981 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:43:28.981 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:43:28.981 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:43:28.981 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:43:28.981 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:43:28.981 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 252634 2024-09-13T14:43:28.981 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.981 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:43:28.981 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.981 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:43:28.981 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.956369+0000", 2024-09-13T14:43:28.981 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:43:28.981 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.982 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.982 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:43:28.982 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.956369+0000", 2024-09-13T14:43:28.982 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:43:28.982 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.982 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.982 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:43:28.982 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.956369+0000", 2024-09-13T14:43:28.982 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4799999999999999e-07 2024-09-13T14:43:28.982 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.982 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.982 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:43:28.982 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.956371+0000", 2024-09-13T14:43:28.982 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0789999999999999e-06 2024-09-13T14:43:28.982 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.983 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.983 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:43:28.983 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.956378+0000", 2024-09-13T14:43:28.983 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0780000000000001e-06 2024-09-13T14:43:28.983 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.983 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.983 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:43:28.983 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.956382+0000", 2024-09-13T14:43:28.983 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.729e-06 2024-09-13T14:43:28.984 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.984 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.984 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:43:28.984 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.964298+0000", 2024-09-13T14:43:28.984 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0079153690000000002 2024-09-13T14:43:28.984 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.984 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.984 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:43:28.984 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.964405+0000", 2024-09-13T14:43:28.984 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010784400000000001 2024-09-13T14:43:28.984 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.984 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.985 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:28.985 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.964512+0000", 2024-09-13T14:43:28.985 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000106085 2024-09-13T14:43:28.985 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.985 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.985 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:43:28.985 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.971643+0000", 2024-09-13T14:43:28.985 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.007131147 2024-09-13T14:43:28.985 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.985 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.985 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:28.985 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.974017+0000", 2024-09-13T14:43:28.986 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0023739159999999998 2024-09-13T14:43:28.986 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:43:28.986 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:43:28.986 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:43:28.986 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.986 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.986 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:252640 5.6 5:69033126:::benchmark_data_smithi096_138841_object122444:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-09-13T14:43:28.986 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:43:28.962145+0000", 2024-09-13T14:43:28.986 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.015934579000000001, 2024-09-13T14:43:28.986 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.016048759999999999, 2024-09-13T14:43:28.986 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:43:28.986 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:43:28.986 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:43:28.986 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:43:28.987 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:43:28.987 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:43:28.987 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 252640 2024-09-13T14:43:28.987 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.987 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:43:28.987 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.987 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:43:28.987 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.962145+0000", 2024-09-13T14:43:28.987 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:43:28.987 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.987 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.987 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:43:28.987 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.962145+0000", 2024-09-13T14:43:28.987 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:43:28.988 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.988 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.988 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:43:28.988 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.962146+0000", 2024-09-13T14:43:28.988 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.083e-06 2024-09-13T14:43:28.988 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.988 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.988 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:43:28.988 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.962150+0000", 2024-09-13T14:43:28.988 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2209999999999999e-06 2024-09-13T14:43:28.988 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.988 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.988 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:43:28.988 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.962163+0000", 2024-09-13T14:43:28.989 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2668e-05 2024-09-13T14:43:28.989 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.989 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.989 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:43:28.989 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.962169+0000", 2024-09-13T14:43:28.989 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.02e-06 2024-09-13T14:43:28.989 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.989 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.989 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:43:28.989 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.973227+0000", 2024-09-13T14:43:28.989 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.01105765 2024-09-13T14:43:28.989 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.989 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.989 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:43:28.989 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.973323+0000", 2024-09-13T14:43:28.990 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.6096999999999998e-05 2024-09-13T14:43:28.990 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.990 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.990 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:43:28.990 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.973430+0000", 2024-09-13T14:43:28.990 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010720100000000001 2024-09-13T14:43:28.990 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.990 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.990 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:43:28.990 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.978127+0000", 2024-09-13T14:43:28.990 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0046971119999999998 2024-09-13T14:43:28.990 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:43:28.990 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:43:28.990 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:43:28.990 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.991 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.991 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:252644 5.8 5:10fa5405:::benchmark_data_smithi096_138841_object122448:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-09-13T14:43:28.991 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:43:28.969542+0000", 2024-09-13T14:43:28.991 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0085378590000000001, 2024-09-13T14:43:28.991 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0087229620000000008, 2024-09-13T14:43:28.991 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:43:28.991 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:43:28.991 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:43:28.991 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:43:28.991 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:43:28.991 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:43:28.991 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 252644 2024-09-13T14:43:28.991 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.992 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:43:28.992 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.992 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:43:28.992 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.969542+0000", 2024-09-13T14:43:28.992 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:43:28.992 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.992 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.992 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:43:28.992 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.969542+0000", 2024-09-13T14:43:28.992 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:43:28.992 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.992 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.992 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:43:28.993 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.969543+0000", 2024-09-13T14:43:28.993 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5850000000000001e-06 2024-09-13T14:43:28.993 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.993 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.993 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:43:28.993 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.969557+0000", 2024-09-13T14:43:28.993 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3935e-05 2024-09-13T14:43:28.993 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.993 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.993 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:43:28.993 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.969570+0000", 2024-09-13T14:43:28.993 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2642e-05 2024-09-13T14:43:28.993 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.993 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.994 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:43:28.994 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.969578+0000", 2024-09-13T14:43:28.994 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.8099999999999998e-06 2024-09-13T14:43:28.994 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.994 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.994 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:43:28.994 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.969637+0000", 2024-09-13T14:43:28.994 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9428000000000001e-05 2024-09-13T14:43:28.994 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.994 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.994 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:43:28.995 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.969746+0000", 2024-09-13T14:43:28.995 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010880899999999999 2024-09-13T14:43:28.995 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.995 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.995 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:43:28.995 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.969850+0000", 2024-09-13T14:43:28.995 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010461799999999999 2024-09-13T14:43:28.995 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:43:28.995 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:43:28.995 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:43:28.995 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.995 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.995 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:252645 5.d 5:bbbf8e63:::benchmark_data_smithi096_138841_object122449:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-09-13T14:43:28.995 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:43:28.971547+0000", 2024-09-13T14:43:28.996 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0065321579999999997, 2024-09-13T14:43:28.996 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.006781011, 2024-09-13T14:43:28.996 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:43:28.996 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:43:28.996 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:43:28.996 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:43:28.996 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:43:28.996 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:43:28.996 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 252645 2024-09-13T14:43:28.996 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.996 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:43:28.996 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.996 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:43:28.996 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.971547+0000", 2024-09-13T14:43:28.997 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:43:28.997 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.997 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.997 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:43:28.997 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.971547+0000", 2024-09-13T14:43:28.997 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:43:28.997 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.997 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.997 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:43:28.997 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.971553+0000", 2024-09-13T14:43:28.997 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4659999999999998e-06 2024-09-13T14:43:28.997 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.997 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.998 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:43:28.998 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.971559+0000", 2024-09-13T14:43:28.998 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7189999999999998e-06 2024-09-13T14:43:28.998 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.998 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.998 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:43:28.998 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.971570+0000", 2024-09-13T14:43:28.998 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1357e-05 2024-09-13T14:43:28.998 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.998 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.998 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:43:28.998 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.971578+0000", 2024-09-13T14:43:28.998 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6079999999999996e-06 2024-09-13T14:43:28.999 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.999 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.999 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:43:28.999 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.973062+0000", 2024-09-13T14:43:28.999 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001483596 2024-09-13T14:43:28.999 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.999 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.999 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:43:28.999 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.973180+0000", 2024-09-13T14:43:28.999 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011796 2024-09-13T14:43:28.999 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:28.999 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:28.999 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:43:29.000 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.973292+0000", 2024-09-13T14:43:29.000 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000111896 2024-09-13T14:43:29.000 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:43:29.000 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:43:29.000 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:43:29.000 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.000 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.000 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:252646 5.8 5:169c6f3a:::benchmark_data_smithi096_138841_object122450:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-09-13T14:43:29.000 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:43:28.974246+0000", 2024-09-13T14:43:29.000 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0038330360000000002, 2024-09-13T14:43:29.000 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.004143412, 2024-09-13T14:43:29.000 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:43:29.000 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:43:29.000 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:43:29.001 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:43:29.001 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:43:29.001 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:43:29.001 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 252646 2024-09-13T14:43:29.001 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.001 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:43:29.001 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.001 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:43:29.001 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.974246+0000", 2024-09-13T14:43:29.001 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:43:29.001 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.001 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.001 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:43:29.001 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.974246+0000", 2024-09-13T14:43:29.001 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:43:29.002 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.002 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.002 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:43:29.002 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.974247+0000", 2024-09-13T14:43:29.002 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.037e-06 2024-09-13T14:43:29.002 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.002 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.002 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:43:29.002 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.974252+0000", 2024-09-13T14:43:29.002 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2339999999999996e-06 2024-09-13T14:43:29.002 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.002 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.002 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:43:29.003 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.974263+0000", 2024-09-13T14:43:29.003 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0856000000000001e-05 2024-09-13T14:43:29.003 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.003 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.003 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:43:29.003 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.974270+0000", 2024-09-13T14:43:29.003 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9940000000000003e-06 2024-09-13T14:43:29.003 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.003 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.003 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:43:29.003 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.975417+0000", 2024-09-13T14:43:29.003 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001147019 2024-09-13T14:43:29.003 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.003 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.003 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:43:29.004 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.975541+0000", 2024-09-13T14:43:29.004 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000123967 2024-09-13T14:43:29.004 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.004 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.004 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:43:29.004 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.975694+0000", 2024-09-13T14:43:29.004 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000153072 2024-09-13T14:43:29.004 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:43:29.004 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:43:29.004 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:43:29.004 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.004 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.004 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:252643 5.2 e141/107 5:47cdf15b:::benchmark_data_smithi096_138841_object122447:head v 141'15870, mlcod=141'15868)", 2024-09-13T14:43:29.005 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:43:28.977968+0000", 2024-09-13T14:43:29.005 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.000111907, 2024-09-13T14:43:29.005 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00050350999999999998, 2024-09-13T14:43:29.005 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:43:29.005 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:43:29.005 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:43:29.005 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:43:29.005 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.005 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:43:29.005 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.977968+0000", 2024-09-13T14:43:29.005 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:43:29.005 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.005 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.005 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.977968+0000", 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.977968+0000", 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.61e-07 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.977973+0000", 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7890000000000002e-06 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:43:29.006 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.977983+0000", 2024-09-13T14:43:29.007 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0142999999999999e-05 2024-09-13T14:43:29.007 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.007 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.007 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:43:29.007 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.977990+0000", 2024-09-13T14:43:29.007 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4799999999999998e-06 2024-09-13T14:43:29.007 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.007 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.007 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:43:29.007 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.978063+0000", 2024-09-13T14:43:29.007 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2821e-05 2024-09-13T14:43:29.007 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:43:29.007 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:43:29.007 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:43:29.007 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:43:28.978082+0000", 2024-09-13T14:43:29.008 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9392999999999999e-05 2024-09-13T14:43:29.008 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:43:29.008 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:43:29.008 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:43:29.008 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:43:29.008 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2024-09-13T14:43:29.008 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 6 2024-09-13T14:43:29.008 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:43:29.056 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:29.097 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:29.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:29.199 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:29.238 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:29.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:29.340 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:29.375 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:29.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:29.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:29 smithi204 ceph-mon[87071]: pgmap v388: 99 pgs: 99 active+clean; 694 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 944 op/s 2024-09-13T14:43:29.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:29 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:29 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:29.477 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:29.511 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:29.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:29 smithi096 ceph-mon[98901]: pgmap v388: 99 pgs: 99 active+clean; 694 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 944 op/s 2024-09-13T14:43:29.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:29 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:29.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:29 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:29 smithi096 ceph-mon[102928]: pgmap v388: 99 pgs: 99 active+clean; 694 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 944 op/s 2024-09-13T14:43:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:29 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:29 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:29.614 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:29.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.656 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:29.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:29.758 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:29.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.803 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:29.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:29.904 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:29.944 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:29.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:30.046 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:30.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.077 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:30.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:30.178 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:30.217 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:30.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:30.319 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:30.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.364 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:30.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:30.465 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:30.472 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:43:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:43:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:30.502 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:30.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:30.604 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:30.638 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:30.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:30.739 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:30.771 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:30.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:30.873 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:30.902 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:30.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:31.004 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:31.045 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:31.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:31.147 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:31.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.182 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:31.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:31.283 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:31.314 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:31.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:31.416 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:31.449 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:31.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:31.481 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:43:31.551 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:31.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.577 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:31.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:31.579 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:31.602 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:31 smithi204 ceph-mon[87071]: pgmap v389: 99 pgs: 99 active+clean; 544 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 942 op/s 2024-09-13T14:43:31.679 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:31.696 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:31 smithi096 ceph-mon[98901]: pgmap v389: 99 pgs: 99 active+clean; 544 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 942 op/s 2024-09-13T14:43:31.696 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:31 smithi096 ceph-mon[102928]: pgmap v389: 99 pgs: 99 active+clean; 544 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 942 op/s 2024-09-13T14:43:31.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.742 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:31.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:31.843 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:31.881 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:31.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:31.983 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:32.022 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:32.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:32.124 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:32.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.184 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:32.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:32.285 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:32.314 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:32.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:32.416 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:32.449 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:32.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:32.551 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:32.583 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:32.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:32.684 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:32.715 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:32.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:32.821 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:32.849 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:32.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:32.952 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:32.994 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:32.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:33.096 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:33.124 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:33.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:33.226 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:33.279 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:33.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:33.381 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:33.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.419 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:33.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:33.520 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:33.536 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:33 smithi096 ceph-mon[98901]: pgmap v390: 99 pgs: 99 active+clean; 508 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 944 op/s 2024-09-13T14:43:33.536 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:33.536 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:33 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1964311749' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:43:33.536 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:33.536 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:33 smithi096 ceph-mon[102928]: pgmap v390: 99 pgs: 99 active+clean; 508 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 944 op/s 2024-09-13T14:43:33.537 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:33.537 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:33 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1964311749' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:43:33.537 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:33.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.559 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:33.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:33.660 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:33.707 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:33.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:33.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:33 smithi204 ceph-mon[87071]: pgmap v390: 99 pgs: 99 active+clean; 508 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 944 op/s 2024-09-13T14:43:33.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:33.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:33 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1964311749' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:43:33.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:33.809 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:33.867 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:43:33.867 INFO:teuthology.orchestra.run.smithi096.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:43:33.868 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:43:33.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.874 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:33.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:33.975 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:34.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.008 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:34.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:34.086 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:43:34.109 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:34.156 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:34.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:34.258 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:34.284 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:34.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:34.386 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:34.417 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:34.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:34.519 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:34.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.556 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:34.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:34.658 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:34.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.695 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:34.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:34.710 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:34 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:34.710 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:34 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1964311749' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:43:34.710 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:34 smithi096 ceph-mon[98901]: osdmap e142: 8 total, 8 up, 7 in 2024-09-13T14:43:34.710 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:34 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:34.710 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:34 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1964311749' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:43:34.710 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:34 smithi096 ceph-mon[102928]: osdmap e142: 8 total, 8 up, 7 in 2024-09-13T14:43:34.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:34 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:34.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:34 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1964311749' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:43:34.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:34 smithi204 ceph-mon[87071]: osdmap e142: 8 total, 8 up, 7 in 2024-09-13T14:43:34.797 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:34.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.830 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:34.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:34.931 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:34.960 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:34.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:35.062 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:35.095 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:35.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:35.198 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:35.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.236 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:35.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:35.338 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:35.374 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:35.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:35.476 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:35.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.503 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:35.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:35.604 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:35.638 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:35.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:35.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:35 smithi204 ceph-mon[87071]: pgmap v391: 99 pgs: 99 active+clean; 345 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 909 op/s 2024-09-13T14:43:35.740 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:35.778 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:35 smithi096 ceph-mon[98901]: pgmap v391: 99 pgs: 99 active+clean; 345 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 909 op/s 2024-09-13T14:43:35.779 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:35 smithi096 ceph-mon[102928]: pgmap v391: 99 pgs: 99 active+clean; 345 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 909 op/s 2024-09-13T14:43:35.779 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:35.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:35.882 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:35.919 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:35.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:36.021 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:36.048 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:36.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:36.150 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:36.184 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:36.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:36.286 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:36.325 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:36.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:36.430 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:36.460 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:36.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:36.561 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:36.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:36.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:36.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.611 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:36.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:36.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:36.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:36.753 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:36.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:36.837 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:43:36.855 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:36.880 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:36.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:36.982 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:37.007 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:37.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:37.043 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:37.080 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:43:37.081 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:43:37.108 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:37.121 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Removed 130193 objects 2024-09-13T14:43:37.121 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Clean up completed and total clean up time :134.389 2024-09-13T14:43:37.125 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:43:37.125 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:43:37.138 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:37.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:37.144 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:43:37.144 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:43:37.161 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:43:37.161 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:43:37.177 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:43:37.177 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:43:37.192 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:43:37.192 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:43:37.208 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:43:37.208 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:43:37.224 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:43:37.224 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:43:37.240 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:37.262 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:37.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:37.364 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:37.385 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:37.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:37.487 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:37.509 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:37.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:37.592 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:37 smithi096 ceph-mon[98901]: pgmap v393: 99 pgs: 99 active+clean; 253 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 954 op/s 2024-09-13T14:43:37.593 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:37 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1849886712' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:37.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:37 smithi096 ceph-mon[102928]: pgmap v393: 99 pgs: 99 active+clean; 253 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 954 op/s 2024-09-13T14:43:37.593 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:37 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1849886712' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:37.611 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:37.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.633 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:37.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:37.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:37 smithi204 ceph-mon[87071]: pgmap v393: 99 pgs: 99 active+clean; 253 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 954 op/s 2024-09-13T14:43:37.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:37 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1849886712' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:37.735 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:37.756 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:37.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:37.858 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:37.879 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:37.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:37.981 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:38.004 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:38.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:38.106 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:38.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.128 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:38.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:38.178 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:43:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:43:37] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.51.0" 2024-09-13T14:43:38.229 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:38.250 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:38.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:38.352 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:38.373 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:38.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:38.405 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-09-13T14:43:38.405 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-09-13T14:43:38.475 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:38.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.497 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:38.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:38.598 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:38.619 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:38.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:38.721 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:38.742 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:38.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:38.844 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:38.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.866 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:38.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:38.967 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:38.990 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:38.990 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:38.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:39.092 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:39.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.115 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:39.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:39.216 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:39.237 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:39.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:39.339 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:39.361 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:39.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:39.463 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:39.485 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:39.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:39.538 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:39.587 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:39.609 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:39.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:39.640 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:39 smithi096 ceph-mon[98901]: pgmap v394: 99 pgs: 99 active+clean; 102 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 947 op/s 2024-09-13T14:43:39.641 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:39 smithi096 ceph-mon[102928]: pgmap v394: 99 pgs: 99 active+clean; 102 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 947 op/s 2024-09-13T14:43:39.711 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:39.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:39 smithi204 ceph-mon[87071]: pgmap v394: 99 pgs: 99 active+clean; 102 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 947 op/s 2024-09-13T14:43:39.734 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:39.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:39.744 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:43:39.744 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:43:39.835 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:39.857 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:39.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:39.959 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:39.981 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:40.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:40.017 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:43:40.108 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:40.130 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:40.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:40.232 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:40.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.257 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:40.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:40.358 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:40.379 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:40.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:40.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:43:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:43:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:40.481 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:40.496 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:43:40.505 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:40.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:40.606 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:40.630 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:40.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:40.732 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:40.755 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:40.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:40.818 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:43:40.819 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:43:40.857 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:40.881 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:40.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:40.983 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:41.009 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:41.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:41.111 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:41.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.134 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:41.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:41.235 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:41.256 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:41.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:41.358 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:41.384 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:41.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:41.486 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:41.508 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:41.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:41.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:41 smithi096 ceph-mon[98901]: pgmap v395: 99 pgs: 99 active+clean; 12 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 851 op/s 2024-09-13T14:43:41.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:41 smithi096 ceph-mon[102928]: pgmap v395: 99 pgs: 99 active+clean; 12 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 851 op/s 2024-09-13T14:43:41.610 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:41.631 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:41.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:41.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:41 smithi204 ceph-mon[87071]: pgmap v395: 99 pgs: 99 active+clean; 12 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 851 op/s 2024-09-13T14:43:41.733 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:41.758 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:41.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:41.860 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:41.885 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:41.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:41.987 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:42.011 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:42.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:42.113 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:42.138 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:42.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:42.240 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:42.266 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:42.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:42.368 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:42.394 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:42.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:42.495 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:42.521 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:42.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:42.623 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:42.649 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:42.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:42.751 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:42.777 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:42.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:42.879 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:42.900 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:42.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:43.002 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:43.027 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:43.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:43.129 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:43.155 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:43.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:43.257 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:43.281 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:43.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:43.383 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:43.408 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:43.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:43.510 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:43.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.536 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:43.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:43.637 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:43.658 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:43.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:43.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:43 smithi204 ceph-mon[87071]: pgmap v396: 99 pgs: 99 active+clean; 9.4 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 798 op/s 2024-09-13T14:43:43.760 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:43.785 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:43.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:43.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:43 smithi096 ceph-mon[98901]: pgmap v396: 99 pgs: 99 active+clean; 9.4 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 798 op/s 2024-09-13T14:43:43.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:43 smithi096 ceph-mon[102928]: pgmap v396: 99 pgs: 99 active+clean; 9.4 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 798 op/s 2024-09-13T14:43:43.888 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:43.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.914 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:43.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:44.015 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:44.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.040 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:44.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:44.141 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:44.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.169 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:44.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:44.270 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:44.292 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:44.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:44.394 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:44.400 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:44.403 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:44.419 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:44.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:44.521 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:44.548 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:44.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:44.651 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:44.673 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:44.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:44.777 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:44.803 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:44.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:44.905 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:44.930 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:44.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:45.032 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:45.056 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:45.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:45.158 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:45.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.187 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:45.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:45.288 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:45.314 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:45.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:45.416 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:45.442 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:45.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:45.544 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:45.570 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:45.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:45.658 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:45 smithi096 ceph-mon[98901]: pgmap v397: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 544 op/s 2024-09-13T14:43:45.658 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:45 smithi096 ceph-mon[102928]: pgmap v397: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 544 op/s 2024-09-13T14:43:45.672 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:45.698 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:45.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:45.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:45 smithi204 ceph-mon[87071]: pgmap v397: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 544 op/s 2024-09-13T14:43:45.800 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:45.825 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:45.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:45.927 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:45.949 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:45.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:46.051 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:46.072 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:46.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:46.174 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:46.200 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:46.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:46.302 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:46.328 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:46.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:46.418 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'unique_pool_0' removed 2024-09-13T14:43:46.430 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:46.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.457 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:46.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:46.558 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:46.580 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:46.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:46.682 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:46.683 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:46 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2170287766' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:43:46.683 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:46 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:43:46.683 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:46 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2170287766' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:43:46.683 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:46 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:43:46.704 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:46.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:46.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:46 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2170287766' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:43:46.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:46 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:43:46.806 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:46.828 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:46.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:46.930 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:46.951 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:46.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:47.053 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:47.074 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:47.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:47.175 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:47.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.197 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:47.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:47.298 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:47.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.321 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:47.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:47.423 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:47.445 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:47.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:47.547 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:47.563 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:47 smithi096 ceph-mon[98901]: pgmap v398: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 468 op/s 2024-09-13T14:43:47.563 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:47 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:43:47.564 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:47 smithi096 ceph-mon[98901]: osdmap e143: 8 total, 8 up, 7 in 2024-09-13T14:43:47.564 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:47 smithi096 ceph-mon[102928]: pgmap v398: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 468 op/s 2024-09-13T14:43:47.564 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:47 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:43:47.564 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:47 smithi096 ceph-mon[102928]: osdmap e143: 8 total, 8 up, 7 in 2024-09-13T14:43:47.568 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:47.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:47.670 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:47.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.692 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:47.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:47.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:47 smithi204 ceph-mon[87071]: pgmap v398: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 468 op/s 2024-09-13T14:43:47.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:47 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:43:47.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:47 smithi204 ceph-mon[87071]: osdmap e143: 8 total, 8 up, 7 in 2024-09-13T14:43:47.793 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:47.814 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:47.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:47.916 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:47.937 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:47.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:48.039 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:48.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.066 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:48.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:48.167 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:48.177 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:43:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:43:47] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.51.0" 2024-09-13T14:43:48.188 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:48.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:48.198 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:43:48.199 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:43:48.199 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-09-13T14:43:48.200 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_1 16 2024-09-13T14:43:48.289 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:48.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.312 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:48.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:48.413 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:48.419 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:43:48.432 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:48 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2291892222' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:48.433 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:48 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2291892222' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:48.436 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:48.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:48.538 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:48.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.561 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:48.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:48.662 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:48.683 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:48.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:48.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:48 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2291892222' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:48.785 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:48.806 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:48.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:48.908 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:48.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.930 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:48.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:49.031 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:49.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.052 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:49.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:49.154 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:49.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.179 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:49.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:49.280 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:49.302 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:49.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:49.404 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:49.428 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:49.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:49.530 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:49.552 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:49.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:49.653 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:49.675 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:49.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:49.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:49 smithi204 ceph-mon[87071]: pgmap v400: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:49.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:49 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:49.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:49 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2291892222' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:49.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:49 smithi204 ceph-mon[87071]: osdmap e144: 8 total, 8 up, 7 in 2024-09-13T14:43:49.777 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:49.800 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:49.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:49.810 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:49 smithi096 ceph-mon[98901]: pgmap v400: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:49.810 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:49 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:49.810 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:49 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2291892222' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:49.811 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:49 smithi096 ceph-mon[98901]: osdmap e144: 8 total, 8 up, 7 in 2024-09-13T14:43:49.811 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:49 smithi096 ceph-mon[102928]: pgmap v400: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:49.811 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:49 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:49.811 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:49 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2291892222' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:49.811 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:49 smithi096 ceph-mon[102928]: osdmap e144: 8 total, 8 up, 7 in 2024-09-13T14:43:49.902 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:49.924 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:49.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:50.025 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:50.047 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:50.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:50.149 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:50.172 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:50.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:50.274 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:50.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.301 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:50.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:50.403 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:50.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.427 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:50.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:50.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:43:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:43:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:50.528 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:50.550 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:50.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:50.651 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:50.672 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:50.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:50.775 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:50.801 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:50.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:50.903 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:50.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.925 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:50.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:51.026 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:51.047 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:51.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:51.149 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:51.170 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:51.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:51.182 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:43:51.188 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:43:51.188 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:43:51.272 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:51.293 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:51.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:51.395 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:51.416 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:51.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:51.518 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:51.540 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:51.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:51.605 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:51 smithi096 ceph-mon[98901]: pgmap v402: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:51.606 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:51.606 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:51 smithi096 ceph-mon[102928]: pgmap v402: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:51.606 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:51.643 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:51.664 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:51.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:51.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:51 smithi204 ceph-mon[87071]: pgmap v402: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:51.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:51.767 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:51.789 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:51.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:51.891 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:51.913 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:51.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:52.015 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:52.037 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:52.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:52.139 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:52.160 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:52.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:52.262 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:52.284 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:52.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:52.386 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:52.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.408 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:52.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:52.509 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:52.531 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:52.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:52.633 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:52.655 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:52.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:52.719 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:52.743 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:43:52.756 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:52.778 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:52.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:52.880 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:52.902 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:52.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:53.004 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:53.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.026 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:53.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:53.128 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:53.149 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:53.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:53.208 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:43:53.208 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:43:53.225 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:43:53.225 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:43:53.251 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:53.254 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:43:53.255 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:43:53.273 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:53.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:53.286 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:43:53.287 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:43:53.314 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:43:53.314 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:43:53.342 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:43:53.342 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:43:53.369 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:43:53.370 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:43:53.375 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:53.393 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:43:53.393 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:43:53.396 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:53.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:53.498 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:53.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.520 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:53.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:53.543 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:53 smithi096 ceph-mon[98901]: pgmap v403: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:53.543 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:53 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:53.543 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:53 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:53.543 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:53 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:43:53.543 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:53 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:43:53.543 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:53 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:53.543 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:53 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/335538438' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:53.543 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:53 smithi096 ceph-mon[102928]: pgmap v403: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:53.544 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:53 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:53.544 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:53 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:53.544 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:53 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:43:53.544 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:53 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:43:53.544 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:53 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:53.544 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:53 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/335538438' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:53.622 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:53.643 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:53.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:53.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:53 smithi204 ceph-mon[87071]: pgmap v403: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:53.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:53 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:53.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:53 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:53 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:43:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:53 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:43:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:53 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:43:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:53 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/335538438' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:53.745 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:53.767 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:53.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:53.869 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:53.890 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:53.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:53.992 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:54.014 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:54.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:54.116 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:54.138 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:54.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:54.241 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:54.263 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:54.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:54.364 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:54.389 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:54.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:54.406 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:54.406 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:54.491 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:54.514 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:54.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:54.616 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:54.637 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:54.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:54.739 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:54.760 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:54.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:54.862 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:54.883 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:54.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:54.985 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:55.009 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:55.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:55.111 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:55.137 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:55.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:55.238 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:55.264 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:55.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:55.366 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:55.388 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:55.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:55.490 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:55.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.513 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:55.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:55.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:55 smithi096 ceph-mon[102928]: pgmap v404: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:55.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:55 smithi096 ceph-mon[98901]: pgmap v404: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:55.614 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:55.641 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:55.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:55.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:55 smithi204 ceph-mon[87071]: pgmap v404: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:55.743 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:55.769 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:55.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:55.871 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:55.897 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:55.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:55.998 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:56.024 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:56.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:56.126 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:56.149 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:56.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:56.251 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:56.277 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:56.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:56.379 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:56.405 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:56.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:56.507 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:56.530 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:56.632 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:56.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.654 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:56.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:56.755 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:56.781 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:56.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:56.883 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:56.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.905 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:56.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:57.006 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:57.032 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:57.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:57.134 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:57.156 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:57.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:57.257 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:57.279 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'unique_pool_1' created 2024-09-13T14:43:57.285 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:57.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:57.387 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:57.409 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:57.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:57.511 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:57.532 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:57.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:57.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:57 smithi096 ceph-mon[98901]: pgmap v405: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:57.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:57 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/109524932' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-13T14:43:57.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:57 smithi096 ceph-mon[102928]: pgmap v405: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:57.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:57 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/109524932' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-13T14:43:57.634 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:57.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.657 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:57.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:57.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:57 smithi204 ceph-mon[87071]: pgmap v405: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:57.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:57 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/109524932' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-13T14:43:57.721 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:43:57.721 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:43:57.722 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 0 2024-09-13T14:43:57.722 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:43:57.758 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:57.780 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:57.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:57.882 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:57.909 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:58.011 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:58.011 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:43:58.037 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:58.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:58.139 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:58.165 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:58.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:58.267 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:58.285 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:43:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:43:57] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.51.0" 2024-09-13T14:43:58.296 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:58.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:58.398 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:58.421 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:58.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:58.425 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:43:58.524 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:58.545 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:58.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:58.554 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:58 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/109524932' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-13T14:43:58.554 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:58 smithi096 ceph-mon[98901]: osdmap e145: 8 total, 8 up, 7 in 2024-09-13T14:43:58.554 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:58 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/109524932' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-13T14:43:58.554 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:58 smithi096 ceph-mon[102928]: osdmap e145: 8 total, 8 up, 7 in 2024-09-13T14:43:58.647 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:58.668 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:58.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:58.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:58 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/109524932' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-13T14:43:58.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:58 smithi204 ceph-mon[87071]: osdmap e145: 8 total, 8 up, 7 in 2024-09-13T14:43:58.770 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:58.792 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:58.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:58.893 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:58.916 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:58.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:59.017 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:59.039 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:59.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:59.141 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:59.167 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:59.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:59.269 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:59.291 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:59.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:59.300 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-09-13T14:43:59.351 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:59.394 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:59.414 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:59.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:59.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:59.542 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:59.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:59 smithi096 ceph-mon[98901]: pgmap v407: 99 pgs: 16 unknown, 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:59 smithi096 ceph-mon[98901]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:43:59.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:43:59 smithi096 ceph-mon[98901]: osdmap e146: 8 total, 8 up, 7 in 2024-09-13T14:43:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:59 smithi096 ceph-mon[102928]: pgmap v407: 99 pgs: 16 unknown, 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:59 smithi096 ceph-mon[102928]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:43:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:43:59 smithi096 ceph-mon[102928]: osdmap e146: 8 total, 8 up, 7 in 2024-09-13T14:43:59.644 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:59.670 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:59.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:59.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:59 smithi204 ceph-mon[87071]: pgmap v407: 99 pgs: 16 unknown, 83 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:43:59.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:59 smithi204 ceph-mon[87071]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:43:59.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:43:59 smithi204 ceph-mon[87071]: osdmap e146: 8 total, 8 up, 7 in 2024-09-13T14:43:59.772 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:59.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.795 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:43:59.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:59.848 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:59.897 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:59.922 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:43:59.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:00.024 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:00.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.047 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:00.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:00.148 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:00.173 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:00.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:00.275 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:00.302 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:00.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:00.306 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:44:00.404 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:00.429 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:00.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:00 smithi204 ceph-mon[87071]: osdmap e147: 8 total, 8 up, 7 in 2024-09-13T14:44:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:00 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4047367625' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:44:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:44:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:00.531 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:00.548 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:00 smithi096 ceph-mon[102928]: osdmap e147: 8 total, 8 up, 7 in 2024-09-13T14:44:00.548 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:00 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4047367625' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:00.548 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:00 smithi096 ceph-mon[98901]: osdmap e147: 8 total, 8 up, 7 in 2024-09-13T14:44:00.548 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:00 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4047367625' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:00.554 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:00.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:00.655 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:00.676 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:00.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:00.778 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:00.799 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:00.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:00.900 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:00.922 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:00.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:01.024 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:01.050 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:01.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:01.151 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:01.173 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:01.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:01.274 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:01.295 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:01.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:01.397 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:01.419 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:01.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:01.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:01 smithi096 ceph-mon[98901]: pgmap v410: 99 pgs: 13 unknown, 86 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:44:01.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:01 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:01.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:01 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4047367625' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:44:01.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:01 smithi096 ceph-mon[98901]: osdmap e148: 8 total, 8 up, 7 in 2024-09-13T14:44:01.493 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:01 smithi096 ceph-mon[102928]: pgmap v410: 99 pgs: 13 unknown, 86 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:44:01.493 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:01 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:01.493 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:01 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4047367625' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:44:01.493 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:01 smithi096 ceph-mon[102928]: osdmap e148: 8 total, 8 up, 7 in 2024-09-13T14:44:01.521 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:01.544 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:01.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:01.646 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:01.669 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:01.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:01.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:01 smithi204 ceph-mon[87071]: pgmap v410: 99 pgs: 13 unknown, 86 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:44:01.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:01 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:01.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:01 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4047367625' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:44:01.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:01 smithi204 ceph-mon[87071]: osdmap e148: 8 total, 8 up, 7 in 2024-09-13T14:44:01.771 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:01.792 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:01.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:01.894 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:01.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.917 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:01.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:02.018 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:02.046 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:02.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:02.147 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:02.164 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:02.170 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:02.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:02.272 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:02.293 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:02.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:02.395 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:02.416 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:02.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:02.518 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:02.541 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:02.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:02.643 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:02.666 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:02.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:02.769 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:02.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.792 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:02.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:02.893 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:02.918 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:02.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:03.020 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:03.042 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:03.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:03.144 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:03.165 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:03.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:03.245 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:44:03.267 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:03.309 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:03.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:03.411 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:03.433 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:03.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:03.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:03 smithi096 ceph-mon[98901]: pgmap v412: 99 pgs: 10 unknown, 89 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:44:03.484 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:03 smithi096 ceph-mon[102928]: pgmap v412: 99 pgs: 10 unknown, 89 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:44:03.535 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:03.560 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:03.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:03.662 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:03.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.688 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:03.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:03.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:03 smithi204 ceph-mon[87071]: pgmap v412: 99 pgs: 10 unknown, 89 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:44:03.790 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:03.812 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:03.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:03.914 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:03.935 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:03.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:04.037 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:04.057 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:04.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:04.159 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:04.181 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:04.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:04.283 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:04.304 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:04.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:04.406 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:04.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.428 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:04.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:04.530 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:04.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.558 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:04.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:04.660 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:04.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.687 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:04.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:04.789 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:04.814 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:04.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:04.917 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:04.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.941 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:04.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:05.042 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:05.068 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:05.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:05.170 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:05.193 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:05.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:05.295 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:05.320 INFO:teuthology.orchestra.run.smithi096.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-09-13T14:44:05.322 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:05.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:05.425 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:05.451 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:05.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:05.553 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:05.578 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:05.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:05.664 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:05 smithi096 ceph-mon[98901]: pgmap v413: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:44:05.664 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:05 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4071596020' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:44:05.665 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:05 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:44:05.665 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:05 smithi096 ceph-mon[102928]: pgmap v413: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:44:05.665 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:05 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4071596020' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:44:05.665 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:05 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:44:05.680 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:05.705 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:05.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:05 smithi204 ceph-mon[87071]: pgmap v413: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:44:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:05 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4071596020' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:44:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:05 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:44:05.807 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:05.857 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:44:05.857 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:44:05.858 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:44:05.858 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:44:05.858 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:44:05.858 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:44:05.858 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:44:05.858 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:05.859 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:05.859 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:05.859 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:05.859 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:05.859 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:05.859 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:05.859 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:05.859 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:44:05.863 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:05.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:05.965 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:05.985 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:05.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:06.087 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:06.113 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:06.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:06.215 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:06.237 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:06.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:06.339 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:06.360 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:06.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:06.462 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:06.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:06 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:44:06.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:06 smithi096 ceph-mon[98901]: osdmap e149: 8 total, 8 up, 7 in 2024-09-13T14:44:06.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:06 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2725973593' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:06.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:44:06.478 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:06.478 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:06 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:44:06.478 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:06 smithi096 ceph-mon[102928]: osdmap e149: 8 total, 8 up, 7 in 2024-09-13T14:44:06.478 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:06 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2725973593' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:06.478 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:44:06.478 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:06.484 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:06.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:06.586 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:06.608 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:06.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:06.710 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:06.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:06 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:44:06.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:06 smithi204 ceph-mon[87071]: osdmap e149: 8 total, 8 up, 7 in 2024-09-13T14:44:06.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:06 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2725973593' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:06.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:44:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:06.736 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:06.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:06.738 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:06.738 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:06.838 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:06.861 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:06.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:06.963 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:06.984 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:06.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:07.085 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:07.106 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:07.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:07.208 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:07.229 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:07.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:07.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:07.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.352 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:07.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:07.386 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:07 smithi096 ceph-mon[98901]: pgmap v415: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:44:07.386 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:07 smithi096 ceph-mon[98901]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:44:07.386 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:07 smithi096 ceph-mon[102928]: pgmap v415: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:44:07.386 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:07 smithi096 ceph-mon[102928]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:44:07.409 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:44:07.451 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:07.453 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:07.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.481 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:07.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:07.582 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:07.609 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:07.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:07.710 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:07.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:07 smithi204 ceph-mon[87071]: pgmap v415: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:44:07.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:07 smithi204 ceph-mon[87071]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:44:07.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.732 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:07.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:07.833 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:07.858 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:07.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:07.961 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:07.986 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:07.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:08.088 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:08.109 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:08.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:08.165 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:44:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:44:07] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.51.0" 2024-09-13T14:44:08.211 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:08.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.234 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:08.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:08.335 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:08.356 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:08.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:08.458 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:08.484 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:08.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:08.538 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:08.556 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:44:08.586 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:08.612 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:08.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:08.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:08.738 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:08.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:08.839 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:08.865 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:08.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:08.874 DEBUG:teuthology.orchestra.run.smithi096:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-09-13T14:44:08.874 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T14:44:08.874 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:44:08.876 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:44:08.877 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:44:08.924 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:hints = 1 2024-09-13T14:44:08.925 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:44:08.925 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Object prefix: benchmark_data_smithi096_167087 2024-09-13T14:44:08.925 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:44:08.925 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:44:08.967 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:09.000 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:09.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:09.104 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:09.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.150 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:09.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:09.251 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:09.281 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:09.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:09.383 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:09.413 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:09.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:09.515 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:09.545 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:09.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:09.648 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:09.687 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:09.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:09.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:09 smithi204 ceph-mon[87071]: pgmap v416: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:44:09.789 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:09.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.812 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:09.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:09.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:09 smithi096 ceph-mon[98901]: pgmap v416: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:44:09.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:09 smithi096 ceph-mon[102928]: pgmap v416: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:44:09.913 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:09.925 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 1 16 3283 3267 204.122 204.188 0.00403123 0.00487992 2024-09-13T14:44:09.949 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:09.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:10.051 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:10.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.102 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:10.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:10.203 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:10.246 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:10.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:10.348 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:10.385 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:10.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:10.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:44:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:44:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:10.487 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:10.517 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:10.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:10.596 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:10.601 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:10.619 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:10.666 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:10.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:10.768 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:10.789 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:10.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:10.890 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:10.911 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:10.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:10.925 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 2 16 6117 6101 190.613 177.125 0.00106401 0.00503796 2024-09-13T14:44:11.012 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:11.049 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:11.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:11.151 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:11.184 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:11.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:11.286 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:11.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.319 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:11.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:11.420 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:11.455 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:11 smithi096 ceph-mon[102928]: pgmap v417: 99 pgs: 99 active+clean; 11 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 707 KiB/s wr, 11 op/s 2024-09-13T14:44:11.455 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:11 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:44:11.455 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:11.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.456 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:11 smithi096 ceph-mon[98901]: pgmap v417: 99 pgs: 99 active+clean; 11 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 707 KiB/s wr, 11 op/s 2024-09-13T14:44:11.456 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:11 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:44:11.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:11.558 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:11.593 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:11.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:11.695 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:11 smithi204 ceph-mon[87071]: pgmap v417: 99 pgs: 99 active+clean; 11 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 707 KiB/s wr, 11 op/s 2024-09-13T14:44:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:11 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:44:11.733 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:11.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:11.835 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:11.872 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:11.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:11.925 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 3 16 8966 8950 186.424 178.062 0.00833751 0.00535853 2024-09-13T14:44:11.974 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:12.006 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:12.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:12.108 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:12.138 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:12.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:12.240 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:12.269 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:12.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:12.355 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:44:12.371 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:12.412 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:12.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:12.513 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:12.549 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:12.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:12.632 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:12 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/99609541' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:12.632 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:12 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:12.632 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:12 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/99609541' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:12.632 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:12 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:12.652 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:12.679 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:12.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:12.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:12 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/99609541' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:12.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:12 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:12.781 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:12.811 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:12.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:12.913 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:12.925 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 4 16 12193 12177 190.236 201.688 0.00682963 0.00524886 2024-09-13T14:44:12.942 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:12.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:13.044 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:13.082 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:13.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:13.184 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:13.214 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:13.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:13.316 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:13.347 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:13.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:13.412 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:13 smithi096 ceph-mon[98901]: pgmap v418: 99 pgs: 99 active+clean; 107 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 10 MiB/s wr, 163 op/s 2024-09-13T14:44:13.412 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:13 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:44:13.412 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:13 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:44:13.412 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:13 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:13.412 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:13 smithi096 ceph-mon[98901]: osdmap e150: 8 total, 8 up, 7 in 2024-09-13T14:44:13.413 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:13 smithi096 ceph-mon[102928]: pgmap v418: 99 pgs: 99 active+clean; 107 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 10 MiB/s wr, 163 op/s 2024-09-13T14:44:13.413 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:13 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:44:13.413 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:13 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:44:13.413 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:13 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:13.413 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:13 smithi096 ceph-mon[102928]: osdmap e150: 8 total, 8 up, 7 in 2024-09-13T14:44:13.448 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:13.475 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:13.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:13.577 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:13.580 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:13.581 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:13.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.617 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:13.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:13.718 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:13.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:13 smithi204 ceph-mon[87071]: pgmap v418: 99 pgs: 99 active+clean; 107 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 10 MiB/s wr, 163 op/s 2024-09-13T14:44:13.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:13 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:44:13.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:13 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:44:13.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:13 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:13.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:13 smithi204 ceph-mon[87071]: osdmap e150: 8 total, 8 up, 7 in 2024-09-13T14:44:13.756 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:13.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:13.858 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:13.896 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:13.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:13.925 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 5 16 14968 14952 186.873 173.438 0.00640624 0.00534532 2024-09-13T14:44:13.999 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:14.028 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:14.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:14.130 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:14.163 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:14.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:14.265 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:14.295 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:14.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:14.397 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:14.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.423 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:14.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:14.524 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:14.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.549 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:14.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:14.650 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:14.685 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:14.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:14.787 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:14.832 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:14.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:14.925 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 6 16 17459 17443 181.671 155.688 0.00469472 0.00549838 2024-09-13T14:44:14.934 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:14.977 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:14.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:15.079 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:15.112 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:15.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:15.213 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:15.253 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:15.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:15.355 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:15.391 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:15.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:15.493 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:15.523 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:15.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:15.624 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:15.663 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:15.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:15.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:15 smithi204 ceph-mon[87071]: pgmap v420: 99 pgs: 99 active+clean; 564 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:44:15.766 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:15.797 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:15.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:15.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:15 smithi096 ceph-mon[98901]: pgmap v420: 99 pgs: 99 active+clean; 564 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:44:15.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:15 smithi096 ceph-mon[102928]: pgmap v420: 99 pgs: 99 active+clean; 564 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:44:15.899 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:15.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.924 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:15.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:15.925 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 7 16 20048 20032 178.831 161.812 0.00278889 0.00557385 2024-09-13T14:44:16.025 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:16.053 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:16.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:16.155 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:16.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.197 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:16.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:16.298 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:16.341 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:16.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:16.443 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:16.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.466 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:16.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:16.568 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:16.599 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:16.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:16.701 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:16.729 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:16.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:16.831 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:16.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.872 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:16.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:16.900 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:44:16.926 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 8 16 22514 22498 175.741 154.125 0.00597255 0.00568454 2024-09-13T14:44:16.973 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:16.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.995 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:16.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:17.096 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:17.132 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:17.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:17.220 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:17.234 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:17.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.272 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:17.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:17.373 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:17.407 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:17.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:17.509 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:17.537 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:17.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:17.639 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:17.677 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:17.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:17.690 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:17 smithi204 ceph-mon[87071]: pgmap v421: 99 pgs: 99 active+clean; 620 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 61 MiB/s wr, 983 op/s 2024-09-13T14:44:17.729 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:17 smithi096 ceph-mon[98901]: pgmap v421: 99 pgs: 99 active+clean; 620 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 61 MiB/s wr, 983 op/s 2024-09-13T14:44:17.729 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:17 smithi096 ceph-mon[102928]: pgmap v421: 99 pgs: 99 active+clean; 620 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 61 MiB/s wr, 983 op/s 2024-09-13T14:44:17.779 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:17.845 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:17.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:17.926 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 9 16 24548 24532 170.339 127.125 0.00719342 0.00586377 2024-09-13T14:44:17.947 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:17.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.986 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:17.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:18.057 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:44:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:44:17] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.51.0" 2024-09-13T14:44:18.087 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:18.124 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:18.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:18.226 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:18.253 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:18.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:18.355 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:18.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.393 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:18.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:18.494 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:18.530 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:18.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:18.631 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:18.667 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:18.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:18.694 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:18 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2001247528' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:18.769 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:18.777 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:18 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2001247528' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:18.777 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:18 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2001247528' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:18.802 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:18.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:18.904 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:18.926 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 10 16 26621 26605 166.26 129.562 0.00663828 0.00600935 2024-09-13T14:44:18.941 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:18.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:19.043 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:19.078 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:19.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:19.180 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:19.206 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:44:19.206 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:44:19.206 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:44:19.206 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:44:19.206 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:44:19.206 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:44:19.206 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:44:19.207 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:44:19.207 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:19.207 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:19.207 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:19.207 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:19.207 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:19.207 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:19.207 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:19.207 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:19.211 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:19.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:19.313 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:19.337 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:44:19.337 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:44:19.338 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.338 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:27520 6.e 6:7bfb0ea4:::benchmark_data_smithi096_167087_object27519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:44:19.338 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:19.332307+0000", 2024-09-13T14:44:19.338 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0042480729999999998, 2024-09-13T14:44:19.338 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0042821530000000003, 2024-09-13T14:44:19.338 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:19.338 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:44:19.338 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:19.338 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:44:19.338 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:44:19.338 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 27520 2024-09-13T14:44:19.338 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.338 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:19.338 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.339 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:19.339 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.332307+0000", 2024-09-13T14:44:19.339 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:19.339 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.339 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.339 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:19.339 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.332307+0000", 2024-09-13T14:44:19.339 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:19.339 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.339 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.339 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:19.339 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.332306+0000", 2024-09-13T14:44:19.339 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:19.340 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.340 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:19.340 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.332338+0000", 2024-09-13T14:44:19.340 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2150000000000002e-05 2024-09-13T14:44:19.340 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.340 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:19.340 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.332346+0000", 2024-09-13T14:44:19.340 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0649999999999999e-06 2024-09-13T14:44:19.340 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.340 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:19.340 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.332351+0000", 2024-09-13T14:44:19.341 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7759999999999997e-06 2024-09-13T14:44:19.341 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.341 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.341 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:19.341 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.333605+0000", 2024-09-13T14:44:19.341 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001253601 2024-09-13T14:44:19.341 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.341 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.341 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:19.341 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.333734+0000", 2024-09-13T14:44:19.341 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012904799999999999 2024-09-13T14:44:19.341 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.341 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.342 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:44:19.342 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.333861+0000", 2024-09-13T14:44:19.342 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012714099999999999 2024-09-13T14:44:19.342 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.342 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.342 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:19.342 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.334882+0000", 2024-09-13T14:44:19.342 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001021304 2024-09-13T14:44:19.342 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.342 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.342 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:19.342 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.335994+0000", 2024-09-13T14:44:19.342 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001112133 2024-09-13T14:44:19.342 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.342 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.343 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:19.343 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.336577+0000", 2024-09-13T14:44:19.343 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00058264600000000003 2024-09-13T14:44:19.343 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:19.343 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:19.343 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:19.343 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.343 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.343 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.45167.0:27520 6.e e150/145 ondisk, result = 0)", 2024-09-13T14:44:19.343 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:19.336392+0000", 2024-09-13T14:44:19.343 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.00016288399999999999, 2024-09-13T14:44:19.343 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000297853, 2024-09-13T14:44:19.343 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:19.344 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:44:19.344 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:19.344 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.344 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:19.344 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.336392+0000", 2024-09-13T14:44:19.344 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:19.344 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.344 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.344 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:19.344 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.336392+0000", 2024-09-13T14:44:19.344 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:19.344 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.344 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.344 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:19.345 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.336392+0000", 2024-09-13T14:44:19.345 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:19.345 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.345 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.345 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:19.345 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.336394+0000", 2024-09-13T14:44:19.345 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.785e-06 2024-09-13T14:44:19.345 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.345 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.345 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:19.345 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.336400+0000", 2024-09-13T14:44:19.345 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0449999999999997e-06 2024-09-13T14:44:19.345 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.345 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.346 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:19.346 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.336404+0000", 2024-09-13T14:44:19.346 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7919999999999999e-06 2024-09-13T14:44:19.346 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.346 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.346 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:19.346 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.336564+0000", 2024-09-13T14:44:19.346 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015934900000000001 2024-09-13T14:44:19.346 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:19.346 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:19.346 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:19.347 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:19.336570+0000", 2024-09-13T14:44:19.347 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4699999999999999e-06 2024-09-13T14:44:19.347 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:19.347 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:19.347 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:19.347 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:19.347 INFO:teuthology.orchestra.run.smithi204.stdout: ], 2024-09-13T14:44:19.347 INFO:teuthology.orchestra.run.smithi204.stdout: "num_ops": 2 2024-09-13T14:44:19.347 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:44:19.349 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:19.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:19.450 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:19.489 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:19.490 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:44:19.591 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:19.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.629 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:19.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:19.696 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:19 smithi204 ceph-mon[87071]: pgmap v422: 99 pgs: 99 active+clean; 1.3 GiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-09-13T14:44:19.729 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:19.763 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:19.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:19.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:19 smithi096 ceph-mon[102928]: pgmap v422: 99 pgs: 99 active+clean; 1.3 GiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-09-13T14:44:19.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:19 smithi096 ceph-mon[98901]: pgmap v422: 99 pgs: 99 active+clean; 1.3 GiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-09-13T14:44:19.864 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:19.909 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:19.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:19.926 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 11 16 28768 28752 163.342 134.188 0.00469357 0.00611611 2024-09-13T14:44:20.010 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:20.043 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:44:20.046 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:20.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:20.148 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:20.188 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:20.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:20.290 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:20.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.325 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:20.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:20.426 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:20.439 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:20.463 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:20.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:44:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:44:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:20.565 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:20.603 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:20.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:20.705 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:20.742 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:20.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:20.844 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:20.896 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:20.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:20.926 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 12 16 30877 30861 160.714 131.812 0.0258201 0.0062147 2024-09-13T14:44:20.998 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:21.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.040 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:21.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:21.141 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:21.163 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:21.191 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:21.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:21.293 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:21.320 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:21.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:21.422 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:21.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.461 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:21.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:21.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:21 smithi204 ceph-mon[87071]: pgmap v423: 99 pgs: 99 active+clean; 1.4 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-09-13T14:44:21.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:44:21.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:21.515 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:44:21.516 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:44:21.562 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:21.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:21 smithi096 ceph-mon[98901]: pgmap v423: 99 pgs: 99 active+clean; 1.4 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-09-13T14:44:21.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:44:21.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:21.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:21 smithi096 ceph-mon[102928]: pgmap v423: 99 pgs: 99 active+clean; 1.4 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-09-13T14:44:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:44:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:21.603 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:21.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:21.705 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:21.740 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:21.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:21.842 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:21.871 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:21.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:21.926 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 13 16 32811 32795 157.649 120.875 0.00328266 0.00633753 2024-09-13T14:44:21.974 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:22.014 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:22.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:22.116 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:22.152 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:22.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:22.254 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:22.285 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:22.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:22.387 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:22.414 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:22.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:22.422 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:44:22.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:22.542 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:22.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:22.600 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":423,"stamp":"2024-09-13T14:44:19.725191+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1513297869,"num_objects":23046,"num_object_clones":0,"num_object_copies":69138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23046,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":23925,"num_write_kb":1485775,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":4203971,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":25150,"ondisk_log_size":25150,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5938696,"kb_used_data":4371536,"kb_used_omap":34,"kb_used_meta":1567069,"kb_avail":650277368,"statfs":{"total":671965249536,"available":665884024832,"internally_reserved":0,"allocated":4476452864,"data_stored":4470132207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35126,"internal_metadata":1604679370},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,5,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1501691904,"num_objects":22914,"num_object_clones":0,"num_object_copies":68742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22914,"num_write_kb":1466496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003040"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":165,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.360361+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:44:12.360361+0000","last_peered":"2024-09-13T14:44:12.360361+0000","last_clean":"2024-09-13T14:44:12.360361+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:44:12.360361+0000","last_undegraded":"2024-09-13T14:44:12.360361+0000","last_fullsized":"2024-09-13T14:44:12.360361+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":116,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.319428+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:44:05.319428+0000","last_peered":"2024-09-13T14:44:05.319428+0000","last_clean":"2024-09-13T14:44:05.319428+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:44:05.319428+0000","last_undegraded":"2024-09-13T14:44:05.319428+0000","last_fullsized":"2024-09-13T14:44:05.319428+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":113,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.320358+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:44:05.320358+0000","last_peered":"2024-09-13T14:44:05.320358+0000","last_clean":"2024-09-13T14:44:05.320358+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:44:05.320358+0000","last_undegraded":"2024-09-13T14:44:05.320358+0000","last_fullsized":"2024-09-13T14:44:05.320358+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":158,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.362384+0000","last_change":"2024-09-13T14:38:13.977954+0000","last_active":"2024-09-13T14:44:12.362384+0000","last_peered":"2024-09-13T14:44:12.362384+0000","last_clean":"2024-09-13T14:44:12.362384+0000","last_became_active":"2024-09-13T14:38:13.977619+0000","last_became_peered":"2024-09-13T14:38:13.977619+0000","last_unstale":"2024-09-13T14:44:12.362384+0000","last_undegraded":"2024-09-13T14:44:12.362384+0000","last_fullsized":"2024-09-13T14:44:12.362384+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:58:30.259967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":159,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.363024+0000","last_change":"2024-09-13T14:38:13.978232+0000","last_active":"2024-09-13T14:44:12.363024+0000","last_peered":"2024-09-13T14:44:12.363024+0000","last_clean":"2024-09-13T14:44:12.363024+0000","last_became_active":"2024-09-13T14:38:13.977921+0000","last_became_peered":"2024-09-13T14:38:13.977921+0000","last_unstale":"2024-09-13T14:44:12.363024+0000","last_undegraded":"2024-09-13T14:44:12.363024+0000","last_fullsized":"2024-09-13T14:44:12.363024+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:10.127626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":107,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.319230+0000","last_change":"2024-09-13T14:39:42.765301+0000","last_active":"2024-09-13T14:44:05.319230+0000","last_peered":"2024-09-13T14:44:05.319230+0000","last_clean":"2024-09-13T14:44:05.319230+0000","last_became_active":"2024-09-13T14:39:42.764977+0000","last_became_peered":"2024-09-13T14:39:42.764977+0000","last_unstale":"2024-09-13T14:44:05.319230+0000","last_undegraded":"2024-09-13T14:44:05.319230+0000","last_fullsized":"2024-09-13T14:44:05.319230+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:25.223972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":145,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.362606+0000","last_change":"2024-09-13T14:37:40.925691+0000","last_active":"2024-09-13T14:44:12.362606+0000","last_peered":"2024-09-13T14:44:12.362606+0000","last_clean":"2024-09-13T14:44:12.362606+0000","last_became_active":"2024-09-13T14:37:40.925360+0000","last_became_peered":"2024-09-13T14:37:40.925360+0000","last_unstale":"2024-09-13T14:44:12.362606+0000","last_undegraded":"2024-09-13T14:44:12.362606+0000","last_fullsized":"2024-09-13T14:44:12.362606+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:07.337967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.361603+0000","last_change":"2024-09-13T14:39:42.781872+0000","last_active":"2024-09-13T14:44:12.361603+0000","last_peered":"2024-09-13T14:44:12.361603+0000","last_clean":"2024-09-13T14:44:12.361603+0000","last_became_active":"2024-09-13T14:39:42.770254+0000","last_became_peered":"2024-09-13T14:39:42.770254+0000","last_unstale":"2024-09-13T14:44:12.361603+0000","last_undegraded":"2024-09-13T14:44:12.361603+0000","last_fullsized":"2024-09-13T14:44:12.361603+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:31.211081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"150'1408","reported_seq":1420,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:16.809398+0000","last_change":"2024-09-13T14:43:58.298744+0000","last_active":"2024-09-13T14:44:16.809398+0000","last_peered":"2024-09-13T14:44:16.809398+0000","last_clean":"2024-09-13T14:44:16.809398+0000","last_became_active":"2024-09-13T14:43:58.298443+0000","last_became_peered":"2024-09-13T14:43:58.298443+0000","last_unstale":"2024-09-13T14:44:16.809398+0000","last_undegraded":"2024-09-13T14:44:16.809398+0000","last_fullsized":"2024-09-13T14:44:16.809398+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1408,"log_dups_size":0,"ondisk_log_size":1408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:26:37.065070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92274688,"num_objects":1408,"num_object_clones":0,"num_object_copies":4224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1408,"num_write_kb":90112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":149,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.358047+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:44:12.358047+0000","last_peered":"2024-09-13T14:44:12.358047+0000","last_clean":"2024-09-13T14:44:12.358047+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:44:12.358047+0000","last_undegraded":"2024-09-13T14:44:12.358047+0000","last_fullsized":"2024-09-13T14:44:12.358047+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":106,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.319150+0000","last_change":"2024-09-13T14:39:42.764876+0000","last_active":"2024-09-13T14:44:05.319150+0000","last_peered":"2024-09-13T14:44:05.319150+0000","last_clean":"2024-09-13T14:44:05.319150+0000","last_became_active":"2024-09-13T14:39:42.764082+0000","last_became_peered":"2024-09-13T14:39:42.764082+0000","last_unstale":"2024-09-13T14:44:05.319150+0000","last_undegraded":"2024-09-13T14:44:05.319150+0000","last_fullsized":"2024-09-13T14:44:05.319150+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:10:07.458952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"150'1426","reported_seq":1446,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:16.850303+0000","last_change":"2024-09-13T14:43:58.301093+0000","last_active":"2024-09-13T14:44:16.850303+0000","last_peered":"2024-09-13T14:44:16.850303+0000","last_clean":"2024-09-13T14:44:16.850303+0000","last_became_active":"2024-09-13T14:43:58.300782+0000","last_became_peered":"2024-09-13T14:43:58.300782+0000","last_unstale":"2024-09-13T14:44:16.850303+0000","last_undegraded":"2024-09-13T14:44:16.850303+0000","last_fullsized":"2024-09-13T14:44:16.850303+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1426,"log_dups_size":0,"ondisk_log_size":1426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:48:49.617848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93454336,"num_objects":1426,"num_object_clones":0,"num_object_copies":4278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1426,"num_write_kb":91264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":116,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.319589+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:44:05.319589+0000","last_peered":"2024-09-13T14:44:05.319589+0000","last_clean":"2024-09-13T14:44:05.319589+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:44:05.319589+0000","last_undegraded":"2024-09-13T14:44:05.319589+0000","last_fullsized":"2024-09-13T14:44:05.319589+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":141,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.413619+0000","last_change":"2024-09-13T14:38:13.979677+0000","last_active":"2024-09-13T14:44:05.413619+0000","last_peered":"2024-09-13T14:44:05.413619+0000","last_clean":"2024-09-13T14:44:05.413619+0000","last_became_active":"2024-09-13T14:38:13.979323+0000","last_became_peered":"2024-09-13T14:38:13.979323+0000","last_unstale":"2024-09-13T14:44:05.413619+0000","last_undegraded":"2024-09-13T14:44:05.413619+0000","last_fullsized":"2024-09-13T14:44:05.413619+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:29:03.491254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"150'1409","reported_seq":1422,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:16.697274+0000","last_change":"2024-09-13T14:43:58.298623+0000","last_active":"2024-09-13T14:44:16.697274+0000","last_peered":"2024-09-13T14:44:16.697274+0000","last_clean":"2024-09-13T14:44:16.697274+0000","last_became_active":"2024-09-13T14:43:58.298319+0000","last_became_peered":"2024-09-13T14:43:58.298319+0000","last_unstale":"2024-09-13T14:44:16.697274+0000","last_undegraded":"2024-09-13T14:44:16.697274+0000","last_fullsized":"2024-09-13T14:44:16.697274+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1409,"log_dups_size":0,"ondisk_log_size":1409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:45.641123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92340224,"num_objects":1409,"num_object_clones":0,"num_object_copies":4227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1409,"num_write_kb":90176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":112,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.319898+0000","last_change":"2024-09-13T14:39:42.774547+0000","last_active":"2024-09-13T14:44:05.319898+0000","last_peered":"2024-09-13T14:44:05.319898+0000","last_clean":"2024-09-13T14:44:05.319898+0000","last_became_active":"2024-09-13T14:39:42.766006+0000","last_became_peered":"2024-09-13T14:39:42.766006+0000","last_unstale":"2024-09-13T14:44:05.319898+0000","last_undegraded":"2024-09-13T14:44:05.319898+0000","last_fullsized":"2024-09-13T14:44:05.319898+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:06:29.632163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":149,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.358435+0000","last_change":"2024-09-13T14:39:42.765763+0000","last_active":"2024-09-13T14:44:12.358435+0000","last_peered":"2024-09-13T14:44:12.358435+0000","last_clean":"2024-09-13T14:44:12.358435+0000","last_became_active":"2024-09-13T14:39:42.765441+0000","last_became_peered":"2024-09-13T14:39:42.765441+0000","last_unstale":"2024-09-13T14:44:12.358435+0000","last_undegraded":"2024-09-13T14:44:12.358435+0000","last_fullsized":"2024-09-13T14:44:12.358435+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:06.483638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"150'1645","reported_seq":1666,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:18.493401+0000","last_change":"2024-09-13T14:43:58.300633+0000","last_active":"2024-09-13T14:44:18.493401+0000","last_peered":"2024-09-13T14:44:18.493401+0000","last_clean":"2024-09-13T14:44:18.493401+0000","last_became_active":"2024-09-13T14:43:58.299977+0000","last_became_peered":"2024-09-13T14:43:58.299977+0000","last_unstale":"2024-09-13T14:44:18.493401+0000","last_undegraded":"2024-09-13T14:44:18.493401+0000","last_fullsized":"2024-09-13T14:44:18.493401+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1645,"log_dups_size":0,"ondisk_log_size":1645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:12:32.234019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107806720,"num_objects":1645,"num_object_clones":0,"num_object_copies":4935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1645,"num_write_kb":105280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":124,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.320955+0000","last_change":"2024-09-13T14:39:42.765558+0000","last_active":"2024-09-13T14:44:05.320955+0000","last_peered":"2024-09-13T14:44:05.320955+0000","last_clean":"2024-09-13T14:44:05.320955+0000","last_became_active":"2024-09-13T14:39:42.765246+0000","last_became_peered":"2024-09-13T14:39:42.765246+0000","last_unstale":"2024-09-13T14:44:05.320955+0000","last_undegraded":"2024-09-13T14:44:05.320955+0000","last_fullsized":"2024-09-13T14:44:05.320955+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:08.640610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":159,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.362493+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:44:12.362493+0000","last_peered":"2024-09-13T14:44:12.362493+0000","last_clean":"2024-09-13T14:44:12.362493+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:44:12.362493+0000","last_undegraded":"2024-09-13T14:44:12.362493+0000","last_fullsized":"2024-09-13T14:44:12.362493+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"150'1355","reported_seq":1368,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:16.694762+0000","last_change":"2024-09-13T14:43:58.298549+0000","last_active":"2024-09-13T14:44:16.694762+0000","last_peered":"2024-09-13T14:44:16.694762+0000","last_clean":"2024-09-13T14:44:16.694762+0000","last_became_active":"2024-09-13T14:43:58.298281+0000","last_became_peered":"2024-09-13T14:43:58.298281+0000","last_unstale":"2024-09-13T14:44:16.694762+0000","last_undegraded":"2024-09-13T14:44:16.694762+0000","last_fullsized":"2024-09-13T14:44:16.694762+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1355,"log_dups_size":0,"ondisk_log_size":1355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:28:44.626016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88801280,"num_objects":1355,"num_object_clones":0,"num_object_copies":4065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1355,"num_write_kb":86720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":277,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.318981+0000","last_change":"2024-09-13T14:40:32.590204+0000","last_active":"2024-09-13T14:44:05.318981+0000","last_peered":"2024-09-13T14:44:05.318981+0000","last_clean":"2024-09-13T14:44:05.318981+0000","last_became_active":"2024-09-13T14:40:22.949830+0000","last_became_peered":"2024-09-13T14:40:22.949830+0000","last_unstale":"2024-09-13T14:44:05.318981+0000","last_undegraded":"2024-09-13T14:44:05.318981+0000","last_fullsized":"2024-09-13T14:44:05.318981+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:32.472827+0000","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":162,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.358642+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:44:12.358642+0000","last_peered":"2024-09-13T14:44:12.358642+0000","last_clean":"2024-09-13T14:44:12.358642+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:44:12.358642+0000","last_undegraded":"2024-09-13T14:44:12.358642+0000","last_fullsized":"2024-09-13T14:44:12.358642+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":124,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.414018+0000","last_change":"2024-09-13T14:39:42.775584+0000","last_active":"2024-09-13T14:44:05.414018+0000","last_peered":"2024-09-13T14:44:05.414018+0000","last_clean":"2024-09-13T14:44:05.414018+0000","last_became_active":"2024-09-13T14:39:42.775254+0000","last_became_peered":"2024-09-13T14:39:42.775254+0000","last_unstale":"2024-09-13T14:44:05.414018+0000","last_undegraded":"2024-09-13T14:44:05.414018+0000","last_fullsized":"2024-09-13T14:44:05.414018+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:44.607098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"150'1737","reported_seq":1758,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:19.414922+0000","last_change":"2024-09-13T14:43:58.299969+0000","last_active":"2024-09-13T14:44:19.414922+0000","last_peered":"2024-09-13T14:44:19.414922+0000","last_clean":"2024-09-13T14:44:19.414922+0000","last_became_active":"2024-09-13T14:43:58.298592+0000","last_became_peered":"2024-09-13T14:43:58.298592+0000","last_unstale":"2024-09-13T14:44:19.414922+0000","last_undegraded":"2024-09-13T14:44:19.414922+0000","last_fullsized":"2024-09-13T14:44:19.414922+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1737,"log_dups_size":0,"ondisk_log_size":1737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:09:09.791130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113836032,"num_objects":1737,"num_object_clones":0,"num_object_copies":5211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1737,"num_write_kb":111168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":57,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.318612+0000","last_change":"2024-09-13T14:40:27.868402+0000","last_active":"2024-09-13T14:44:05.318612+0000","last_peered":"2024-09-13T14:44:05.318612+0000","last_clean":"2024-09-13T14:44:05.318612+0000","last_became_active":"2024-09-13T14:40:22.966301+0000","last_became_peered":"2024-09-13T14:40:22.966301+0000","last_unstale":"2024-09-13T14:44:05.318612+0000","last_undegraded":"2024-09-13T14:44:05.318612+0000","last_fullsized":"2024-09-13T14:44:05.318612+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:18.660186+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":106,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.413597+0000","last_change":"2024-09-13T14:39:42.776751+0000","last_active":"2024-09-13T14:44:05.413597+0000","last_peered":"2024-09-13T14:44:05.413597+0000","last_clean":"2024-09-13T14:44:05.413597+0000","last_became_active":"2024-09-13T14:39:42.774855+0000","last_became_peered":"2024-09-13T14:39:42.774855+0000","last_unstale":"2024-09-13T14:44:05.413597+0000","last_undegraded":"2024-09-13T14:44:05.413597+0000","last_fullsized":"2024-09-13T14:44:05.413597+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:11.022013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":131,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.414059+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:44:05.414059+0000","last_peered":"2024-09-13T14:44:05.414059+0000","last_clean":"2024-09-13T14:44:05.414059+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:44:05.414059+0000","last_undegraded":"2024-09-13T14:44:05.414059+0000","last_fullsized":"2024-09-13T14:44:05.414059+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"150'1352","reported_seq":1372,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:16.855519+0000","last_change":"2024-09-13T14:43:58.298605+0000","last_active":"2024-09-13T14:44:16.855519+0000","last_peered":"2024-09-13T14:44:16.855519+0000","last_clean":"2024-09-13T14:44:16.855519+0000","last_became_active":"2024-09-13T14:43:58.298341+0000","last_became_peered":"2024-09-13T14:43:58.298341+0000","last_unstale":"2024-09-13T14:44:16.855519+0000","last_undegraded":"2024-09-13T14:44:16.855519+0000","last_fullsized":"2024-09-13T14:44:16.855519+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1352,"log_dups_size":0,"ondisk_log_size":1352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:16.656493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88604672,"num_objects":1352,"num_object_clones":0,"num_object_copies":4056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1352,"num_write_kb":86528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":145,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.362145+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:44:12.362145+0000","last_peered":"2024-09-13T14:44:12.362145+0000","last_clean":"2024-09-13T14:44:12.362145+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:44:12.362145+0000","last_undegraded":"2024-09-13T14:44:12.362145+0000","last_fullsized":"2024-09-13T14:44:12.362145+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":124,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.414165+0000","last_change":"2024-09-13T14:39:42.776187+0000","last_active":"2024-09-13T14:44:05.414165+0000","last_peered":"2024-09-13T14:44:05.414165+0000","last_clean":"2024-09-13T14:44:05.414165+0000","last_became_active":"2024-09-13T14:39:42.775751+0000","last_became_peered":"2024-09-13T14:39:42.775751+0000","last_unstale":"2024-09-13T14:44:05.414165+0000","last_undegraded":"2024-09-13T14:44:05.414165+0000","last_fullsized":"2024-09-13T14:44:05.414165+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:35.155717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"150'1390","reported_seq":1402,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:16.808572+0000","last_change":"2024-09-13T14:43:58.298997+0000","last_active":"2024-09-13T14:44:16.808572+0000","last_peered":"2024-09-13T14:44:16.808572+0000","last_clean":"2024-09-13T14:44:16.808572+0000","last_became_active":"2024-09-13T14:43:58.298647+0000","last_became_peered":"2024-09-13T14:43:58.298647+0000","last_unstale":"2024-09-13T14:44:16.808572+0000","last_undegraded":"2024-09-13T14:44:16.808572+0000","last_fullsized":"2024-09-13T14:44:16.808572+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1390,"log_dups_size":0,"ondisk_log_size":1390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:38:43.997227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91095040,"num_objects":1390,"num_object_clones":0,"num_object_copies":4170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1390,"num_write_kb":88960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":50,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.320651+0000","last_change":"2024-09-13T14:40:25.861254+0000","last_active":"2024-09-13T14:44:05.320651+0000","last_peered":"2024-09-13T14:44:05.320651+0000","last_clean":"2024-09-13T14:44:05.320651+0000","last_became_active":"2024-09-13T14:40:22.957614+0000","last_became_peered":"2024-09-13T14:40:22.957614+0000","last_unstale":"2024-09-13T14:44:05.320651+0000","last_undegraded":"2024-09-13T14:44:05.320651+0000","last_fullsized":"2024-09-13T14:44:05.320651+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:53.388190+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":166,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.359046+0000","last_change":"2024-09-13T14:38:13.981576+0000","last_active":"2024-09-13T14:44:12.359046+0000","last_peered":"2024-09-13T14:44:12.359046+0000","last_clean":"2024-09-13T14:44:12.359046+0000","last_became_active":"2024-09-13T14:38:13.981262+0000","last_became_peered":"2024-09-13T14:38:13.981262+0000","last_unstale":"2024-09-13T14:44:12.359046+0000","last_undegraded":"2024-09-13T14:44:12.359046+0000","last_fullsized":"2024-09-13T14:44:12.359046+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:20.284250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":147,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.359343+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:44:12.359343+0000","last_peered":"2024-09-13T14:44:12.359343+0000","last_clean":"2024-09-13T14:44:12.359343+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:44:12.359343+0000","last_undegraded":"2024-09-13T14:44:12.359343+0000","last_fullsized":"2024-09-13T14:44:12.359343+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"150'1350","reported_seq":1361,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:16.813403+0000","last_change":"2024-09-13T14:43:58.298370+0000","last_active":"2024-09-13T14:44:16.813403+0000","last_peered":"2024-09-13T14:44:16.813403+0000","last_clean":"2024-09-13T14:44:16.813403+0000","last_became_active":"2024-09-13T14:43:58.298041+0000","last_became_peered":"2024-09-13T14:43:58.298041+0000","last_unstale":"2024-09-13T14:44:16.813403+0000","last_undegraded":"2024-09-13T14:44:16.813403+0000","last_fullsized":"2024-09-13T14:44:16.813403+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1350,"log_dups_size":0,"ondisk_log_size":1350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:28:59.876959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88473600,"num_objects":1350,"num_object_clones":0,"num_object_copies":4050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1350,"num_write_kb":86400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":268,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.319089+0000","last_change":"2024-09-13T14:40:22.639864+0000","last_active":"2024-09-13T14:44:05.319089+0000","last_peered":"2024-09-13T14:44:05.319089+0000","last_clean":"2024-09-13T14:44:05.319089+0000","last_became_active":"2024-09-13T14:40:20.926503+0000","last_became_peered":"2024-09-13T14:40:20.926503+0000","last_unstale":"2024-09-13T14:44:05.319089+0000","last_undegraded":"2024-09-13T14:44:05.319089+0000","last_fullsized":"2024-09-13T14:44:05.319089+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:22.639760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:22.639760+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:08.523563+0000","scrub_duration":0.0051120610000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":676,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.320461+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:44:05.320461+0000","last_peered":"2024-09-13T14:44:05.320461+0000","last_clean":"2024-09-13T14:44:05.320461+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:44:05.320461+0000","last_undegraded":"2024-09-13T14:44:05.320461+0000","last_fullsized":"2024-09-13T14:44:05.320461+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":123,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.359685+0000","last_change":"2024-09-13T14:39:42.768341+0000","last_active":"2024-09-13T14:44:12.359685+0000","last_peered":"2024-09-13T14:44:12.359685+0000","last_clean":"2024-09-13T14:44:12.359685+0000","last_became_active":"2024-09-13T14:39:42.768034+0000","last_became_peered":"2024-09-13T14:39:42.768034+0000","last_unstale":"2024-09-13T14:44:12.359685+0000","last_undegraded":"2024-09-13T14:44:12.359685+0000","last_fullsized":"2024-09-13T14:44:12.359685+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:28:55.502108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":56,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.413899+0000","last_change":"2024-09-13T14:39:42.772096+0000","last_active":"2024-09-13T14:44:05.413899+0000","last_peered":"2024-09-13T14:44:05.413899+0000","last_clean":"2024-09-13T14:44:05.413899+0000","last_became_active":"2024-09-13T14:39:42.771762+0000","last_became_peered":"2024-09-13T14:39:42.771762+0000","last_unstale":"2024-09-13T14:44:05.413899+0000","last_undegraded":"2024-09-13T14:44:05.413899+0000","last_fullsized":"2024-09-13T14:44:05.413899+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:16.172571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"150'1452","reported_seq":1472,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:17.193150+0000","last_change":"2024-09-13T14:43:58.298209+0000","last_active":"2024-09-13T14:44:17.193150+0000","last_peered":"2024-09-13T14:44:17.193150+0000","last_clean":"2024-09-13T14:44:17.193150+0000","last_became_active":"2024-09-13T14:43:58.297919+0000","last_became_peered":"2024-09-13T14:43:58.297919+0000","last_unstale":"2024-09-13T14:44:17.193150+0000","last_undegraded":"2024-09-13T14:44:17.193150+0000","last_fullsized":"2024-09-13T14:44:17.193150+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1452,"log_dups_size":0,"ondisk_log_size":1452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:14.599016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95158272,"num_objects":1452,"num_object_clones":0,"num_object_copies":4356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1452,"num_write_kb":92928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":276,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.358839+0000","last_change":"2024-09-13T14:40:23.784403+0000","last_active":"2024-09-13T14:44:12.358839+0000","last_peered":"2024-09-13T14:44:12.358839+0000","last_clean":"2024-09-13T14:44:12.358839+0000","last_became_active":"2024-09-13T14:40:22.951246+0000","last_became_peered":"2024-09-13T14:40:22.951246+0000","last_unstale":"2024-09-13T14:44:12.358839+0000","last_undegraded":"2024-09-13T14:44:12.358839+0000","last_fullsized":"2024-09-13T14:44:12.358839+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.784263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.784263+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:48.633093+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":361,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.318958+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:44:05.318958+0000","last_peered":"2024-09-13T14:44:05.318958+0000","last_clean":"2024-09-13T14:44:05.318958+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:44:05.318958+0000","last_undegraded":"2024-09-13T14:44:05.318958+0000","last_fullsized":"2024-09-13T14:44:05.318958+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":167,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.362233+0000","last_change":"2024-09-13T14:38:13.977962+0000","last_active":"2024-09-13T14:44:12.362233+0000","last_peered":"2024-09-13T14:44:12.362233+0000","last_clean":"2024-09-13T14:44:12.362233+0000","last_became_active":"2024-09-13T14:38:13.977638+0000","last_became_peered":"2024-09-13T14:38:13.977638+0000","last_unstale":"2024-09-13T14:44:12.362233+0000","last_undegraded":"2024-09-13T14:44:12.362233+0000","last_fullsized":"2024-09-13T14:44:12.362233+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:53:16.902157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":170,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.363207+0000","last_change":"2024-09-13T14:37:40.927444+0000","last_active":"2024-09-13T14:44:12.363207+0000","last_peered":"2024-09-13T14:44:12.363207+0000","last_clean":"2024-09-13T14:44:12.363207+0000","last_became_active":"2024-09-13T14:37:40.927140+0000","last_became_peered":"2024-09-13T14:37:40.927140+0000","last_unstale":"2024-09-13T14:44:12.363207+0000","last_undegraded":"2024-09-13T14:44:12.363207+0000","last_fullsized":"2024-09-13T14:44:12.363207+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:32:54.207568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"150'1312","reported_seq":1325,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:16.658490+0000","last_change":"2024-09-13T14:43:58.299024+0000","last_active":"2024-09-13T14:44:16.658490+0000","last_peered":"2024-09-13T14:44:16.658490+0000","last_clean":"2024-09-13T14:44:16.658490+0000","last_became_active":"2024-09-13T14:43:58.298323+0000","last_became_peered":"2024-09-13T14:43:58.298323+0000","last_unstale":"2024-09-13T14:44:16.658490+0000","last_undegraded":"2024-09-13T14:44:16.658490+0000","last_fullsized":"2024-09-13T14:44:16.658490+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1312,"log_dups_size":0,"ondisk_log_size":1312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:32.680552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85983232,"num_objects":1312,"num_object_clones":0,"num_object_copies":3936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1312,"num_write_kb":83968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":277,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.319138+0000","last_change":"2024-09-13T14:40:28.602999+0000","last_active":"2024-09-13T14:44:05.319138+0000","last_peered":"2024-09-13T14:44:05.319138+0000","last_clean":"2024-09-13T14:44:05.319138+0000","last_became_active":"2024-09-13T14:40:22.949002+0000","last_became_peered":"2024-09-13T14:40:22.949002+0000","last_unstale":"2024-09-13T14:44:05.319138+0000","last_undegraded":"2024-09-13T14:44:05.319138+0000","last_fullsized":"2024-09-13T14:44:05.319138+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:28.602942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:28.602942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:29:46.037596+0000","scrub_duration":0.00945176,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":571,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.357110+0000","last_change":"2024-09-13T14:38:13.978679+0000","last_active":"2024-09-13T14:44:12.357110+0000","last_peered":"2024-09-13T14:44:12.357110+0000","last_clean":"2024-09-13T14:44:12.357110+0000","last_became_active":"2024-09-13T14:38:13.978171+0000","last_became_peered":"2024-09-13T14:38:13.978171+0000","last_unstale":"2024-09-13T14:44:12.357110+0000","last_undegraded":"2024-09-13T14:44:12.357110+0000","last_fullsized":"2024-09-13T14:44:12.357110+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:57.691198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":154,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.358321+0000","last_change":"2024-09-13T14:39:42.769691+0000","last_active":"2024-09-13T14:44:12.358321+0000","last_peered":"2024-09-13T14:44:12.358321+0000","last_clean":"2024-09-13T14:44:12.358321+0000","last_became_active":"2024-09-13T14:39:42.769266+0000","last_became_peered":"2024-09-13T14:39:42.769266+0000","last_unstale":"2024-09-13T14:44:12.358321+0000","last_undegraded":"2024-09-13T14:44:12.358321+0000","last_fullsized":"2024-09-13T14:44:12.358321+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:11.637732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":180,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.359218+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:44:12.359218+0000","last_peered":"2024-09-13T14:44:12.359218+0000","last_clean":"2024-09-13T14:44:12.359218+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:44:12.359218+0000","last_undegraded":"2024-09-13T14:44:12.359218+0000","last_fullsized":"2024-09-13T14:44:12.359218+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"150'1377","reported_seq":1398,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:16.826579+0000","last_change":"2024-09-13T14:43:58.301790+0000","last_active":"2024-09-13T14:44:16.826579+0000","last_peered":"2024-09-13T14:44:16.826579+0000","last_clean":"2024-09-13T14:44:16.826579+0000","last_became_active":"2024-09-13T14:43:58.301504+0000","last_became_peered":"2024-09-13T14:43:58.301504+0000","last_unstale":"2024-09-13T14:44:16.826579+0000","last_undegraded":"2024-09-13T14:44:16.826579+0000","last_fullsized":"2024-09-13T14:44:16.826579+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1377,"log_dups_size":0,"ondisk_log_size":1377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:05:44.303022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90243072,"num_objects":1377,"num_object_clones":0,"num_object_copies":4131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1377,"num_write_kb":88128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":50,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.318638+0000","last_change":"2024-09-13T14:40:26.896691+0000","last_active":"2024-09-13T14:44:05.318638+0000","last_peered":"2024-09-13T14:44:05.318638+0000","last_clean":"2024-09-13T14:44:05.318638+0000","last_became_active":"2024-09-13T14:40:22.949772+0000","last_became_peered":"2024-09-13T14:40:22.949772+0000","last_unstale":"2024-09-13T14:44:05.318638+0000","last_undegraded":"2024-09-13T14:44:05.318638+0000","last_fullsized":"2024-09-13T14:44:05.318638+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:26.896613+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:26.896613+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:01.615433+0000","scrub_duration":0.004120828,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":740,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.320872+0000","last_change":"2024-09-13T14:39:42.805055+0000","last_active":"2024-09-13T14:44:05.320872+0000","last_peered":"2024-09-13T14:44:05.320872+0000","last_clean":"2024-09-13T14:44:05.320872+0000","last_became_active":"2024-09-13T14:39:42.762456+0000","last_became_peered":"2024-09-13T14:39:42.762456+0000","last_unstale":"2024-09-13T14:44:05.320872+0000","last_undegraded":"2024-09-13T14:44:05.320872+0000","last_fullsized":"2024-09-13T14:44:05.320872+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:54:40.839991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":125,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.413998+0000","last_change":"2024-09-13T14:39:42.775159+0000","last_active":"2024-09-13T14:44:05.413998+0000","last_peered":"2024-09-13T14:44:05.413998+0000","last_clean":"2024-09-13T14:44:05.413998+0000","last_became_active":"2024-09-13T14:39:42.774841+0000","last_became_peered":"2024-09-13T14:39:42.774841+0000","last_unstale":"2024-09-13T14:44:05.413998+0000","last_undegraded":"2024-09-13T14:44:05.413998+0000","last_fullsized":"2024-09-13T14:44:05.413998+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:35:45.579397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":158,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.358262+0000","last_change":"2024-09-13T14:38:13.981294+0000","last_active":"2024-09-13T14:44:12.358262+0000","last_peered":"2024-09-13T14:44:12.358262+0000","last_clean":"2024-09-13T14:44:12.358262+0000","last_became_active":"2024-09-13T14:38:13.980975+0000","last_became_peered":"2024-09-13T14:38:13.980975+0000","last_unstale":"2024-09-13T14:44:12.358262+0000","last_undegraded":"2024-09-13T14:44:12.358262+0000","last_fullsized":"2024-09-13T14:44:12.358262+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:17.545179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"150'1597","reported_seq":1618,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:18.499084+0000","last_change":"2024-09-13T14:43:58.301421+0000","last_active":"2024-09-13T14:44:18.499084+0000","last_peered":"2024-09-13T14:44:18.499084+0000","last_clean":"2024-09-13T14:44:18.499084+0000","last_became_active":"2024-09-13T14:43:58.301132+0000","last_became_peered":"2024-09-13T14:43:58.301132+0000","last_unstale":"2024-09-13T14:44:18.499084+0000","last_undegraded":"2024-09-13T14:44:18.499084+0000","last_fullsized":"2024-09-13T14:44:18.499084+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1597,"log_dups_size":0,"ondisk_log_size":1597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:58.812457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104660992,"num_objects":1597,"num_object_clones":0,"num_object_copies":4791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1597,"num_write_kb":102208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":85,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.358869+0000","last_change":"2024-09-13T14:40:23.093425+0000","last_active":"2024-09-13T14:44:12.358869+0000","last_peered":"2024-09-13T14:44:12.358869+0000","last_clean":"2024-09-13T14:44:12.358869+0000","last_became_active":"2024-09-13T14:40:22.947358+0000","last_became_peered":"2024-09-13T14:40:22.947358+0000","last_unstale":"2024-09-13T14:44:12.358869+0000","last_undegraded":"2024-09-13T14:44:12.358869+0000","last_fullsized":"2024-09-13T14:44:12.358869+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.093356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.093356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:39:00.143147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":131,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.320449+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:44:05.320449+0000","last_peered":"2024-09-13T14:44:05.320449+0000","last_clean":"2024-09-13T14:44:05.320449+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:44:05.320449+0000","last_undegraded":"2024-09-13T14:44:05.320449+0000","last_fullsized":"2024-09-13T14:44:05.320449+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":664,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.359750+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:44:12.359750+0000","last_peered":"2024-09-13T14:44:12.359750+0000","last_clean":"2024-09-13T14:44:12.359750+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:44:12.359750+0000","last_undegraded":"2024-09-13T14:44:12.359750+0000","last_fullsized":"2024-09-13T14:44:12.359750+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":136,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.358762+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:44:12.358762+0000","last_peered":"2024-09-13T14:44:12.358762+0000","last_clean":"2024-09-13T14:44:12.358762+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:44:12.358762+0000","last_undegraded":"2024-09-13T14:44:12.358762+0000","last_fullsized":"2024-09-13T14:44:12.358762+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"150'1457","reported_seq":1478,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:17.174441+0000","last_change":"2024-09-13T14:43:58.299661+0000","last_active":"2024-09-13T14:44:17.174441+0000","last_peered":"2024-09-13T14:44:17.174441+0000","last_clean":"2024-09-13T14:44:17.174441+0000","last_became_active":"2024-09-13T14:43:58.299408+0000","last_became_peered":"2024-09-13T14:43:58.299408+0000","last_unstale":"2024-09-13T14:44:17.174441+0000","last_undegraded":"2024-09-13T14:44:17.174441+0000","last_fullsized":"2024-09-13T14:44:17.174441+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1457,"log_dups_size":0,"ondisk_log_size":1457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:21:13.142607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95485952,"num_objects":1457,"num_object_clones":0,"num_object_copies":4371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1457,"num_write_kb":93248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":231,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.362533+0000","last_change":"2024-09-13T14:40:23.334441+0000","last_active":"2024-09-13T14:44:12.362533+0000","last_peered":"2024-09-13T14:44:12.362533+0000","last_clean":"2024-09-13T14:44:12.362533+0000","last_became_active":"2024-09-13T14:40:22.952908+0000","last_became_peered":"2024-09-13T14:40:22.952908+0000","last_unstale":"2024-09-13T14:44:12.362533+0000","last_undegraded":"2024-09-13T14:44:12.362533+0000","last_fullsized":"2024-09-13T14:44:12.362533+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.334379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.334379+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:31:23.257148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":147,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.359650+0000","last_change":"2024-09-13T14:37:40.921861+0000","last_active":"2024-09-13T14:44:12.359650+0000","last_peered":"2024-09-13T14:44:12.359650+0000","last_clean":"2024-09-13T14:44:12.359650+0000","last_became_active":"2024-09-13T14:37:40.921587+0000","last_became_peered":"2024-09-13T14:37:40.921587+0000","last_unstale":"2024-09-13T14:44:12.359650+0000","last_undegraded":"2024-09-13T14:44:12.359650+0000","last_fullsized":"2024-09-13T14:44:12.359650+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:24.619352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":363,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.320017+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:44:05.320017+0000","last_peered":"2024-09-13T14:44:05.320017+0000","last_clean":"2024-09-13T14:44:05.320017+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:44:05.320017+0000","last_undegraded":"2024-09-13T14:44:05.320017+0000","last_fullsized":"2024-09-13T14:44:05.320017+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":149,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.362402+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:44:12.362402+0000","last_peered":"2024-09-13T14:44:12.362402+0000","last_clean":"2024-09-13T14:44:12.362402+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:44:12.362402+0000","last_undegraded":"2024-09-13T14:44:12.362402+0000","last_fullsized":"2024-09-13T14:44:12.362402+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"150'1347","reported_seq":1360,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:16.723469+0000","last_change":"2024-09-13T14:43:58.300122+0000","last_active":"2024-09-13T14:44:16.723469+0000","last_peered":"2024-09-13T14:44:16.723469+0000","last_clean":"2024-09-13T14:44:16.723469+0000","last_became_active":"2024-09-13T14:43:58.299830+0000","last_became_peered":"2024-09-13T14:43:58.299830+0000","last_unstale":"2024-09-13T14:44:16.723469+0000","last_undegraded":"2024-09-13T14:44:16.723469+0000","last_fullsized":"2024-09-13T14:44:16.723469+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1347,"log_dups_size":0,"ondisk_log_size":1347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:18.285498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88276992,"num_objects":1347,"num_object_clones":0,"num_object_copies":4041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1347,"num_write_kb":86208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":51,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.413767+0000","last_change":"2024-09-13T14:40:30.500108+0000","last_active":"2024-09-13T14:44:05.413767+0000","last_peered":"2024-09-13T14:44:05.413767+0000","last_clean":"2024-09-13T14:44:05.413767+0000","last_became_active":"2024-09-13T14:40:22.949882+0000","last_became_peered":"2024-09-13T14:40:22.949882+0000","last_unstale":"2024-09-13T14:44:05.413767+0000","last_undegraded":"2024-09-13T14:44:05.413767+0000","last_fullsized":"2024-09-13T14:44:05.413767+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:40:30.500046+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:22:58.480202+0000","scrub_duration":0.00248832,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":162,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.359009+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:44:12.359009+0000","last_peered":"2024-09-13T14:44:12.359009+0000","last_clean":"2024-09-13T14:44:12.359009+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:44:12.359009+0000","last_undegraded":"2024-09-13T14:44:12.359009+0000","last_fullsized":"2024-09-13T14:44:12.359009+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":485,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.359639+0000","last_change":"2024-09-13T14:39:44.990216+0000","last_active":"2024-09-13T14:44:12.359639+0000","last_peered":"2024-09-13T14:44:12.359639+0000","last_clean":"2024-09-13T14:44:12.359639+0000","last_became_active":"2024-09-13T14:39:44.989433+0000","last_became_peered":"2024-09-13T14:39:44.989433+0000","last_unstale":"2024-09-13T14:44:12.359639+0000","last_undegraded":"2024-09-13T14:44:12.359639+0000","last_fullsized":"2024-09-13T14:44:12.359639+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:00.426042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":160,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.359264+0000","last_change":"2024-09-13T14:38:13.979255+0000","last_active":"2024-09-13T14:44:12.359264+0000","last_peered":"2024-09-13T14:44:12.359264+0000","last_clean":"2024-09-13T14:44:12.359264+0000","last_became_active":"2024-09-13T14:38:13.978868+0000","last_became_peered":"2024-09-13T14:38:13.978868+0000","last_unstale":"2024-09-13T14:44:12.359264+0000","last_undegraded":"2024-09-13T14:44:12.359264+0000","last_fullsized":"2024-09-13T14:44:12.359264+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:15:26.003538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"150'1404","reported_seq":1417,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:16.698858+0000","last_change":"2024-09-13T14:43:58.299991+0000","last_active":"2024-09-13T14:44:16.698858+0000","last_peered":"2024-09-13T14:44:16.698858+0000","last_clean":"2024-09-13T14:44:16.698858+0000","last_became_active":"2024-09-13T14:43:58.299386+0000","last_became_peered":"2024-09-13T14:43:58.299386+0000","last_unstale":"2024-09-13T14:44:16.698858+0000","last_undegraded":"2024-09-13T14:44:16.698858+0000","last_fullsized":"2024-09-13T14:44:16.698858+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":1404,"log_dups_size":0,"ondisk_log_size":1404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:24:28.041082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92012544,"num_objects":1404,"num_object_clones":0,"num_object_copies":4212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1404,"num_write_kb":89856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":90,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.360077+0000","last_change":"2024-09-13T14:40:24.124750+0000","last_active":"2024-09-13T14:44:12.360077+0000","last_peered":"2024-09-13T14:44:12.360077+0000","last_clean":"2024-09-13T14:44:12.360077+0000","last_became_active":"2024-09-13T14:40:22.958820+0000","last_became_peered":"2024-09-13T14:40:22.958820+0000","last_unstale":"2024-09-13T14:44:12.360077+0000","last_undegraded":"2024-09-13T14:44:12.360077+0000","last_fullsized":"2024-09-13T14:44:12.360077+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:24.124531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:24.124531+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:48:48.270340+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":170,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.362644+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:44:12.362644+0000","last_peered":"2024-09-13T14:44:12.362644+0000","last_clean":"2024-09-13T14:44:12.362644+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:44:12.362644+0000","last_undegraded":"2024-09-13T14:44:12.362644+0000","last_fullsized":"2024-09-13T14:44:12.362644+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1067,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.319755+0000","last_change":"2024-09-13T14:39:44.990389+0000","last_active":"2024-09-13T14:44:05.319755+0000","last_peered":"2024-09-13T14:44:05.319755+0000","last_clean":"2024-09-13T14:44:05.319755+0000","last_became_active":"2024-09-13T14:39:44.989805+0000","last_became_peered":"2024-09-13T14:39:44.989805+0000","last_unstale":"2024-09-13T14:44:05.319755+0000","last_undegraded":"2024-09-13T14:44:05.319755+0000","last_fullsized":"2024-09-13T14:44:05.319755+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:15.392445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":107,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.413430+0000","last_change":"2024-09-13T14:39:42.769954+0000","last_active":"2024-09-13T14:44:05.413430+0000","last_peered":"2024-09-13T14:44:05.413430+0000","last_clean":"2024-09-13T14:44:05.413430+0000","last_became_active":"2024-09-13T14:39:42.769635+0000","last_became_peered":"2024-09-13T14:39:42.769635+0000","last_unstale":"2024-09-13T14:44:05.413430+0000","last_undegraded":"2024-09-13T14:44:05.413430+0000","last_fullsized":"2024-09-13T14:44:05.413430+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:51.588071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":124,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.413871+0000","last_change":"2024-09-13T14:39:42.775357+0000","last_active":"2024-09-13T14:44:05.413871+0000","last_peered":"2024-09-13T14:44:05.413871+0000","last_clean":"2024-09-13T14:44:05.413871+0000","last_became_active":"2024-09-13T14:39:42.775041+0000","last_became_peered":"2024-09-13T14:39:42.775041+0000","last_unstale":"2024-09-13T14:44:05.413871+0000","last_undegraded":"2024-09-13T14:44:05.413871+0000","last_fullsized":"2024-09-13T14:44:05.413871+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:44.768910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":161,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.362291+0000","last_change":"2024-09-13T14:39:42.787151+0000","last_active":"2024-09-13T14:44:12.362291+0000","last_peered":"2024-09-13T14:44:12.362291+0000","last_clean":"2024-09-13T14:44:12.362291+0000","last_became_active":"2024-09-13T14:39:42.771712+0000","last_became_peered":"2024-09-13T14:39:42.771712+0000","last_unstale":"2024-09-13T14:44:12.362291+0000","last_undegraded":"2024-09-13T14:44:12.362291+0000","last_fullsized":"2024-09-13T14:44:12.362291+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:54.216663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":154,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.358616+0000","last_change":"2024-09-13T14:39:42.772974+0000","last_active":"2024-09-13T14:44:12.358616+0000","last_peered":"2024-09-13T14:44:12.358616+0000","last_clean":"2024-09-13T14:44:12.358616+0000","last_became_active":"2024-09-13T14:39:42.771449+0000","last_became_peered":"2024-09-13T14:39:42.771449+0000","last_unstale":"2024-09-13T14:44:12.358616+0000","last_undegraded":"2024-09-13T14:44:12.358616+0000","last_fullsized":"2024-09-13T14:44:12.358616+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:46.441079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":130,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.319719+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:44:05.319719+0000","last_peered":"2024-09-13T14:44:05.319719+0000","last_clean":"2024-09-13T14:44:05.319719+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:44:05.319719+0000","last_undegraded":"2024-09-13T14:44:05.319719+0000","last_fullsized":"2024-09-13T14:44:05.319719+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.320191+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:44:05.320191+0000","last_peered":"2024-09-13T14:44:05.320191+0000","last_clean":"2024-09-13T14:44:05.320191+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:44:05.320191+0000","last_undegraded":"2024-09-13T14:44:05.320191+0000","last_fullsized":"2024-09-13T14:44:05.320191+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":123,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.358625+0000","last_change":"2024-09-13T14:39:42.767764+0000","last_active":"2024-09-13T14:44:12.358625+0000","last_peered":"2024-09-13T14:44:12.358625+0000","last_clean":"2024-09-13T14:44:12.358625+0000","last_became_active":"2024-09-13T14:39:42.767450+0000","last_became_peered":"2024-09-13T14:39:42.767450+0000","last_unstale":"2024-09-13T14:44:12.358625+0000","last_undegraded":"2024-09-13T14:44:12.358625+0000","last_fullsized":"2024-09-13T14:44:12.358625+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:47.729382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":158,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.362704+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:44:12.362704+0000","last_peered":"2024-09-13T14:44:12.362704+0000","last_clean":"2024-09-13T14:44:12.362704+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:44:12.362704+0000","last_undegraded":"2024-09-13T14:44:12.362704+0000","last_fullsized":"2024-09-13T14:44:12.362704+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":172,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.359348+0000","last_change":"2024-09-13T14:39:42.772266+0000","last_active":"2024-09-13T14:44:12.359348+0000","last_peered":"2024-09-13T14:44:12.359348+0000","last_clean":"2024-09-13T14:44:12.359348+0000","last_became_active":"2024-09-13T14:39:42.771880+0000","last_became_peered":"2024-09-13T14:39:42.771880+0000","last_unstale":"2024-09-13T14:44:12.359348+0000","last_undegraded":"2024-09-13T14:44:12.359348+0000","last_fullsized":"2024-09-13T14:44:12.359348+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:41:01.160105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":104,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.320296+0000","last_change":"2024-09-13T14:39:42.773890+0000","last_active":"2024-09-13T14:44:05.320296+0000","last_peered":"2024-09-13T14:44:05.320296+0000","last_clean":"2024-09-13T14:44:05.320296+0000","last_became_active":"2024-09-13T14:39:42.773523+0000","last_became_peered":"2024-09-13T14:39:42.773523+0000","last_unstale":"2024-09-13T14:44:05.320296+0000","last_undegraded":"2024-09-13T14:44:05.320296+0000","last_fullsized":"2024-09-13T14:44:05.320296+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:13.156261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":183,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.359180+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:44:12.359180+0000","last_peered":"2024-09-13T14:44:12.359180+0000","last_clean":"2024-09-13T14:44:12.359180+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:44:12.359180+0000","last_undegraded":"2024-09-13T14:44:12.359180+0000","last_fullsized":"2024-09-13T14:44:12.359180+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":160,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.363115+0000","last_change":"2024-09-13T14:38:13.979604+0000","last_active":"2024-09-13T14:44:12.363115+0000","last_peered":"2024-09-13T14:44:12.363115+0000","last_clean":"2024-09-13T14:44:12.363115+0000","last_became_active":"2024-09-13T14:38:13.979288+0000","last_became_peered":"2024-09-13T14:38:13.979288+0000","last_unstale":"2024-09-13T14:44:12.363115+0000","last_undegraded":"2024-09-13T14:44:12.363115+0000","last_fullsized":"2024-09-13T14:44:12.363115+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:21.031527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":146,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.360055+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:44:12.360055+0000","last_peered":"2024-09-13T14:44:12.360055+0000","last_clean":"2024-09-13T14:44:12.360055+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:44:12.360055+0000","last_undegraded":"2024-09-13T14:44:12.360055+0000","last_fullsized":"2024-09-13T14:44:12.360055+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":179,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.358793+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:44:12.358793+0000","last_peered":"2024-09-13T14:44:12.358793+0000","last_clean":"2024-09-13T14:44:12.358793+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:44:12.358793+0000","last_undegraded":"2024-09-13T14:44:12.358793+0000","last_fullsized":"2024-09-13T14:44:12.358793+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":133,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.414032+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:44:05.414032+0000","last_peered":"2024-09-13T14:44:05.414032+0000","last_clean":"2024-09-13T14:44:05.414032+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:44:05.414032+0000","last_undegraded":"2024-09-13T14:44:05.414032+0000","last_fullsized":"2024-09-13T14:44:05.414032+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":185,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.356652+0000","last_change":"2024-09-13T14:38:13.976950+0000","last_active":"2024-09-13T14:44:12.356652+0000","last_peered":"2024-09-13T14:44:12.356652+0000","last_clean":"2024-09-13T14:44:12.356652+0000","last_became_active":"2024-09-13T14:38:13.976639+0000","last_became_peered":"2024-09-13T14:38:13.976639+0000","last_unstale":"2024-09-13T14:44:12.356652+0000","last_undegraded":"2024-09-13T14:44:12.356652+0000","last_fullsized":"2024-09-13T14:44:12.356652+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:24.126890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":151,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.362294+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:44:12.362294+0000","last_peered":"2024-09-13T14:44:12.362294+0000","last_clean":"2024-09-13T14:44:12.362294+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:44:12.362294+0000","last_undegraded":"2024-09-13T14:44:12.362294+0000","last_fullsized":"2024-09-13T14:44:12.362294+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":131,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.413837+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:44:05.413837+0000","last_peered":"2024-09-13T14:44:05.413837+0000","last_clean":"2024-09-13T14:44:05.413837+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:44:05.413837+0000","last_undegraded":"2024-09-13T14:44:05.413837+0000","last_fullsized":"2024-09-13T14:44:05.413837+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":116,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.320789+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:44:05.320789+0000","last_peered":"2024-09-13T14:44:05.320789+0000","last_clean":"2024-09-13T14:44:05.320789+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:44:05.320789+0000","last_undegraded":"2024-09-13T14:44:05.320789+0000","last_fullsized":"2024-09-13T14:44:05.320789+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":131,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.413806+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:44:05.413806+0000","last_peered":"2024-09-13T14:44:05.413806+0000","last_clean":"2024-09-13T14:44:05.413806+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:44:05.413806+0000","last_undegraded":"2024-09-13T14:44:05.413806+0000","last_fullsized":"2024-09-13T14:44:05.413806+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":141,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.320450+0000","last_change":"2024-09-13T14:39:42.778413+0000","last_active":"2024-09-13T14:44:05.320450+0000","last_peered":"2024-09-13T14:44:05.320450+0000","last_clean":"2024-09-13T14:44:05.320450+0000","last_became_active":"2024-09-13T14:39:42.766624+0000","last_became_peered":"2024-09-13T14:39:42.766624+0000","last_unstale":"2024-09-13T14:44:05.320450+0000","last_undegraded":"2024-09-13T14:44:05.320450+0000","last_fullsized":"2024-09-13T14:44:05.320450+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:13.435604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":105,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.319904+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:44:05.319904+0000","last_peered":"2024-09-13T14:44:05.319904+0000","last_clean":"2024-09-13T14:44:05.319904+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:44:05.319904+0000","last_undegraded":"2024-09-13T14:44:05.319904+0000","last_fullsized":"2024-09-13T14:44:05.319904+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":179,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.359805+0000","last_change":"2024-09-13T14:39:42.767451+0000","last_active":"2024-09-13T14:44:12.359805+0000","last_peered":"2024-09-13T14:44:12.359805+0000","last_clean":"2024-09-13T14:44:12.359805+0000","last_became_active":"2024-09-13T14:39:42.767086+0000","last_became_peered":"2024-09-13T14:39:42.767086+0000","last_unstale":"2024-09-13T14:44:12.359805+0000","last_undegraded":"2024-09-13T14:44:12.359805+0000","last_fullsized":"2024-09-13T14:44:12.359805+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:02.674069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":117,"reported_epoch":149,"state":"active+clean","last_fresh":"2024-09-13T14:44:05.413544+0000","last_change":"2024-09-13T14:39:42.768653+0000","last_active":"2024-09-13T14:44:05.413544+0000","last_peered":"2024-09-13T14:44:05.413544+0000","last_clean":"2024-09-13T14:44:05.413544+0000","last_became_active":"2024-09-13T14:39:42.768333+0000","last_became_peered":"2024-09-13T14:39:42.768333+0000","last_unstale":"2024-09-13T14:44:05.413544+0000","last_undegraded":"2024-09-13T14:44:05.413544+0000","last_fullsized":"2024-09-13T14:44:05.413544+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:25.655793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":163,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.362756+0000","last_change":"2024-09-13T14:37:40.924860+0000","last_active":"2024-09-13T14:44:12.362756+0000","last_peered":"2024-09-13T14:44:12.362756+0000","last_clean":"2024-09-13T14:44:12.362756+0000","last_became_active":"2024-09-13T14:37:40.924522+0000","last_became_peered":"2024-09-13T14:37:40.924522+0000","last_unstale":"2024-09-13T14:44:12.362756+0000","last_undegraded":"2024-09-13T14:44:12.362756+0000","last_fullsized":"2024-09-13T14:44:12.362756+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:10.340511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1508507648,"num_objects":23018,"num_object_clones":0,"num_object_copies":69054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23018,"num_write_kb":1473152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4656070656,"data_stored":4656070656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":23018,"ondisk_log_size":23018,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":9648,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397724,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1006748,"kb_used_data":696916,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92738404,"statfs":{"total":95995035648,"available":94964125696,"internally_reserved":0,"allocated":713641984,"data_stored":712732678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":184683593953,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790182,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":725200,"kb_used_data":427848,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93019952,"statfs":{"total":95995035648,"available":95252430848,"internally_reserved":0,"allocated":438116352,"data_stored":437215347,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986411,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":704912,"kb_used_data":407560,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93040240,"statfs":{"total":95995035648,"available":95273205760,"internally_reserved":0,"allocated":417341440,"data_stored":416439622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598859,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":928584,"kb_used_data":810816,"kb_used_omap":6,"kb_used_meta":117753,"kb_avail":92816568,"statfs":{"total":95995035648,"available":95044165632,"internally_reserved":0,"allocated":830275584,"data_stored":829382210,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":120580064},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762386,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1307676,"kb_used_data":1063572,"kb_used_omap":8,"kb_used_meta":244087,"kb_avail":92437476,"statfs":{"total":95995035648,"available":94655975424,"internally_reserved":0,"allocated":1089097728,"data_stored":1088189174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":249945546},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,5,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726925912,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":689160,"kb_used_data":554752,"kb_used_omap":7,"kb_used_meta":134392,"kb_avail":93055992,"statfs":{"total":95995035648,"available":95289335808,"internally_reserved":0,"allocated":568066048,"data_stored":567172323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":137618121},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122152,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":576416,"kb_used_data":410072,"kb_used_omap":7,"kb_used_meta":166328,"kb_avail":93168736,"statfs":{"total":95995035648,"available":95404785664,"internally_reserved":0,"allocated":419913728,"data_stored":419000853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":170320463},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":424607744,"data_stored":424607744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":568918016,"data_stored":568918016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1136787456,"data_stored":1136787456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":908001280,"data_stored":908001280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":442957824,"data_stored":442957824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":452395008,"data_stored":452395008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":722403328,"data_stored":722403328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:44:22.605 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 11 2024-09-13T14:44:22.643 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:22.670 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:22.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:22.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:22 smithi204 ceph-mon[87071]: from='client.45185 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:22.763 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:22 smithi096 ceph-mon[98901]: from='client.45185 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:22.764 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:22 smithi096 ceph-mon[102928]: from='client.45185 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:22.772 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:22.793 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:22.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:22.896 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:22.926 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 14 16 34215 34199 152.653 87.75 0.0108288 0.00654528 2024-09-13T14:44:22.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.940 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:22.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:23.041 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:23.087 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:23.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:23.189 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:23.214 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:23.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:23.316 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:23.345 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:23.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:23.391 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:44:23.391 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:44:23.391 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T14:44:23.391 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T14:44:23.391 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:44:23.447 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:23.476 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:23.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:23.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:23 smithi096 ceph-mon[98901]: pgmap v424: 99 pgs: 99 active+clean; 1.5 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 146 MiB/s wr, 2.33k op/s 2024-09-13T14:44:23.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:23 smithi096 ceph-mon[102928]: pgmap v424: 99 pgs: 99 active+clean; 1.5 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 146 MiB/s wr, 2.33k op/s 2024-09-13T14:44:23.579 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:23.599 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:23.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:23.701 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:23.711 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:23.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:23 smithi204 ceph-mon[87071]: pgmap v424: 99 pgs: 99 active+clean; 1.5 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 146 MiB/s wr, 2.33k op/s 2024-09-13T14:44:23.750 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:23.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.761 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:23.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:23.862 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:23.893 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:23.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:23.926 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 15 16 35718 35702 148.738 93.9375 0.0045783 0.00669811 2024-09-13T14:44:23.995 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:24.016 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:24.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:24.118 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:24.158 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:24.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:24.260 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:24.281 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:24.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:24.382 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:24.404 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:24.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:24.506 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:24.545 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:24.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:24.647 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:24.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.677 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:24.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:24.778 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:24.799 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:24.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:24.901 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:24.927 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 16 16 37125 37109 144.938 87.9375 0.00496256 0.0068935 2024-09-13T14:44:24.930 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:24.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:25.032 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:25.068 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:25.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:25.170 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:25.191 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:25.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:25.293 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:25.327 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:25.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:25.429 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:25.454 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:25.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:25.556 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:25.598 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:25.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:25.699 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:25.713 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:25 smithi096 ceph-mon[98901]: pgmap v425: 99 pgs: 99 active+clean; 2.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-09-13T14:44:25.713 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:25 smithi096 ceph-mon[102928]: pgmap v425: 99 pgs: 99 active+clean; 2.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-09-13T14:44:25.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:25 smithi204 ceph-mon[87071]: pgmap v425: 99 pgs: 99 active+clean; 2.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-09-13T14:44:25.722 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:25.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:25.824 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:25.863 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:25.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:25.927 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 17 16 38417 38401 141.162 80.75 0.0119176 0.00707813 2024-09-13T14:44:25.964 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:25.994 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:25.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:26.095 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:26.123 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:26.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:26.225 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:26.246 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:26.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:26.347 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:26.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.392 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:26.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:26.493 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:26.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.524 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:26.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:26.625 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:26.655 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:26.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:26.757 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:26.792 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:26.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:26.894 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:26.918 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:26.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:26.927 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 18 16 39625 39609 137.514 75.5 0.0179814 0.00726089 2024-09-13T14:44:27.020 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:27.047 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:27.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:27.149 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:27.189 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:27.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:27.291 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:27.328 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:27.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:27.427 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:44:27.430 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:27.449 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:27 smithi096 ceph-mon[102928]: pgmap v426: 99 pgs: 99 active+clean; 2.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-09-13T14:44:27.450 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:27 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2837248364' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:27.450 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:27 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/49384892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:27.450 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:27 smithi096 ceph-mon[98901]: pgmap v426: 99 pgs: 99 active+clean; 2.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-09-13T14:44:27.450 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:27 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2837248364' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:27.450 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:27 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/49384892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:27.469 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:27.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:27.571 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:27.574 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:27 smithi204 ceph-mon[87071]: pgmap v426: 99 pgs: 99 active+clean; 2.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-09-13T14:44:27.574 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:27 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2837248364' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:27.574 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:27 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/49384892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:27.597 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:27.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:27.604 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:44:27.699 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:27.739 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:27.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:27.841 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:27.873 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:27.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:27.927 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 19 16 41061 41045 135 89.75 0.015224 0.00740203 2024-09-13T14:44:27.975 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:27.980 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:27.996 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:27.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:28.098 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:28.135 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:28.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:28.144 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:44:28.144 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:44:28.145 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:44:28.145 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:44:28.145 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:44:28.145 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:44:28.145 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:44:28.145 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:44:28.145 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:28.145 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:28.145 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:28.145 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:28.145 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:28.145 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:28.145 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:28.146 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:28.237 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:28.267 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:28.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:28.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:44:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:44:27] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-13T14:44:28.369 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:28.400 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:28.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:28.502 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:28.529 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:28.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:28.631 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:28.657 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:28.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:28.665 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:28 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:28.666 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:28 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2837248364' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:44:28.666 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:28 smithi096 ceph-mon[98901]: osdmap e151: 8 total, 8 up, 7 in 2024-09-13T14:44:28.666 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:28 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:28.666 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:28 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2837248364' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:44:28.666 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:28 smithi096 ceph-mon[102928]: osdmap e151: 8 total, 8 up, 7 in 2024-09-13T14:44:28.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:28 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:28.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:28 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2837248364' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:44:28.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:28 smithi204 ceph-mon[87071]: osdmap e151: 8 total, 8 up, 7 in 2024-09-13T14:44:28.760 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:28.788 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:28.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:28.890 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:28.926 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:28.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:28.927 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:44:28.927362+0000 min lat: 0.00106401 max lat: 0.202978 avg lat: 0.00749346 2024-09-13T14:44:28.927 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:44:28.927 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 20 16 42693 42677 133.349 102 0.0058269 0.00749346 2024-09-13T14:44:29.028 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:29.066 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:29.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:29.168 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:29.203 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:29.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:29.305 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:29.345 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:29.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:29.447 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:29.477 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:29.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:29.539 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:29 smithi096 ceph-mon[98901]: pgmap v428: 99 pgs: 99 active+clean; 2.4 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 113 MiB/s wr, 1.80k op/s 2024-09-13T14:44:29.539 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:29 smithi096 ceph-mon[102928]: pgmap v428: 99 pgs: 99 active+clean; 2.4 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 113 MiB/s wr, 1.80k op/s 2024-09-13T14:44:29.578 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:29.606 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:29.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:29.708 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:29.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:29 smithi204 ceph-mon[87071]: pgmap v428: 99 pgs: 99 active+clean; 2.4 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 113 MiB/s wr, 1.80k op/s 2024-09-13T14:44:29.738 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:29.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:29.786 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:29.786 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:29.841 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:29.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.866 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:29.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:29.927 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 21 16 44363 44347 131.969 104.375 0.0105278 0.00757272 2024-09-13T14:44:29.967 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:29.994 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:29.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:30.096 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:30.134 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:30.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:30.236 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:30.265 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:30.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:30.367 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:30.403 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:30.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:30.431 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:44:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:44:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:30.451 INFO:teuthology.orchestra.run.smithi096.stderr:set pool 1 pgp_num to 11 2024-09-13T14:44:30.505 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:30.541 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:44:30.545 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:30.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:30.646 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:30.687 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:30.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:30.709 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:30 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2589711847' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:44:30.709 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:30 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:44:30.709 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:30 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2589711847' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:44:30.709 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:30 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:44:30.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:30 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2589711847' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:44:30.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:30 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:44:30.790 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:30.821 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:30.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:30.905 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:30.923 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:30.927 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 22 16 46092 46076 130.882 108.062 0.00882416 0.0076358 2024-09-13T14:44:30.957 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:30.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:31.059 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:31.096 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:31.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:31.198 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:31.232 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:31.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:31.247 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:44:31.334 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:31.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.367 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:31.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:31.468 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:31.494 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:31.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:31.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:31 smithi096 ceph-mon[102928]: pgmap v429: 99 pgs: 99 active+clean; 2.5 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-09-13T14:44:31.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:31 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:44:31.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:31 smithi096 ceph-mon[102928]: osdmap e152: 8 total, 8 up, 7 in 2024-09-13T14:44:31.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:31 smithi096 ceph-mon[98901]: pgmap v429: 99 pgs: 99 active+clean; 2.5 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-09-13T14:44:31.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:31 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:44:31.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:31 smithi096 ceph-mon[98901]: osdmap e152: 8 total, 8 up, 7 in 2024-09-13T14:44:31.595 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:31.622 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:31.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:31.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:31 smithi204 ceph-mon[87071]: pgmap v429: 99 pgs: 99 active+clean; 2.5 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-09-13T14:44:31.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:31 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:44:31.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:31 smithi204 ceph-mon[87071]: osdmap e152: 8 total, 8 up, 7 in 2024-09-13T14:44:31.724 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:31.733 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:31.752 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:31.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:31.853 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:31.884 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:31.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:31.927 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 23 16 47727 47711 129.634 102.188 0.0115403 0.00770885 2024-09-13T14:44:31.986 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:32.017 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:32.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:32.119 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:32.153 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:32.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:32.203 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:44:32.256 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:32.276 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:32.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:32.377 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:32.424 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:32.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:32.526 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:32.565 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:32.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:32.666 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:32.697 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:32.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:32.798 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:32.827 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:32.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:32.927 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 24 16 49267 49251 128.243 96.25 0.0088954 0.00779227 2024-09-13T14:44:32.929 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:32.956 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:32.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:33.057 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:33.093 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:33.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:33.194 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:33.232 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:33.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:33.284 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:33.285 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:33.334 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:33.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.369 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:33.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:33.470 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:33.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.501 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:33.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:33.602 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:33.616 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:33 smithi204 ceph-mon[87071]: pgmap v431: 99 pgs: 99 active+clean; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:44:33.628 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:33.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:33.729 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:33.736 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:33 smithi096 ceph-mon[98901]: pgmap v431: 99 pgs: 99 active+clean; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:44:33.736 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:33 smithi096 ceph-mon[102928]: pgmap v431: 99 pgs: 99 active+clean; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:44:33.776 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:33.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:33.878 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:33.906 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:33.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:33.927 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 25 16 50891 50875 127.173 101.5 0.0107104 0.00785798 2024-09-13T14:44:34.005 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:44:34.007 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:34.045 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:34.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:34.147 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:34.174 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:34.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:34.223 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:34.227 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:44:34.227 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:34.263 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:44:34.264 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:34.276 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:34.301 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:44:34.301 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:34.309 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:34.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:34.336 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:44:34.337 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:34.355 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:34.371 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:44:34.371 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:34.402 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:44:34.403 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:34.411 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:34.433 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:44:34.433 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:34.459 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:34.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:34.465 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:44:34.465 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:34.561 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:34.601 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:34.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:34.702 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:34.733 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:34.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:34.835 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:34.864 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:34.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:34.928 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 26 16 52298 52282 125.663 87.9375 0.015615 0.00795261 2024-09-13T14:44:34.966 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:34.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.996 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:34.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:35.097 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:35.134 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:35.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:35.212 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:35 smithi096 ceph-mon[98901]: pgmap v432: 99 pgs: 99 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:44:35.213 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:35 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/895568420' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:35.213 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:35 smithi096 ceph-mon[102928]: pgmap v432: 99 pgs: 99 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:44:35.213 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:35 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/895568420' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:35.236 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:35.265 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:35.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:35.367 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:35.396 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:35.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:35.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:35 smithi204 ceph-mon[87071]: pgmap v432: 99 pgs: 99 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:44:35.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:35 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/895568420' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:35.498 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:35.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.538 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:35.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:35.639 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:35.679 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:35.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:35.781 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:35.811 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:35.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:35.913 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:35.928 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 27 16 53915 53899 124.752 101.062 0.0161382 0.00801086 2024-09-13T14:44:35.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.948 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:35.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:36.050 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:36.081 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:36.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:36.173 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:44:36.173 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:44:36.173 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T14:44:36.173 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T14:44:36.173 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:44:36.183 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:36.222 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:36.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:36.287 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:44:36.287 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:44:36.287 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T14:44:36.287 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:44:36.324 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:36.352 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:36.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:36.455 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:36.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.496 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:36.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:36.597 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:36.628 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:36.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:36.730 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:36.767 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:36.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:36.832 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:44:36.869 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:36.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.906 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:36.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:36.928 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 28 16 55371 55355 123.546 91 0.00553932 0.00808836 2024-09-13T14:44:37.007 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:37.048 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:37.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:37.094 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:36 smithi096 ceph-mon[98901]: pgmap v433: 99 pgs: 99 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:44:37.094 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:44:37.094 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:37.094 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:36 smithi096 ceph-mon[102928]: pgmap v433: 99 pgs: 99 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:44:37.094 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:44:37.095 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:37.150 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:37.170 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:37.192 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:37.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:37.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:36 smithi204 ceph-mon[87071]: pgmap v433: 99 pgs: 99 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:44:37.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:44:37.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:37.294 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:37.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.339 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:37.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:37.440 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:37.477 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:37.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:37.566 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:37.579 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:37.608 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:37.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:37.710 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:37.740 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:37.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:37.842 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:37.869 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:37.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:37.928 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 29 16 56699 56683 122.148 83 0.00748379 0.00818125 2024-09-13T14:44:37.971 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:38.007 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:38.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:38.029 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:44:38.109 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:38.137 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:38.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:38.239 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:38.266 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:38.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:38.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:44:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:44:37] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-13T14:44:38.368 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:38.392 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:38.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:38.494 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:38.534 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:38.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:38.636 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:38.664 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:38.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:38.766 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:38.802 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:38.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:38.844 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:44:38.904 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:38.928 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 30 16 57885 57869 120.547 74.125 0.0127798 0.00829045 2024-09-13T14:44:38.936 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:38.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:39.038 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:39.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:38 smithi096 ceph-mon[98901]: pgmap v434: 99 pgs: 99 active+clean; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-13T14:44:39.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:38 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/567415978' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:38 smithi096 ceph-mon[102928]: pgmap v434: 99 pgs: 99 active+clean; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-13T14:44:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:38 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/567415978' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:39.079 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:39.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:39.174 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:38 smithi204 ceph-mon[87071]: pgmap v434: 99 pgs: 99 active+clean; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-13T14:44:39.174 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:38 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/567415978' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:39.181 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:39.213 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:39.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:39.315 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:39.345 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:39.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:39.447 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:39.488 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:39.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:39.590 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:39.618 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:39.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:39.720 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:39.756 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:39.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:39.804 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:44:39.804 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T14:44:39.804 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T14:44:39.805 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:44:39.805 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.805 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:68080 5.f 5:f42639ae:::benchmark_data_smithi096_138841_object68079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:44:39.805 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:25.517385+0000", 2024-09-13T14:44:39.805 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 254.283370811, 2024-09-13T14:44:39.805 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16118674699999999, 2024-09-13T14:44:39.805 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.805 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.805 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.805 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:44:39.805 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:44:39.805 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 68080 2024-09-13T14:44:39.806 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.806 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.806 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.806 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.806 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.517385+0000", 2024-09-13T14:44:39.806 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.806 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.806 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.806 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.806 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.517385+0000", 2024-09-13T14:44:39.806 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.806 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.807 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.807 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.807 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.517383+0000", 2024-09-13T14:44:39.807 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:39.807 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.807 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.807 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.807 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.517418+0000", 2024-09-13T14:44:39.807 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4798999999999997e-05 2024-09-13T14:44:39.807 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.807 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.808 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.808 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.517432+0000", 2024-09-13T14:44:39.808 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4199000000000001e-05 2024-09-13T14:44:39.808 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.808 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.808 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.808 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.517438+0000", 2024-09-13T14:44:39.808 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4600000000000002e-06 2024-09-13T14:44:39.808 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.808 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.808 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.809 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.517486+0000", 2024-09-13T14:44:39.809 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8133999999999997e-05 2024-09-13T14:44:39.809 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.809 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.809 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.809 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.517605+0000", 2024-09-13T14:44:39.809 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011911100000000001 2024-09-13T14:44:39.809 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.809 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.810 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:44:39.810 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.517740+0000", 2024-09-13T14:44:39.810 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000134707 2024-09-13T14:44:39.810 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.810 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.810 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.810 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.522028+0000", 2024-09-13T14:44:39.810 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0042880560000000002 2024-09-13T14:44:39.810 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.810 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.810 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.810 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.522177+0000", 2024-09-13T14:44:39.810 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014957499999999999 2024-09-13T14:44:39.811 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.811 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.811 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.811 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.678464+0000", 2024-09-13T14:44:39.811 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15628668500000001 2024-09-13T14:44:39.811 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.811 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.811 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.811 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.678531+0000", 2024-09-13T14:44:39.811 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6489999999999995e-05 2024-09-13T14:44:39.811 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.811 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.811 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.812 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.678572+0000", 2024-09-13T14:44:39.812 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1038999999999998e-05 2024-09-13T14:44:39.812 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.812 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.812 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.812 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.812 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.812 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:68083 5.f 5:f7da7ef6:::benchmark_data_smithi096_138841_object68082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:44:39.812 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:25.517811+0000", 2024-09-13T14:44:39.812 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 254.28294436499999, 2024-09-13T14:44:39.812 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16816617, 2024-09-13T14:44:39.812 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.812 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.813 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.813 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:44:39.813 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:44:39.813 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 68083 2024-09-13T14:44:39.813 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.813 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.813 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.813 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.813 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.517811+0000", 2024-09-13T14:44:39.813 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.813 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.813 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.813 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.814 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.517811+0000", 2024-09-13T14:44:39.814 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.814 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.814 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.814 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.814 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.517811+0000", 2024-09-13T14:44:39.814 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:39.814 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.814 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.814 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.814 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.517942+0000", 2024-09-13T14:44:39.814 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013176799999999999 2024-09-13T14:44:39.814 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.815 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.815 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.815 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.517955+0000", 2024-09-13T14:44:39.815 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2459e-05 2024-09-13T14:44:39.815 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.815 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.815 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.815 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.517960+0000", 2024-09-13T14:44:39.815 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.074e-06 2024-09-13T14:44:39.815 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.815 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.815 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.815 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.519672+0000", 2024-09-13T14:44:39.815 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0017121269999999999 2024-09-13T14:44:39.816 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.816 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.816 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.816 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.519792+0000", 2024-09-13T14:44:39.816 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000120419 2024-09-13T14:44:39.816 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.816 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.816 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:44:39.816 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.519952+0000", 2024-09-13T14:44:39.816 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000159376 2024-09-13T14:44:39.816 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.816 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.816 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.817 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.522197+0000", 2024-09-13T14:44:39.817 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0022452119999999999 2024-09-13T14:44:39.817 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.817 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.817 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.817 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.522351+0000", 2024-09-13T14:44:39.817 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015353400000000001 2024-09-13T14:44:39.817 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.817 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.817 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.817 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.685884+0000", 2024-09-13T14:44:39.817 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16353320700000001 2024-09-13T14:44:39.817 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.817 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.818 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.818 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.685938+0000", 2024-09-13T14:44:39.818 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4064000000000003e-05 2024-09-13T14:44:39.818 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.818 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.818 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.818 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.685977+0000", 2024-09-13T14:44:39.818 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9601999999999998e-05 2024-09-13T14:44:39.818 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.818 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.818 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.818 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.819 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.819 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:68093 5.f 5:f675a907:::benchmark_data_smithi096_138841_object68092:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:44:39.819 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:25.525915+0000", 2024-09-13T14:44:39.819 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 254.274840676, 2024-09-13T14:44:39.819 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.160774633, 2024-09-13T14:44:39.819 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.819 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.819 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.819 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:44:39.820 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:44:39.820 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 68093 2024-09-13T14:44:39.820 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.820 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.820 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.820 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.820 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.525915+0000", 2024-09-13T14:44:39.820 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.820 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.821 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.821 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.821 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.525915+0000", 2024-09-13T14:44:39.821 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.821 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.821 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.821 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.821 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.525913+0000", 2024-09-13T14:44:39.821 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:44:39.821 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.821 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.821 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.821 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.525959+0000", 2024-09-13T14:44:39.822 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6051999999999999e-05 2024-09-13T14:44:39.822 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.822 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.822 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.822 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.525976+0000", 2024-09-13T14:44:39.822 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6582000000000001e-05 2024-09-13T14:44:39.822 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.822 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.822 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.822 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.525982+0000", 2024-09-13T14:44:39.822 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9440000000000002e-06 2024-09-13T14:44:39.822 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.822 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.822 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.823 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.526026+0000", 2024-09-13T14:44:39.823 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3976e-05 2024-09-13T14:44:39.823 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.823 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.823 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.823 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.526158+0000", 2024-09-13T14:44:39.823 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000132084 2024-09-13T14:44:39.823 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.823 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.823 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:44:39.823 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.526306+0000", 2024-09-13T14:44:39.823 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014774100000000001 2024-09-13T14:44:39.823 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.823 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.824 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.824 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.528047+0000", 2024-09-13T14:44:39.824 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001741732 2024-09-13T14:44:39.824 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.824 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.824 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.824 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.528194+0000", 2024-09-13T14:44:39.824 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000146978 2024-09-13T14:44:39.824 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.824 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.824 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.824 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.686628+0000", 2024-09-13T14:44:39.825 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15843365600000001 2024-09-13T14:44:39.825 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.825 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.825 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.825 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.686658+0000", 2024-09-13T14:44:39.825 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9765999999999999e-05 2024-09-13T14:44:39.825 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.825 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.825 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.825 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.686690+0000", 2024-09-13T14:44:39.825 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1942e-05 2024-09-13T14:44:39.825 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.825 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.825 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.825 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.826 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.826 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:155322 5.7 5:eae81fbc:::benchmark_data_smithi096_138841_object25126:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:44:39.826 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:41:47.974083+0000", 2024-09-13T14:44:39.826 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 171.826672922, 2024-09-13T14:44:39.826 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16403873299999999, 2024-09-13T14:44:39.826 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.826 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.826 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.826 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:44:39.826 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:44:39.826 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 155322 2024-09-13T14:44:39.826 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.826 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.826 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.826 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.827 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.974083+0000", 2024-09-13T14:44:39.827 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.827 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.827 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.827 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.827 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.974083+0000", 2024-09-13T14:44:39.827 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.827 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.827 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.827 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.827 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.974082+0000", 2024-09-13T14:44:39.827 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:44:39.827 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.827 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.828 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.828 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.974090+0000", 2024-09-13T14:44:39.828 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5900000000000002e-06 2024-09-13T14:44:39.828 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.828 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.828 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.828 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.974101+0000", 2024-09-13T14:44:39.828 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1243999999999999e-05 2024-09-13T14:44:39.828 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.828 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.828 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.828 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.974107+0000", 2024-09-13T14:44:39.828 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8200000000000002e-06 2024-09-13T14:44:39.828 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.828 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.829 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.829 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.978188+0000", 2024-09-13T14:44:39.829 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.004081337 2024-09-13T14:44:39.829 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.829 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.829 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.829 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.978290+0000", 2024-09-13T14:44:39.829 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010212700000000001 2024-09-13T14:44:39.829 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.829 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.829 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:44:39.829 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.978378+0000", 2024-09-13T14:44:39.829 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8160999999999998e-05 2024-09-13T14:44:39.830 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.830 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.830 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.830 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.989784+0000", 2024-09-13T14:44:39.830 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011405976999999999 2024-09-13T14:44:39.830 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.830 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.830 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.830 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.994006+0000", 2024-09-13T14:44:39.830 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0042215359999999997 2024-09-13T14:44:39.830 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.830 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.830 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.831 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:48.138031+0000", 2024-09-13T14:44:39.831 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.144024871 2024-09-13T14:44:39.831 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.831 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.831 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.831 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:48.138080+0000", 2024-09-13T14:44:39.831 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8964000000000001e-05 2024-09-13T14:44:39.831 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.831 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.831 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.831 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:48.138121+0000", 2024-09-13T14:44:39.831 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1829999999999998e-05 2024-09-13T14:44:39.831 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.832 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.832 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.832 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.832 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.832 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:155325 5.7 5:ee3ba1a9:::benchmark_data_smithi096_138841_object25129:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:44:39.832 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:41:47.974430+0000", 2024-09-13T14:44:39.832 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 171.82632515899999, 2024-09-13T14:44:39.832 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.163834804, 2024-09-13T14:44:39.832 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.832 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.832 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.832 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:44:39.832 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:44:39.832 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 155325 2024-09-13T14:44:39.832 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.833 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.833 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.833 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.833 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.974430+0000", 2024-09-13T14:44:39.833 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.833 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.833 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.833 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.833 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.974430+0000", 2024-09-13T14:44:39.833 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.833 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.833 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.833 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.834 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.974430+0000", 2024-09-13T14:44:39.834 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:39.834 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.834 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.834 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.834 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.974433+0000", 2024-09-13T14:44:39.834 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.444e-06 2024-09-13T14:44:39.834 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.834 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.834 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.834 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.974440+0000", 2024-09-13T14:44:39.834 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7950000000000002e-06 2024-09-13T14:44:39.835 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.835 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.835 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.835 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.974445+0000", 2024-09-13T14:44:39.835 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9069999999999998e-06 2024-09-13T14:44:39.835 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.835 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.835 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.835 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.982400+0000", 2024-09-13T14:44:39.835 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0079549419999999996 2024-09-13T14:44:39.835 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.835 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.835 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.836 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.982506+0000", 2024-09-13T14:44:39.836 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010554 2024-09-13T14:44:39.836 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.836 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.836 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:44:39.836 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.982628+0000", 2024-09-13T14:44:39.836 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012219999999999999 2024-09-13T14:44:39.836 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.836 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.836 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.836 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.990140+0000", 2024-09-13T14:44:39.836 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0075126020000000002 2024-09-13T14:44:39.836 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.837 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.837 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.837 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.994029+0000", 2024-09-13T14:44:39.837 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003888168 2024-09-13T14:44:39.837 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.837 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.837 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.837 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:48.138167+0000", 2024-09-13T14:44:39.837 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.14413800299999999 2024-09-13T14:44:39.837 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.837 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.837 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.837 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:48.138216+0000", 2024-09-13T14:44:39.838 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9042999999999998e-05 2024-09-13T14:44:39.838 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.838 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.838 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.838 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:48.138265+0000", 2024-09-13T14:44:39.838 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9715000000000001e-05 2024-09-13T14:44:39.838 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.838 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.838 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.838 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.838 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.839 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:155327 5.7 5:eb2cb127:::benchmark_data_smithi096_138841_object25131:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:44:39.839 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:41:47.977027+0000", 2024-09-13T14:44:39.839 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 171.82372842000001, 2024-09-13T14:44:39.839 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.174912766, 2024-09-13T14:44:39.839 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.839 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.839 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.839 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:44:39.839 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:44:39.839 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 155327 2024-09-13T14:44:39.839 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.839 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.839 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.839 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.840 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.977027+0000", 2024-09-13T14:44:39.840 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.840 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.840 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.840 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.840 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.977027+0000", 2024-09-13T14:44:39.840 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.840 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.840 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.840 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.840 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.977026+0000", 2024-09-13T14:44:39.840 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:39.840 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.841 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.841 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.841 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.977034+0000", 2024-09-13T14:44:39.841 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1939999999999998e-06 2024-09-13T14:44:39.841 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.841 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.841 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.841 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.977050+0000", 2024-09-13T14:44:39.841 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5789000000000001e-05 2024-09-13T14:44:39.841 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.841 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.841 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.841 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.977055+0000", 2024-09-13T14:44:39.841 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.682e-06 2024-09-13T14:44:39.842 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.842 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.842 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.842 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.986200+0000", 2024-09-13T14:44:39.842 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0091449070000000007 2024-09-13T14:44:39.842 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.842 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.842 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.842 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.986270+0000", 2024-09-13T14:44:39.842 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9203999999999997e-05 2024-09-13T14:44:39.842 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.842 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.842 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:44:39.843 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.986343+0000", 2024-09-13T14:44:39.843 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.3213999999999994e-05 2024-09-13T14:44:39.843 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.843 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.843 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.843 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.990171+0000", 2024-09-13T14:44:39.843 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0038285440000000001 2024-09-13T14:44:39.843 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.843 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.843 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.843 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:47.994052+0000", 2024-09-13T14:44:39.843 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003880737 2024-09-13T14:44:39.843 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.843 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.844 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.844 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:48.151844+0000", 2024-09-13T14:44:39.844 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15779196600000001 2024-09-13T14:44:39.844 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.844 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.844 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.844 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:48.151893+0000", 2024-09-13T14:44:39.844 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9311000000000003e-05 2024-09-13T14:44:39.844 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.844 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.844 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.844 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:41:48.151940+0000", 2024-09-13T14:44:39.844 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6675999999999998e-05 2024-09-13T14:44:39.845 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.845 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.845 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.845 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.845 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.845 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:34157 6.a 6:5e33e65f:::benchmark_data_smithi096_167087_object34156:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:44:39.845 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.714114+0000", 2024-09-13T14:44:39.845 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 17.086641968999999, 2024-09-13T14:44:39.845 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18609726400000001, 2024-09-13T14:44:39.845 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.845 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.845 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.845 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:44:39.845 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:44:39.846 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 34157 2024-09-13T14:44:39.846 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.846 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.846 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.846 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.846 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.714114+0000", 2024-09-13T14:44:39.846 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.846 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.846 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.846 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.846 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.714114+0000", 2024-09-13T14:44:39.846 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.846 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.846 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.847 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.847 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.714113+0000", 2024-09-13T14:44:39.847 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:39.847 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.847 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.847 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.847 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.714222+0000", 2024-09-13T14:44:39.847 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000109004 2024-09-13T14:44:39.847 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.847 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.847 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.847 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.714229+0000", 2024-09-13T14:44:39.847 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7780000000000002e-06 2024-09-13T14:44:39.848 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.848 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.848 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.848 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.714232+0000", 2024-09-13T14:44:39.848 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4910000000000001e-06 2024-09-13T14:44:39.848 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.848 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.848 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.848 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.896703+0000", 2024-09-13T14:44:39.848 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18247081400000001 2024-09-13T14:44:39.848 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.848 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.848 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.849 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.896851+0000", 2024-09-13T14:44:39.849 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014819200000000001 2024-09-13T14:44:39.849 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.849 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.849 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:44:39.849 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897007+0000", 2024-09-13T14:44:39.849 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015580499999999999 2024-09-13T14:44:39.849 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.849 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.849 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.849 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.900092+0000", 2024-09-13T14:44:39.849 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0030850069999999999 2024-09-13T14:44:39.849 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.849 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.850 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.850 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.900117+0000", 2024-09-13T14:44:39.850 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4715e-05 2024-09-13T14:44:39.850 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.850 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.850 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.850 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.900131+0000", 2024-09-13T14:44:39.850 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3706000000000001e-05 2024-09-13T14:44:39.850 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.850 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.850 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.850 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.900177+0000", 2024-09-13T14:44:39.850 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6368000000000001e-05 2024-09-13T14:44:39.850 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.851 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.851 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.851 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.900211+0000", 2024-09-13T14:44:39.851 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4048000000000003e-05 2024-09-13T14:44:39.851 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.851 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.851 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.851 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.851 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.851 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:34158 6.a 6:51e93c5b:::benchmark_data_smithi096_167087_object34157:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:44:39.851 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.715451+0000", 2024-09-13T14:44:39.851 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 17.085304470000001, 2024-09-13T14:44:39.851 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18665584800000001, 2024-09-13T14:44:39.852 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.852 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.852 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.852 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:44:39.852 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:44:39.852 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 34158 2024-09-13T14:44:39.852 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.852 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.852 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.852 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.852 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.715451+0000", 2024-09-13T14:44:39.852 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.852 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.853 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.853 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.853 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.715451+0000", 2024-09-13T14:44:39.853 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.853 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.853 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.853 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.853 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.715451+0000", 2024-09-13T14:44:39.853 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:39.853 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.853 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.853 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.853 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.715506+0000", 2024-09-13T14:44:39.853 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5887999999999998e-05 2024-09-13T14:44:39.854 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.854 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.854 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.854 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897119+0000", 2024-09-13T14:44:39.854 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18161281900000001 2024-09-13T14:44:39.854 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.854 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.854 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.854 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897129+0000", 2024-09-13T14:44:39.854 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.012e-05 2024-09-13T14:44:39.854 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.854 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.854 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.854 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.898401+0000", 2024-09-13T14:44:39.855 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0012716680000000001 2024-09-13T14:44:39.855 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.855 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.855 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.855 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.898507+0000", 2024-09-13T14:44:39.855 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000106284 2024-09-13T14:44:39.855 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.855 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.855 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:44:39.855 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.898632+0000", 2024-09-13T14:44:39.855 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000124382 2024-09-13T14:44:39.855 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.855 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.855 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.856 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.900219+0000", 2024-09-13T14:44:39.856 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001587254 2024-09-13T14:44:39.856 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.856 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.856 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.856 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.901838+0000", 2024-09-13T14:44:39.856 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0016194849999999999 2024-09-13T14:44:39.856 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.856 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.856 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.856 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.902033+0000", 2024-09-13T14:44:39.856 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00019475100000000001 2024-09-13T14:44:39.857 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.857 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.857 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.857 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.902075+0000", 2024-09-13T14:44:39.857 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1508999999999998e-05 2024-09-13T14:44:39.857 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.857 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.857 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.857 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.902107+0000", 2024-09-13T14:44:39.857 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2326000000000002e-05 2024-09-13T14:44:39.857 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.857 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.857 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.858 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.858 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.858 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:35698 6.7 6:e6b4f37e:::benchmark_data_smithi096_167087_object35697:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:44:39.858 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:23.872707+0000", 2024-09-13T14:44:39.858 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 15.928048950999999, 2024-09-13T14:44:39.858 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16889400600000001, 2024-09-13T14:44:39.858 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.858 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.858 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.858 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:44:39.858 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:44:39.858 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 35698 2024-09-13T14:44:39.858 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.858 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.859 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.859 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.859 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.872707+0000", 2024-09-13T14:44:39.859 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.859 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.859 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.859 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.859 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.872707+0000", 2024-09-13T14:44:39.859 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.859 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.859 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.859 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.859 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.872706+0000", 2024-09-13T14:44:39.860 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:44:39.860 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.860 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.860 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.860 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.872769+0000", 2024-09-13T14:44:39.860 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2912999999999999e-05 2024-09-13T14:44:39.860 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.860 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.860 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.860 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.872779+0000", 2024-09-13T14:44:39.860 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0275000000000001e-05 2024-09-13T14:44:39.860 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.860 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.861 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.861 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.872785+0000", 2024-09-13T14:44:39.861 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6459999999999998e-06 2024-09-13T14:44:39.861 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.861 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.861 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.861 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.872827+0000", 2024-09-13T14:44:39.861 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2653e-05 2024-09-13T14:44:39.861 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.861 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.861 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.861 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.872949+0000", 2024-09-13T14:44:39.861 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012174 2024-09-13T14:44:39.861 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.862 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.862 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:44:39.862 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.873078+0000", 2024-09-13T14:44:39.862 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000129498 2024-09-13T14:44:39.862 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.862 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.862 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.862 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.874422+0000", 2024-09-13T14:44:39.862 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0013438879999999999 2024-09-13T14:44:39.862 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.862 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:39.864 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.864 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.864 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.874549+0000", 2024-09-13T14:44:39.864 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012653999999999999 2024-09-13T14:44:39.864 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.864 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.865 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.865 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041533+0000", 2024-09-13T14:44:39.865 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16698364500000001 2024-09-13T14:44:39.865 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.865 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.865 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.865 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041570+0000", 2024-09-13T14:44:39.865 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7036999999999998e-05 2024-09-13T14:44:39.865 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.865 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.865 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.865 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041601+0000", 2024-09-13T14:44:39.865 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1161000000000002e-05 2024-09-13T14:44:39.866 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.866 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.866 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.866 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.866 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.866 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:35706 6.7 6:e8183a67:::benchmark_data_smithi096_167087_object35705:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:44:39.866 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:23.877374+0000", 2024-09-13T14:44:39.866 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 15.923382118999999, 2024-09-13T14:44:39.866 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.164322141, 2024-09-13T14:44:39.866 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.866 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.866 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.866 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:44:39.866 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:44:39.867 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 35706 2024-09-13T14:44:39.867 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.867 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.867 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.867 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.867 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877374+0000", 2024-09-13T14:44:39.867 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.867 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.867 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.867 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.867 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877374+0000", 2024-09-13T14:44:39.867 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.867 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.868 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.868 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.868 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877370+0000", 2024-09-13T14:44:39.868 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999967 2024-09-13T14:44:39.868 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.868 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.868 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.868 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877425+0000", 2024-09-13T14:44:39.868 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5092e-05 2024-09-13T14:44:39.868 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.868 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.868 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.869 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877433+0000", 2024-09-13T14:44:39.869 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3399999999999998e-06 2024-09-13T14:44:39.869 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.869 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.869 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.869 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877439+0000", 2024-09-13T14:44:39.869 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9209999999999997e-06 2024-09-13T14:44:39.869 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.869 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.869 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.869 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877468+0000", 2024-09-13T14:44:39.869 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9132000000000002e-05 2024-09-13T14:44:39.870 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.870 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.870 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.870 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877538+0000", 2024-09-13T14:44:39.870 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9627000000000003e-05 2024-09-13T14:44:39.870 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.870 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.870 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:44:39.870 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877623+0000", 2024-09-13T14:44:39.870 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4818e-05 2024-09-13T14:44:39.870 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.870 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.870 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.870 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879775+0000", 2024-09-13T14:44:39.871 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0021525429999999998 2024-09-13T14:44:39.871 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.871 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.871 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.871 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880043+0000", 2024-09-13T14:44:39.871 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00026706400000000002 2024-09-13T14:44:39.871 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.871 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.871 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.871 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041630+0000", 2024-09-13T14:44:39.871 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16158742500000001 2024-09-13T14:44:39.871 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.871 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.871 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.872 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041667+0000", 2024-09-13T14:44:39.872 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6996000000000003e-05 2024-09-13T14:44:39.872 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.872 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.872 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.872 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041696+0000", 2024-09-13T14:44:39.872 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8713000000000001e-05 2024-09-13T14:44:39.872 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.872 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.872 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.872 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.872 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.872 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:35707 6.d 6:b1318a2f:::benchmark_data_smithi096_167087_object35706:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:44:39.873 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:23.877446+0000", 2024-09-13T14:44:39.873 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 15.923309315999999, 2024-09-13T14:44:39.873 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16405186599999999, 2024-09-13T14:44:39.873 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.873 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.873 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.873 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:44:39.873 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:44:39.873 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 35707 2024-09-13T14:44:39.873 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.873 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.873 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.873 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.873 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877446+0000", 2024-09-13T14:44:39.874 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.874 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.874 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.874 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.874 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877446+0000", 2024-09-13T14:44:39.874 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.874 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.874 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.874 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.874 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877446+0000", 2024-09-13T14:44:39.874 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:39.874 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.874 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.875 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.875 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877492+0000", 2024-09-13T14:44:39.875 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6020000000000003e-05 2024-09-13T14:44:39.875 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.875 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.875 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.875 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877498+0000", 2024-09-13T14:44:39.875 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.677e-06 2024-09-13T14:44:39.875 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.875 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.875 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.875 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877501+0000", 2024-09-13T14:44:39.875 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2660000000000002e-06 2024-09-13T14:44:39.875 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.876 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.876 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.876 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877546+0000", 2024-09-13T14:44:39.876 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5278999999999998e-05 2024-09-13T14:44:39.876 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.876 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.876 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.876 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877626+0000", 2024-09-13T14:44:39.876 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0267999999999998e-05 2024-09-13T14:44:39.876 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.876 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.876 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:44:39.876 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877736+0000", 2024-09-13T14:44:39.876 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000109279 2024-09-13T14:44:39.877 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.877 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.877 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.877 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.878873+0000", 2024-09-13T14:44:39.877 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0011373500000000001 2024-09-13T14:44:39.877 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.877 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.877 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.877 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879733+0000", 2024-09-13T14:44:39.877 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00086003200000000005 2024-09-13T14:44:39.877 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.877 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.877 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.877 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041412+0000", 2024-09-13T14:44:39.878 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16167838800000001 2024-09-13T14:44:39.878 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.878 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.878 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.878 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041457+0000", 2024-09-13T14:44:39.878 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5933999999999998e-05 2024-09-13T14:44:39.878 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.878 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.878 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.878 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041498+0000", 2024-09-13T14:44:39.878 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0763999999999998e-05 2024-09-13T14:44:39.878 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.878 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.879 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.879 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.879 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.879 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:35709 6.7 6:e0e710df:::benchmark_data_smithi096_167087_object35708:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:44:39.879 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:23.877506+0000", 2024-09-13T14:44:39.879 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 15.923249449, 2024-09-13T14:44:39.879 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16429860900000001, 2024-09-13T14:44:39.879 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.879 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.879 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.879 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:44:39.879 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:44:39.879 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 35709 2024-09-13T14:44:39.879 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.880 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.880 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.880 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.880 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877506+0000", 2024-09-13T14:44:39.880 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.880 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.880 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.880 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.880 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877506+0000", 2024-09-13T14:44:39.880 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.880 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.880 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.880 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.881 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877506+0000", 2024-09-13T14:44:39.881 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:39.881 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.881 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.881 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.881 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877546+0000", 2024-09-13T14:44:39.881 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0237000000000001e-05 2024-09-13T14:44:39.881 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.881 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.881 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.881 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877551+0000", 2024-09-13T14:44:39.881 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.057e-06 2024-09-13T14:44:39.881 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.881 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.882 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.882 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.877554+0000", 2024-09-13T14:44:39.882 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8370000000000001e-06 2024-09-13T14:44:39.882 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.882 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.882 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.882 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.878542+0000", 2024-09-13T14:44:39.882 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00098817400000000008 2024-09-13T14:44:39.882 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.882 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.882 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.882 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.878608+0000", 2024-09-13T14:44:39.882 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5757000000000001e-05 2024-09-13T14:44:39.883 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.883 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.883 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:44:39.883 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.878719+0000", 2024-09-13T14:44:39.883 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000111229 2024-09-13T14:44:39.883 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.883 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.883 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.883 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879888+0000", 2024-09-13T14:44:39.883 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001168697 2024-09-13T14:44:39.883 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.883 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.883 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.884 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880071+0000", 2024-09-13T14:44:39.884 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018338800000000001 2024-09-13T14:44:39.884 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.884 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.884 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.884 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041735+0000", 2024-09-13T14:44:39.884 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16166398200000001 2024-09-13T14:44:39.884 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.884 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.884 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.884 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041774+0000", 2024-09-13T14:44:39.884 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8890000000000002e-05 2024-09-13T14:44:39.884 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.884 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.885 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.885 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041805+0000", 2024-09-13T14:44:39.885 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0676999999999998e-05 2024-09-13T14:44:39.885 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.885 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.885 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.885 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.885 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.885 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:35711 6.a 6:50fe20ca:::benchmark_data_smithi096_167087_object35710:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:44:39.885 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:23.880133+0000", 2024-09-13T14:44:39.885 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 15.920623099, 2024-09-13T14:44:39.885 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16518801999999999, 2024-09-13T14:44:39.886 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.886 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.886 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.886 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:44:39.886 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:44:39.886 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 35711 2024-09-13T14:44:39.886 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.886 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.886 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.886 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.886 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880133+0000", 2024-09-13T14:44:39.886 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.886 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.886 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.887 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.887 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880133+0000", 2024-09-13T14:44:39.887 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.887 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.887 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.887 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.887 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880132+0000", 2024-09-13T14:44:39.887 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:44:39.887 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.887 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.887 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.887 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880194+0000", 2024-09-13T14:44:39.887 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2444e-05 2024-09-13T14:44:39.887 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.888 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.888 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.888 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880204+0000", 2024-09-13T14:44:39.888 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0001e-05 2024-09-13T14:44:39.888 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.888 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.888 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.888 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880209+0000", 2024-09-13T14:44:39.888 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1800000000000004e-06 2024-09-13T14:44:39.888 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.888 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.888 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.888 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880249+0000", 2024-09-13T14:44:39.889 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9583999999999999e-05 2024-09-13T14:44:39.889 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.889 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.889 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.889 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880366+0000", 2024-09-13T14:44:39.889 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000117644 2024-09-13T14:44:39.889 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.889 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.889 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:44:39.889 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041168+0000", 2024-09-13T14:44:39.889 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16080183100000001 2024-09-13T14:44:39.889 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.889 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.889 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.890 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.044359+0000", 2024-09-13T14:44:39.890 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0031908219999999998 2024-09-13T14:44:39.890 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.890 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.890 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.890 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.044734+0000", 2024-09-13T14:44:39.890 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00037446799999999998 2024-09-13T14:44:39.890 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.890 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.890 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.890 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.045222+0000", 2024-09-13T14:44:39.890 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00048817799999999997 2024-09-13T14:44:39.890 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.890 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.891 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.891 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.045275+0000", 2024-09-13T14:44:39.891 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2852000000000002e-05 2024-09-13T14:44:39.891 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.891 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.891 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.891 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.045321+0000", 2024-09-13T14:44:39.891 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5972e-05 2024-09-13T14:44:39.891 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.891 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.891 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.891 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.891 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.891 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:35714 6.a 6:57dff3c1:::benchmark_data_smithi096_167087_object35713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:44:39.892 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:23.880309+0000", 2024-09-13T14:44:39.892 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 15.920446282, 2024-09-13T14:44:39.892 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.167047275, 2024-09-13T14:44:39.892 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.892 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.892 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.892 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:44:39.892 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:44:39.892 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 35714 2024-09-13T14:44:39.892 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.892 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.892 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.893 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.893 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880309+0000", 2024-09-13T14:44:39.893 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.893 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.893 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.893 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.893 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880309+0000", 2024-09-13T14:44:39.893 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.893 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.893 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.893 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.893 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880309+0000", 2024-09-13T14:44:39.893 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:39.893 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.894 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.894 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.894 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880366+0000", 2024-09-13T14:44:39.894 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.698e-05 2024-09-13T14:44:39.894 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.894 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.894 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.894 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880372+0000", 2024-09-13T14:44:39.894 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5819999999999998e-06 2024-09-13T14:44:39.894 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.894 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.894 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.894 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880376+0000", 2024-09-13T14:44:39.895 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2669999999999998e-06 2024-09-13T14:44:39.895 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.895 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.895 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.895 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.042614+0000", 2024-09-13T14:44:39.895 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.162238462 2024-09-13T14:44:39.895 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.895 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.895 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.895 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.042737+0000", 2024-09-13T14:44:39.895 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012278500000000001 2024-09-13T14:44:39.895 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.895 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.895 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:44:39.896 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.042915+0000", 2024-09-13T14:44:39.896 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00017834 2024-09-13T14:44:39.896 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.896 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.896 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.896 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.044497+0000", 2024-09-13T14:44:39.896 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0015813579999999999 2024-09-13T14:44:39.896 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.896 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.896 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.896 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.044789+0000", 2024-09-13T14:44:39.896 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00029261499999999998 2024-09-13T14:44:39.896 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.896 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.897 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.897 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.047271+0000", 2024-09-13T14:44:39.897 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0024817099999999998 2024-09-13T14:44:39.897 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.897 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.897 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.897 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.047323+0000", 2024-09-13T14:44:39.897 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2187e-05 2024-09-13T14:44:39.897 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.897 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.897 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.897 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.047357+0000", 2024-09-13T14:44:39.898 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3534000000000001e-05 2024-09-13T14:44:39.898 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.898 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.898 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.898 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.898 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.898 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:37449 6.a 6:5ba95c76:::benchmark_data_smithi096_167087_object37448:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:44:39.898 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:25.087743+0000", 2024-09-13T14:44:39.898 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 14.713013077999999, 2024-09-13T14:44:39.899 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18353551400000001, 2024-09-13T14:44:39.899 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.899 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.899 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.899 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:44:39.899 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:44:39.899 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 37449 2024-09-13T14:44:39.899 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.899 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.899 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.899 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.900 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.087743+0000", 2024-09-13T14:44:39.900 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.900 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.900 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.900 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.900 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.087743+0000", 2024-09-13T14:44:39.900 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.900 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.900 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.900 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.900 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.087742+0000", 2024-09-13T14:44:39.900 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:44:39.901 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.901 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.901 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.901 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.087821+0000", 2024-09-13T14:44:39.901 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8831000000000005e-05 2024-09-13T14:44:39.901 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.901 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.901 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.901 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.087833+0000", 2024-09-13T14:44:39.901 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1992e-05 2024-09-13T14:44:39.901 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.901 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.901 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.901 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.087837+0000", 2024-09-13T14:44:39.902 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3440000000000003e-06 2024-09-13T14:44:39.902 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.902 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.902 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.902 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.087988+0000", 2024-09-13T14:44:39.902 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000151108 2024-09-13T14:44:39.902 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.902 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.902 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.902 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.088060+0000", 2024-09-13T14:44:39.902 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2290000000000001e-05 2024-09-13T14:44:39.902 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.902 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.903 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:44:39.903 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.088136+0000", 2024-09-13T14:44:39.903 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5424000000000002e-05 2024-09-13T14:44:39.903 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.903 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.903 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.903 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.088987+0000", 2024-09-13T14:44:39.903 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00085111400000000004 2024-09-13T14:44:39.903 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.903 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.903 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.903 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269133+0000", 2024-09-13T14:44:39.903 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18014588000000001 2024-09-13T14:44:39.903 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.904 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.904 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.904 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271190+0000", 2024-09-13T14:44:39.904 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0020575609999999999 2024-09-13T14:44:39.904 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.904 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.904 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.904 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271238+0000", 2024-09-13T14:44:39.904 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7617000000000002e-05 2024-09-13T14:44:39.904 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.904 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.904 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.904 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271278+0000", 2024-09-13T14:44:39.904 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0182999999999998e-05 2024-09-13T14:44:39.905 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.905 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.905 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.905 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.905 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.905 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:37453 6.a 6:596a97d1:::benchmark_data_smithi096_167087_object37452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:44:39.905 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:25.090231+0000", 2024-09-13T14:44:39.905 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 14.710525088000001, 2024-09-13T14:44:39.905 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18115868900000001, 2024-09-13T14:44:39.905 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.905 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.905 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.905 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:44:39.906 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:44:39.906 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 37453 2024-09-13T14:44:39.906 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.906 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.906 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.906 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.906 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.090231+0000", 2024-09-13T14:44:39.906 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.906 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.906 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.906 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.906 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.090231+0000", 2024-09-13T14:44:39.906 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.906 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.907 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.907 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.907 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.090230+0000", 2024-09-13T14:44:39.907 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:39.907 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.907 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.907 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.907 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.090278+0000", 2024-09-13T14:44:39.907 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8267999999999999e-05 2024-09-13T14:44:39.907 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.907 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.907 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.908 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.090287+0000", 2024-09-13T14:44:39.908 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5730000000000008e-06 2024-09-13T14:44:39.908 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.908 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.908 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.908 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.090291+0000", 2024-09-13T14:44:39.908 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1359999999999999e-06 2024-09-13T14:44:39.908 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.908 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.908 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.908 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.090326+0000", 2024-09-13T14:44:39.908 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5429999999999998e-05 2024-09-13T14:44:39.908 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.908 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.909 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.909 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.090397+0000", 2024-09-13T14:44:39.909 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0234000000000006e-05 2024-09-13T14:44:39.909 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.909 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.909 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:44:39.909 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.090467+0000", 2024-09-13T14:44:39.909 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0777000000000004e-05 2024-09-13T14:44:39.909 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.909 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.909 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.909 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.092245+0000", 2024-09-13T14:44:39.910 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001778017 2024-09-13T14:44:39.910 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.910 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.910 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.910 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269157+0000", 2024-09-13T14:44:39.910 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.17691184300000001 2024-09-13T14:44:39.910 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.910 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.910 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.910 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271317+0000", 2024-09-13T14:44:39.910 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0021595220000000001 2024-09-13T14:44:39.910 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.910 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.910 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.911 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271354+0000", 2024-09-13T14:44:39.911 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7564999999999997e-05 2024-09-13T14:44:39.911 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.911 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.911 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.911 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271389+0000", 2024-09-13T14:44:39.911 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4996999999999997e-05 2024-09-13T14:44:39.911 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.911 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.911 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.911 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.911 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.911 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:37454 6.a 6:54f62d09:::benchmark_data_smithi096_167087_object37453:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:44:39.911 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:25.090298+0000", 2024-09-13T14:44:39.912 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 14.710457357999999, 2024-09-13T14:44:39.912 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18119716499999999, 2024-09-13T14:44:39.912 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.912 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.912 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.912 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:44:39.912 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:44:39.912 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 37454 2024-09-13T14:44:39.912 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.912 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.912 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.912 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.912 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.090298+0000", 2024-09-13T14:44:39.912 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.913 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.913 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.913 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.913 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.090298+0000", 2024-09-13T14:44:39.913 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.913 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.913 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.913 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.913 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.090298+0000", 2024-09-13T14:44:39.913 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:39.913 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.913 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.913 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.913 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.090342+0000", 2024-09-13T14:44:39.914 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4060000000000002e-05 2024-09-13T14:44:39.914 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.914 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.914 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.914 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.090347+0000", 2024-09-13T14:44:39.914 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2140000000000004e-06 2024-09-13T14:44:39.914 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.914 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.914 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.914 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.090350+0000", 2024-09-13T14:44:39.914 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7939999999999998e-06 2024-09-13T14:44:39.914 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.914 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.914 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.915 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.091090+0000", 2024-09-13T14:44:39.915 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000740214 2024-09-13T14:44:39.915 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.915 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.915 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.915 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.091157+0000", 2024-09-13T14:44:39.915 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6589999999999998e-05 2024-09-13T14:44:39.915 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.915 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.915 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:44:39.915 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.091258+0000", 2024-09-13T14:44:39.915 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000100715 2024-09-13T14:44:39.915 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.916 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.916 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.916 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.092383+0000", 2024-09-13T14:44:39.916 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001125217 2024-09-13T14:44:39.916 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.916 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.916 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.916 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269178+0000", 2024-09-13T14:44:39.916 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.17679544999999999 2024-09-13T14:44:39.916 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.916 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.916 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.917 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271430+0000", 2024-09-13T14:44:39.917 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0022515970000000001 2024-09-13T14:44:39.917 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.917 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.917 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.917 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271463+0000", 2024-09-13T14:44:39.917 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3030000000000001e-05 2024-09-13T14:44:39.917 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.917 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.917 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.917 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271495+0000", 2024-09-13T14:44:39.917 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2636999999999999e-05 2024-09-13T14:44:39.918 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.918 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.918 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.918 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.918 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.918 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:37458 6.d 6:bd7f47d9:::benchmark_data_smithi096_167087_object37457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:44:39.918 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:25.092444+0000", 2024-09-13T14:44:39.918 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 14.708311887000001, 2024-09-13T14:44:39.918 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.17686362899999999, 2024-09-13T14:44:39.918 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.918 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.918 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:44:39.918 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:44:39.919 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:44:39.919 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 37458 2024-09-13T14:44:39.919 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.919 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.919 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.919 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.919 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.092444+0000", 2024-09-13T14:44:39.919 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.919 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.919 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.919 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.919 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.092444+0000", 2024-09-13T14:44:39.919 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.919 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.920 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.920 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.920 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.092443+0000", 2024-09-13T14:44:39.920 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:44:39.920 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.920 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.920 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.920 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.092500+0000", 2024-09-13T14:44:39.920 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7126999999999997e-05 2024-09-13T14:44:39.920 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.920 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.920 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.920 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.092512+0000", 2024-09-13T14:44:39.920 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2337000000000001e-05 2024-09-13T14:44:39.921 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.921 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.921 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.921 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.092518+0000", 2024-09-13T14:44:39.921 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9959999999999999e-06 2024-09-13T14:44:39.921 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.921 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.921 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.921 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.092564+0000", 2024-09-13T14:44:39.921 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5797000000000002e-05 2024-09-13T14:44:39.921 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.921 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.921 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.922 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.092683+0000", 2024-09-13T14:44:39.922 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000118732 2024-09-13T14:44:39.922 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.922 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.922 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:44:39.922 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.092808+0000", 2024-09-13T14:44:39.922 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012560499999999999 2024-09-13T14:44:39.922 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.922 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.922 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:44:39.922 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.094123+0000", 2024-09-13T14:44:39.922 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0013142989999999999 2024-09-13T14:44:39.922 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.922 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.923 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.923 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.096021+0000", 2024-09-13T14:44:39.923 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0018984570000000001 2024-09-13T14:44:39.923 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.923 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.923 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:39.923 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269216+0000", 2024-09-13T14:44:39.923 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.17319517000000001 2024-09-13T14:44:39.923 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.923 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.923 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.923 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269264+0000", 2024-09-13T14:44:39.923 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8257999999999998e-05 2024-09-13T14:44:39.923 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.924 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.924 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.924 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269307+0000", 2024-09-13T14:44:39.924 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2901000000000001e-05 2024-09-13T14:44:39.924 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.924 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.924 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.924 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.924 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.924 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:37464 6.c e150/145 6:390d6347:::benchmark_data_smithi096_167087_object37463:head v 150'2306, mlcod=150'2304)", 2024-09-13T14:44:39.924 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:25.096815+0000", 2024-09-13T14:44:39.924 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 14.703940825, 2024-09-13T14:44:39.924 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.174186125, 2024-09-13T14:44:39.924 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.925 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:39.925 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.925 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.925 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.925 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.096815+0000", 2024-09-13T14:44:39.925 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.925 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.925 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.925 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.925 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.096815+0000", 2024-09-13T14:44:39.925 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.925 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.925 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.925 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.926 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.096814+0000", 2024-09-13T14:44:39.926 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:39.926 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.926 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.926 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.926 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.096921+0000", 2024-09-13T14:44:39.926 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010657700000000001 2024-09-13T14:44:39.926 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.926 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.926 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.926 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.096934+0000", 2024-09-13T14:44:39.926 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3388e-05 2024-09-13T14:44:39.926 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.926 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.927 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.927 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269317+0000", 2024-09-13T14:44:39.927 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.17238314800000001 2024-09-13T14:44:39.927 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.927 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.927 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.927 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269359+0000", 2024-09-13T14:44:39.927 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1341999999999998e-05 2024-09-13T14:44:39.927 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.927 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.927 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.927 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269372+0000", 2024-09-13T14:44:39.927 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3560999999999999e-05 2024-09-13T14:44:39.928 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.928 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.928 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:44:39.928 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.270967+0000", 2024-09-13T14:44:39.928 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001595097 2024-09-13T14:44:39.928 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.928 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.928 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.928 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271001+0000", 2024-09-13T14:44:39.928 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3531999999999997e-05 2024-09-13T14:44:39.928 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.928 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.928 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.929 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.929 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.929 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.45167.0:37449 6.a e150/145 ondisk, result = 0)", 2024-09-13T14:44:39.929 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:25.098132+0000", 2024-09-13T14:44:39.929 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 14.702623730999999, 2024-09-13T14:44:39.929 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.171003302, 2024-09-13T14:44:39.929 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:44:39.929 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:44:39.929 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:44:39.929 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.929 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:44:39.929 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.098132+0000", 2024-09-13T14:44:39.929 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.929 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.929 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.930 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:44:39.930 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.098132+0000", 2024-09-13T14:44:39.930 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:44:39.930 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.930 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.930 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:44:39.930 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.098131+0000", 2024-09-13T14:44:39.930 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:39.930 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.930 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.930 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:44:39.930 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.098135+0000", 2024-09-13T14:44:39.930 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7749999999999999e-06 2024-09-13T14:44:39.930 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.930 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.931 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:44:39.931 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.268956+0000", 2024-09-13T14:44:39.931 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.17082051300000001 2024-09-13T14:44:39.931 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.931 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.931 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:44:39.931 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.268972+0000", 2024-09-13T14:44:39.931 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5931e-05 2024-09-13T14:44:39.931 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.931 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.931 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:44:39.931 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269116+0000", 2024-09-13T14:44:39.931 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014477899999999999 2024-09-13T14:44:39.932 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.932 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.932 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:44:39.932 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269125+0000", 2024-09-13T14:44:39.932 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.6119999999999999e-06 2024-09-13T14:44:39.932 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:44:39.932 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:44:39.932 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:44:39.932 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269135+0000", 2024-09-13T14:44:39.932 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0246e-05 2024-09-13T14:44:39.932 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.932 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.932 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.932 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:44:39.933 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:44:39.933 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:44:39.933 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 31 16 59256 59240 119.422 85.6875 0.0102271 0.00836861 2024-09-13T14:44:39.961 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:39.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:40.063 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:40.107 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:40.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:40.209 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:40.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:39 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:40.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:39 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/567415978' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:40.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:39 smithi204 ceph-mon[87071]: osdmap e153: 8 total, 8 up, 7 in 2024-09-13T14:44:40.236 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:40.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:40.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:39 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:40.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:39 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/567415978' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:40.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:39 smithi096 ceph-mon[98901]: osdmap e153: 8 total, 8 up, 7 in 2024-09-13T14:44:40.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:39 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:40.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:39 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/567415978' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:40.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:39 smithi096 ceph-mon[102928]: osdmap e153: 8 total, 8 up, 7 in 2024-09-13T14:44:40.338 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:40.371 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:40.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:40.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:44:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:44:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:40.473 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:40.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.513 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:40.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:40.614 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:40.642 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:40.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:40.742 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:44:40.744 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:40.776 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:40.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:40.878 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:40.911 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:40.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:40.928 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 32 16 60641 60625 118.395 86.5625 0.00745278 0.00844121 2024-09-13T14:44:40.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:40 smithi204 ceph-mon[87071]: pgmap v436: 99 pgs: 99 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T14:44:41.012 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:41.050 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:41.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:41.092 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:41.151 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:41.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.193 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:41.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:41.294 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:41.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:40 smithi096 ceph-mon[98901]: pgmap v436: 99 pgs: 99 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T14:44:41.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:40 smithi096 ceph-mon[102928]: pgmap v436: 99 pgs: 99 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T14:44:41.329 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:41.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:41.431 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:41.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.467 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:41.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:41.569 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:41.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.607 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:41.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:41.708 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:41.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.752 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:41.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:41.853 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:41.882 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:41.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:41.928 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 33 16 61989 61973 117.36 84.25 0.00959555 0.00851552 2024-09-13T14:44:41.984 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:42.209 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:42.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:42.311 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:42.334 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:42.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:42.436 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:42.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.466 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:42.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:42.567 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:42.607 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:42.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:42.709 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:42.735 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:42.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:42.837 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:42.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.865 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:42.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:42.928 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 34 16 63334 63318 116.381 84.0625 0.0238741 0.00858734 2024-09-13T14:44:42.966 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:42.994 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:42.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:43.088 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:44:43.089 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:44:43.089 INFO:teuthology.orchestra.run.smithi204.stdout: "num_ops": 0 2024-09-13T14:44:43.089 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:44:43.089 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:42 smithi204 ceph-mon[87071]: pgmap v437: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-13T14:44:43.095 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:43.129 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:43.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:43.230 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:43.265 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:43.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:43.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:42 smithi096 ceph-mon[98901]: pgmap v437: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-13T14:44:43.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:42 smithi096 ceph-mon[102928]: pgmap v437: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-13T14:44:43.366 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:43.393 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:43.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:43.495 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:43.523 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:43.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:43.625 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:43.659 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:43.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:43.761 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:43.790 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:43.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:43.893 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:43.923 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:43.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:43.928 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 35 16 64584 64568 115.287 78.125 0.00722104 0.00866868 2024-09-13T14:44:44.025 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:44.052 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:44.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:44.154 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:44.183 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:44.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:44.224 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:44:44.285 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:44.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.337 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:44.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:44.438 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:44.469 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:44.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:44.571 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:44.611 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:44.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:44.613 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:44.713 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:44.740 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:44.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:44.842 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:44.882 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:44.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:44.928 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 36 16 65958 65942 114.47 85.875 0.00688358 0.00873091 2024-09-13T14:44:44.984 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:45.002 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:44 smithi204 ceph-mon[87071]: pgmap v438: 99 pgs: 99 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:44:45.026 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:45.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:44 smithi096 ceph-mon[98901]: pgmap v438: 99 pgs: 99 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:44:45.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:44 smithi096 ceph-mon[102928]: pgmap v438: 99 pgs: 99 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:44:45.129 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:45.153 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:45.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:45.255 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:45.298 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:45.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:45.390 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:45.400 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:45.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.466 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:45.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:45.567 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:45.592 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:45.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:45.694 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:45.724 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:44:45.728 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:44:45.732 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:45.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:45.834 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:45.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.881 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:45.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:45.929 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 37 16 67121 67105 113.341 72.6875 0.0210123 0.00881712 2024-09-13T14:44:45.982 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:46.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.013 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:46.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:46.114 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:46.152 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:46.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:46.254 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:46.282 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:46.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:46.384 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:46.390 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:44:46.390 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:44:46.390 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T14:44:46.390 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T14:44:46.390 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:44:46.428 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:46.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:46.530 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:46.560 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:46.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:46.662 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:46.702 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:46.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:46.804 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:46.826 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:44:46.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.843 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:46.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:46.929 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 38 16 68427 68411 112.506 81.625 0.0103538 0.00888262 2024-09-13T14:44:46.944 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:47.004 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:47.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:47.108 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:47.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.134 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:47.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:47.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:46 smithi204 ceph-mon[87071]: from='client.35340 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:47.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:46 smithi204 ceph-mon[87071]: pgmap v439: 99 pgs: 99 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-13T14:44:47.227 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:46 smithi096 ceph-mon[98901]: from='client.35340 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:47.227 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:46 smithi096 ceph-mon[98901]: pgmap v439: 99 pgs: 99 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-13T14:44:47.227 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:46 smithi096 ceph-mon[102928]: from='client.35340 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:47.228 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:46 smithi096 ceph-mon[102928]: pgmap v439: 99 pgs: 99 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-13T14:44:47.235 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:47.274 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:47.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:47.376 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:47.411 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:47.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:47.513 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:47.542 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:47.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:47.644 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:47.670 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":438,"stamp":"2024-09-13T14:44:43.734048+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4062320589,"num_objects":61941,"num_object_clones":0,"num_object_copies":185823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":61941,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":62820,"num_write_kb":3975055,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":4203971,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":64045,"ondisk_log_size":64045,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13394956,"kb_used_data":11680656,"kb_used_omap":34,"kb_used_meta":1714205,"kb_avail":642821108,"statfs":{"total":671965249536,"available":658248814592,"internally_reserved":0,"allocated":11960991744,"data_stored":11954517417,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35126,"internal_metadata":1755346634},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,9,9,6,4,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":935067648,"num_objects":14268,"num_object_clones":0,"num_object_copies":42804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14268,"num_write_kb":913152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004867"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":171,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.856615+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:44:38.856615+0000","last_peered":"2024-09-13T14:44:38.856615+0000","last_clean":"2024-09-13T14:44:38.856615+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:44:38.856615+0000","last_undegraded":"2024-09-13T14:44:38.856615+0000","last_fullsized":"2024-09-13T14:44:38.856615+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":119,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.455442+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:44:30.455442+0000","last_peered":"2024-09-13T14:44:30.455442+0000","last_clean":"2024-09-13T14:44:30.455442+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:44:30.455442+0000","last_undegraded":"2024-09-13T14:44:30.455442+0000","last_fullsized":"2024-09-13T14:44:30.455442+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":116,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.451081+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:44:30.451081+0000","last_peered":"2024-09-13T14:44:30.451081+0000","last_clean":"2024-09-13T14:44:30.451081+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:44:30.451081+0000","last_undegraded":"2024-09-13T14:44:30.451081+0000","last_fullsized":"2024-09-13T14:44:30.451081+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.860498+0000","last_change":"2024-09-13T14:38:13.977954+0000","last_active":"2024-09-13T14:44:38.860498+0000","last_peered":"2024-09-13T14:44:38.860498+0000","last_clean":"2024-09-13T14:44:38.860498+0000","last_became_active":"2024-09-13T14:38:13.977619+0000","last_became_peered":"2024-09-13T14:38:13.977619+0000","last_unstale":"2024-09-13T14:44:38.860498+0000","last_undegraded":"2024-09-13T14:44:38.860498+0000","last_fullsized":"2024-09-13T14:44:38.860498+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:58:30.259967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":165,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.860409+0000","last_change":"2024-09-13T14:38:13.978232+0000","last_active":"2024-09-13T14:44:38.860409+0000","last_peered":"2024-09-13T14:44:38.860409+0000","last_clean":"2024-09-13T14:44:38.860409+0000","last_became_active":"2024-09-13T14:38:13.977921+0000","last_became_peered":"2024-09-13T14:38:13.977921+0000","last_unstale":"2024-09-13T14:44:38.860409+0000","last_undegraded":"2024-09-13T14:44:38.860409+0000","last_fullsized":"2024-09-13T14:44:38.860409+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:10.127626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.455397+0000","last_change":"2024-09-13T14:39:42.765301+0000","last_active":"2024-09-13T14:44:30.455397+0000","last_peered":"2024-09-13T14:44:30.455397+0000","last_clean":"2024-09-13T14:44:30.455397+0000","last_became_active":"2024-09-13T14:39:42.764977+0000","last_became_peered":"2024-09-13T14:39:42.764977+0000","last_unstale":"2024-09-13T14:44:30.455397+0000","last_undegraded":"2024-09-13T14:44:30.455397+0000","last_fullsized":"2024-09-13T14:44:30.455397+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:25.223972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":151,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.860700+0000","last_change":"2024-09-13T14:37:40.925691+0000","last_active":"2024-09-13T14:44:38.860700+0000","last_peered":"2024-09-13T14:44:38.860700+0000","last_clean":"2024-09-13T14:44:38.860700+0000","last_became_active":"2024-09-13T14:37:40.925360+0000","last_became_peered":"2024-09-13T14:37:40.925360+0000","last_unstale":"2024-09-13T14:44:38.860700+0000","last_undegraded":"2024-09-13T14:44:38.860700+0000","last_fullsized":"2024-09-13T14:44:38.860700+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:07.337967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":126,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.861361+0000","last_change":"2024-09-13T14:39:42.781872+0000","last_active":"2024-09-13T14:44:38.861361+0000","last_peered":"2024-09-13T14:44:38.861361+0000","last_clean":"2024-09-13T14:44:38.861361+0000","last_became_active":"2024-09-13T14:39:42.770254+0000","last_became_peered":"2024-09-13T14:39:42.770254+0000","last_unstale":"2024-09-13T14:44:38.861361+0000","last_undegraded":"2024-09-13T14:44:38.861361+0000","last_fullsized":"2024-09-13T14:44:38.861361+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:31.211081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"153'3986","reported_seq":4000,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:41.814138+0000","last_change":"2024-09-13T14:43:58.298744+0000","last_active":"2024-09-13T14:44:41.814138+0000","last_peered":"2024-09-13T14:44:41.814138+0000","last_clean":"2024-09-13T14:44:41.814138+0000","last_became_active":"2024-09-13T14:43:58.298443+0000","last_became_peered":"2024-09-13T14:43:58.298443+0000","last_unstale":"2024-09-13T14:44:41.814138+0000","last_undegraded":"2024-09-13T14:44:41.814138+0000","last_fullsized":"2024-09-13T14:44:41.814138+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":3986,"log_dups_size":0,"ondisk_log_size":3986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:26:37.065070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261226496,"num_objects":3986,"num_object_clones":0,"num_object_copies":11958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3986,"num_write_kb":255104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":156,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.857470+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:44:38.857470+0000","last_peered":"2024-09-13T14:44:38.857470+0000","last_clean":"2024-09-13T14:44:38.857470+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:44:38.857470+0000","last_undegraded":"2024-09-13T14:44:38.857470+0000","last_fullsized":"2024-09-13T14:44:38.857470+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":109,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.448727+0000","last_change":"2024-09-13T14:39:42.764876+0000","last_active":"2024-09-13T14:44:30.448727+0000","last_peered":"2024-09-13T14:44:30.448727+0000","last_clean":"2024-09-13T14:44:30.448727+0000","last_became_active":"2024-09-13T14:39:42.764082+0000","last_became_peered":"2024-09-13T14:39:42.764082+0000","last_unstale":"2024-09-13T14:44:30.448727+0000","last_undegraded":"2024-09-13T14:44:30.448727+0000","last_fullsized":"2024-09-13T14:44:30.448727+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:10:07.458952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"153'3846","reported_seq":3873,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:41.844341+0000","last_change":"2024-09-13T14:43:58.301093+0000","last_active":"2024-09-13T14:44:41.844341+0000","last_peered":"2024-09-13T14:44:41.844341+0000","last_clean":"2024-09-13T14:44:41.844341+0000","last_became_active":"2024-09-13T14:43:58.300782+0000","last_became_peered":"2024-09-13T14:43:58.300782+0000","last_unstale":"2024-09-13T14:44:41.844341+0000","last_undegraded":"2024-09-13T14:44:41.844341+0000","last_fullsized":"2024-09-13T14:44:41.844341+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":3846,"log_dups_size":0,"ondisk_log_size":3846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:48:49.617848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252051456,"num_objects":3846,"num_object_clones":0,"num_object_copies":11538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3846,"num_write_kb":246144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":119,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.455231+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:44:30.455231+0000","last_peered":"2024-09-13T14:44:30.455231+0000","last_clean":"2024-09-13T14:44:30.455231+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:44:30.455231+0000","last_undegraded":"2024-09-13T14:44:30.455231+0000","last_fullsized":"2024-09-13T14:44:30.455231+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":144,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.447128+0000","last_change":"2024-09-13T14:38:13.979677+0000","last_active":"2024-09-13T14:44:30.447128+0000","last_peered":"2024-09-13T14:44:30.447128+0000","last_clean":"2024-09-13T14:44:30.447128+0000","last_became_active":"2024-09-13T14:38:13.979323+0000","last_became_peered":"2024-09-13T14:38:13.979323+0000","last_unstale":"2024-09-13T14:44:30.447128+0000","last_undegraded":"2024-09-13T14:44:30.447128+0000","last_fullsized":"2024-09-13T14:44:30.447128+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:29:03.491254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"153'3918","reported_seq":3934,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:41.690971+0000","last_change":"2024-09-13T14:43:58.298623+0000","last_active":"2024-09-13T14:44:41.690971+0000","last_peered":"2024-09-13T14:44:41.690971+0000","last_clean":"2024-09-13T14:44:41.690971+0000","last_became_active":"2024-09-13T14:43:58.298319+0000","last_became_peered":"2024-09-13T14:43:58.298319+0000","last_unstale":"2024-09-13T14:44:41.690971+0000","last_undegraded":"2024-09-13T14:44:41.690971+0000","last_fullsized":"2024-09-13T14:44:41.690971+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":3918,"log_dups_size":0,"ondisk_log_size":3918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:45.641123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256770048,"num_objects":3918,"num_object_clones":0,"num_object_copies":11754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3918,"num_write_kb":250752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":115,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.455278+0000","last_change":"2024-09-13T14:39:42.774547+0000","last_active":"2024-09-13T14:44:30.455278+0000","last_peered":"2024-09-13T14:44:30.455278+0000","last_clean":"2024-09-13T14:44:30.455278+0000","last_became_active":"2024-09-13T14:39:42.766006+0000","last_became_peered":"2024-09-13T14:39:42.766006+0000","last_unstale":"2024-09-13T14:44:30.455278+0000","last_undegraded":"2024-09-13T14:44:30.455278+0000","last_fullsized":"2024-09-13T14:44:30.455278+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:06:29.632163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":155,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.858072+0000","last_change":"2024-09-13T14:39:42.765763+0000","last_active":"2024-09-13T14:44:38.858072+0000","last_peered":"2024-09-13T14:44:38.858072+0000","last_clean":"2024-09-13T14:44:38.858072+0000","last_became_active":"2024-09-13T14:39:42.765441+0000","last_became_peered":"2024-09-13T14:39:42.765441+0000","last_unstale":"2024-09-13T14:44:38.858072+0000","last_undegraded":"2024-09-13T14:44:38.858072+0000","last_fullsized":"2024-09-13T14:44:38.858072+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:06.483638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"153'4094","reported_seq":4121,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:43.432428+0000","last_change":"2024-09-13T14:43:58.300633+0000","last_active":"2024-09-13T14:44:43.432428+0000","last_peered":"2024-09-13T14:44:43.432428+0000","last_clean":"2024-09-13T14:44:43.432428+0000","last_became_active":"2024-09-13T14:43:58.299977+0000","last_became_peered":"2024-09-13T14:43:58.299977+0000","last_unstale":"2024-09-13T14:44:43.432428+0000","last_undegraded":"2024-09-13T14:44:43.432428+0000","last_fullsized":"2024-09-13T14:44:43.432428+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":4094,"log_dups_size":0,"ondisk_log_size":4094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:12:32.234019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268304384,"num_objects":4094,"num_object_clones":0,"num_object_copies":12282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4094,"num_write_kb":262016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":127,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.449266+0000","last_change":"2024-09-13T14:39:42.765558+0000","last_active":"2024-09-13T14:44:30.449266+0000","last_peered":"2024-09-13T14:44:30.449266+0000","last_clean":"2024-09-13T14:44:30.449266+0000","last_became_active":"2024-09-13T14:39:42.765246+0000","last_became_peered":"2024-09-13T14:39:42.765246+0000","last_unstale":"2024-09-13T14:44:30.449266+0000","last_undegraded":"2024-09-13T14:44:30.449266+0000","last_fullsized":"2024-09-13T14:44:30.449266+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:08.640610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":165,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.860142+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:44:38.860142+0000","last_peered":"2024-09-13T14:44:38.860142+0000","last_clean":"2024-09-13T14:44:38.860142+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:44:38.860142+0000","last_undegraded":"2024-09-13T14:44:38.860142+0000","last_fullsized":"2024-09-13T14:44:38.860142+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"153'3721","reported_seq":3737,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:41.720562+0000","last_change":"2024-09-13T14:43:58.298549+0000","last_active":"2024-09-13T14:44:41.720562+0000","last_peered":"2024-09-13T14:44:41.720562+0000","last_clean":"2024-09-13T14:44:41.720562+0000","last_became_active":"2024-09-13T14:43:58.298281+0000","last_became_peered":"2024-09-13T14:43:58.298281+0000","last_unstale":"2024-09-13T14:44:41.720562+0000","last_undegraded":"2024-09-13T14:44:41.720562+0000","last_fullsized":"2024-09-13T14:44:41.720562+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":3721,"log_dups_size":0,"ondisk_log_size":3721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:28:44.626016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243859456,"num_objects":3721,"num_object_clones":0,"num_object_copies":11163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3721,"num_write_kb":238144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":280,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.454850+0000","last_change":"2024-09-13T14:40:32.590204+0000","last_active":"2024-09-13T14:44:30.454850+0000","last_peered":"2024-09-13T14:44:30.454850+0000","last_clean":"2024-09-13T14:44:30.454850+0000","last_became_active":"2024-09-13T14:40:22.949830+0000","last_became_peered":"2024-09-13T14:40:22.949830+0000","last_unstale":"2024-09-13T14:44:30.454850+0000","last_undegraded":"2024-09-13T14:44:30.454850+0000","last_fullsized":"2024-09-13T14:44:30.454850+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:32.472827+0000","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":168,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.855755+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:44:38.855755+0000","last_peered":"2024-09-13T14:44:38.855755+0000","last_clean":"2024-09-13T14:44:38.855755+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:44:38.855755+0000","last_undegraded":"2024-09-13T14:44:38.855755+0000","last_fullsized":"2024-09-13T14:44:38.855755+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":127,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.446905+0000","last_change":"2024-09-13T14:39:42.775584+0000","last_active":"2024-09-13T14:44:30.446905+0000","last_peered":"2024-09-13T14:44:30.446905+0000","last_clean":"2024-09-13T14:44:30.446905+0000","last_became_active":"2024-09-13T14:39:42.775254+0000","last_became_peered":"2024-09-13T14:39:42.775254+0000","last_unstale":"2024-09-13T14:44:30.446905+0000","last_undegraded":"2024-09-13T14:44:30.446905+0000","last_fullsized":"2024-09-13T14:44:30.446905+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:44.607098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"153'3673","reported_seq":3700,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:39.437573+0000","last_change":"2024-09-13T14:43:58.299969+0000","last_active":"2024-09-13T14:44:39.437573+0000","last_peered":"2024-09-13T14:44:39.437573+0000","last_clean":"2024-09-13T14:44:39.437573+0000","last_became_active":"2024-09-13T14:43:58.298592+0000","last_became_peered":"2024-09-13T14:43:58.298592+0000","last_unstale":"2024-09-13T14:44:39.437573+0000","last_undegraded":"2024-09-13T14:44:39.437573+0000","last_fullsized":"2024-09-13T14:44:39.437573+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":3673,"log_dups_size":0,"ondisk_log_size":3673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:09:09.791130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240713728,"num_objects":3673,"num_object_clones":0,"num_object_copies":11019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3673,"num_write_kb":235072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":60,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.448635+0000","last_change":"2024-09-13T14:40:27.868402+0000","last_active":"2024-09-13T14:44:30.448635+0000","last_peered":"2024-09-13T14:44:30.448635+0000","last_clean":"2024-09-13T14:44:30.448635+0000","last_became_active":"2024-09-13T14:40:22.966301+0000","last_became_peered":"2024-09-13T14:40:22.966301+0000","last_unstale":"2024-09-13T14:44:30.448635+0000","last_undegraded":"2024-09-13T14:44:30.448635+0000","last_fullsized":"2024-09-13T14:44:30.448635+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:18.660186+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":109,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.447478+0000","last_change":"2024-09-13T14:39:42.776751+0000","last_active":"2024-09-13T14:44:30.447478+0000","last_peered":"2024-09-13T14:44:30.447478+0000","last_clean":"2024-09-13T14:44:30.447478+0000","last_became_active":"2024-09-13T14:39:42.774855+0000","last_became_peered":"2024-09-13T14:39:42.774855+0000","last_unstale":"2024-09-13T14:44:30.447478+0000","last_undegraded":"2024-09-13T14:44:30.447478+0000","last_fullsized":"2024-09-13T14:44:30.447478+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:11.022013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":134,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.447358+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:44:30.447358+0000","last_peered":"2024-09-13T14:44:30.447358+0000","last_clean":"2024-09-13T14:44:30.447358+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:44:30.447358+0000","last_undegraded":"2024-09-13T14:44:30.447358+0000","last_fullsized":"2024-09-13T14:44:30.447358+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"153'3808","reported_seq":3834,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:41.853039+0000","last_change":"2024-09-13T14:43:58.298605+0000","last_active":"2024-09-13T14:44:41.853039+0000","last_peered":"2024-09-13T14:44:41.853039+0000","last_clean":"2024-09-13T14:44:41.853039+0000","last_became_active":"2024-09-13T14:43:58.298341+0000","last_became_peered":"2024-09-13T14:43:58.298341+0000","last_unstale":"2024-09-13T14:44:41.853039+0000","last_undegraded":"2024-09-13T14:44:41.853039+0000","last_fullsized":"2024-09-13T14:44:41.853039+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":3808,"log_dups_size":0,"ondisk_log_size":3808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:16.656493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249561088,"num_objects":3808,"num_object_clones":0,"num_object_copies":11424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3808,"num_write_kb":243712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":152,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.859895+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:44:38.859895+0000","last_peered":"2024-09-13T14:44:38.859895+0000","last_clean":"2024-09-13T14:44:38.859895+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:44:38.859895+0000","last_undegraded":"2024-09-13T14:44:38.859895+0000","last_fullsized":"2024-09-13T14:44:38.859895+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":127,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.447187+0000","last_change":"2024-09-13T14:39:42.776187+0000","last_active":"2024-09-13T14:44:30.447187+0000","last_peered":"2024-09-13T14:44:30.447187+0000","last_clean":"2024-09-13T14:44:30.447187+0000","last_became_active":"2024-09-13T14:39:42.775751+0000","last_became_peered":"2024-09-13T14:39:42.775751+0000","last_unstale":"2024-09-13T14:44:30.447187+0000","last_undegraded":"2024-09-13T14:44:30.447187+0000","last_fullsized":"2024-09-13T14:44:30.447187+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:35.155717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"153'3891","reported_seq":3906,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:41.819822+0000","last_change":"2024-09-13T14:43:58.298997+0000","last_active":"2024-09-13T14:44:41.819822+0000","last_peered":"2024-09-13T14:44:41.819822+0000","last_clean":"2024-09-13T14:44:41.819822+0000","last_became_active":"2024-09-13T14:43:58.298647+0000","last_became_peered":"2024-09-13T14:43:58.298647+0000","last_unstale":"2024-09-13T14:44:41.819822+0000","last_undegraded":"2024-09-13T14:44:41.819822+0000","last_fullsized":"2024-09-13T14:44:41.819822+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":3891,"log_dups_size":0,"ondisk_log_size":3891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:38:43.997227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255000576,"num_objects":3891,"num_object_clones":0,"num_object_copies":11673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3891,"num_write_kb":249024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":53,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.448859+0000","last_change":"2024-09-13T14:40:25.861254+0000","last_active":"2024-09-13T14:44:30.448859+0000","last_peered":"2024-09-13T14:44:30.448859+0000","last_clean":"2024-09-13T14:44:30.448859+0000","last_became_active":"2024-09-13T14:40:22.957614+0000","last_became_peered":"2024-09-13T14:40:22.957614+0000","last_unstale":"2024-09-13T14:44:30.448859+0000","last_undegraded":"2024-09-13T14:44:30.448859+0000","last_fullsized":"2024-09-13T14:44:30.448859+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:53.388190+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":173,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.863984+0000","last_change":"2024-09-13T14:38:13.981576+0000","last_active":"2024-09-13T14:44:38.863984+0000","last_peered":"2024-09-13T14:44:38.863984+0000","last_clean":"2024-09-13T14:44:38.863984+0000","last_became_active":"2024-09-13T14:38:13.981262+0000","last_became_peered":"2024-09-13T14:38:13.981262+0000","last_unstale":"2024-09-13T14:44:38.863984+0000","last_undegraded":"2024-09-13T14:44:38.863984+0000","last_fullsized":"2024-09-13T14:44:38.863984+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:20.284250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":153,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.858586+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:44:38.858586+0000","last_peered":"2024-09-13T14:44:38.858586+0000","last_clean":"2024-09-13T14:44:38.858586+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:44:38.858586+0000","last_undegraded":"2024-09-13T14:44:38.858586+0000","last_fullsized":"2024-09-13T14:44:38.858586+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"153'3739","reported_seq":3754,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:41.806042+0000","last_change":"2024-09-13T14:43:58.298370+0000","last_active":"2024-09-13T14:44:41.806042+0000","last_peered":"2024-09-13T14:44:41.806042+0000","last_clean":"2024-09-13T14:44:41.806042+0000","last_became_active":"2024-09-13T14:43:58.298041+0000","last_became_peered":"2024-09-13T14:43:58.298041+0000","last_unstale":"2024-09-13T14:44:41.806042+0000","last_undegraded":"2024-09-13T14:44:41.806042+0000","last_fullsized":"2024-09-13T14:44:41.806042+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":3739,"log_dups_size":0,"ondisk_log_size":3739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:28:59.876959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245039104,"num_objects":3739,"num_object_clones":0,"num_object_copies":11217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3739,"num_write_kb":239296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":271,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.455180+0000","last_change":"2024-09-13T14:40:22.639864+0000","last_active":"2024-09-13T14:44:30.455180+0000","last_peered":"2024-09-13T14:44:30.455180+0000","last_clean":"2024-09-13T14:44:30.455180+0000","last_became_active":"2024-09-13T14:40:20.926503+0000","last_became_peered":"2024-09-13T14:40:20.926503+0000","last_unstale":"2024-09-13T14:44:30.455180+0000","last_undegraded":"2024-09-13T14:44:30.455180+0000","last_fullsized":"2024-09-13T14:44:30.455180+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:22.639760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:22.639760+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:08.523563+0000","scrub_duration":0.0051120610000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":679,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.448999+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:44:30.448999+0000","last_peered":"2024-09-13T14:44:30.448999+0000","last_clean":"2024-09-13T14:44:30.448999+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:44:30.448999+0000","last_undegraded":"2024-09-13T14:44:30.448999+0000","last_fullsized":"2024-09-13T14:44:30.448999+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.855976+0000","last_change":"2024-09-13T14:39:42.768341+0000","last_active":"2024-09-13T14:44:38.855976+0000","last_peered":"2024-09-13T14:44:38.855976+0000","last_clean":"2024-09-13T14:44:38.855976+0000","last_became_active":"2024-09-13T14:39:42.768034+0000","last_became_peered":"2024-09-13T14:39:42.768034+0000","last_unstale":"2024-09-13T14:44:38.855976+0000","last_undegraded":"2024-09-13T14:44:38.855976+0000","last_fullsized":"2024-09-13T14:44:38.855976+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:28:55.502108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":59,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.447237+0000","last_change":"2024-09-13T14:39:42.772096+0000","last_active":"2024-09-13T14:44:30.447237+0000","last_peered":"2024-09-13T14:44:30.447237+0000","last_clean":"2024-09-13T14:44:30.447237+0000","last_became_active":"2024-09-13T14:39:42.771762+0000","last_became_peered":"2024-09-13T14:39:42.771762+0000","last_unstale":"2024-09-13T14:44:30.447237+0000","last_undegraded":"2024-09-13T14:44:30.447237+0000","last_fullsized":"2024-09-13T14:44:30.447237+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:16.172571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"153'3939","reported_seq":3966,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:42.182006+0000","last_change":"2024-09-13T14:43:58.298209+0000","last_active":"2024-09-13T14:44:42.182006+0000","last_peered":"2024-09-13T14:44:42.182006+0000","last_clean":"2024-09-13T14:44:42.182006+0000","last_became_active":"2024-09-13T14:43:58.297919+0000","last_became_peered":"2024-09-13T14:43:58.297919+0000","last_unstale":"2024-09-13T14:44:42.182006+0000","last_undegraded":"2024-09-13T14:44:42.182006+0000","last_fullsized":"2024-09-13T14:44:42.182006+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":3939,"log_dups_size":0,"ondisk_log_size":3939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:14.599016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258146304,"num_objects":3939,"num_object_clones":0,"num_object_copies":11817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3939,"num_write_kb":252096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":284,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.858388+0000","last_change":"2024-09-13T14:40:23.784403+0000","last_active":"2024-09-13T14:44:38.858388+0000","last_peered":"2024-09-13T14:44:38.858388+0000","last_clean":"2024-09-13T14:44:38.858388+0000","last_became_active":"2024-09-13T14:40:22.951246+0000","last_became_peered":"2024-09-13T14:40:22.951246+0000","last_unstale":"2024-09-13T14:44:38.858388+0000","last_undegraded":"2024-09-13T14:44:38.858388+0000","last_fullsized":"2024-09-13T14:44:38.858388+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.784263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.784263+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:46.907407+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":364,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.454735+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:44:30.454735+0000","last_peered":"2024-09-13T14:44:30.454735+0000","last_clean":"2024-09-13T14:44:30.454735+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:44:30.454735+0000","last_undegraded":"2024-09-13T14:44:30.454735+0000","last_fullsized":"2024-09-13T14:44:30.454735+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":173,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.860084+0000","last_change":"2024-09-13T14:38:13.977962+0000","last_active":"2024-09-13T14:44:38.860084+0000","last_peered":"2024-09-13T14:44:38.860084+0000","last_clean":"2024-09-13T14:44:38.860084+0000","last_became_active":"2024-09-13T14:38:13.977638+0000","last_became_peered":"2024-09-13T14:38:13.977638+0000","last_unstale":"2024-09-13T14:44:38.860084+0000","last_undegraded":"2024-09-13T14:44:38.860084+0000","last_fullsized":"2024-09-13T14:44:38.860084+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:53:16.902157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":177,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.860563+0000","last_change":"2024-09-13T14:37:40.927444+0000","last_active":"2024-09-13T14:44:38.860563+0000","last_peered":"2024-09-13T14:44:38.860563+0000","last_clean":"2024-09-13T14:44:38.860563+0000","last_became_active":"2024-09-13T14:37:40.927140+0000","last_became_peered":"2024-09-13T14:37:40.927140+0000","last_unstale":"2024-09-13T14:44:38.860563+0000","last_undegraded":"2024-09-13T14:44:38.860563+0000","last_fullsized":"2024-09-13T14:44:38.860563+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:32:54.207568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"153'3850","reported_seq":3866,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:41.691292+0000","last_change":"2024-09-13T14:43:58.299024+0000","last_active":"2024-09-13T14:44:41.691292+0000","last_peered":"2024-09-13T14:44:41.691292+0000","last_clean":"2024-09-13T14:44:41.691292+0000","last_became_active":"2024-09-13T14:43:58.298323+0000","last_became_peered":"2024-09-13T14:43:58.298323+0000","last_unstale":"2024-09-13T14:44:41.691292+0000","last_undegraded":"2024-09-13T14:44:41.691292+0000","last_fullsized":"2024-09-13T14:44:41.691292+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":3850,"log_dups_size":0,"ondisk_log_size":3850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:32.680552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252313600,"num_objects":3850,"num_object_clones":0,"num_object_copies":11550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3850,"num_write_kb":246400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":280,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.454670+0000","last_change":"2024-09-13T14:40:28.602999+0000","last_active":"2024-09-13T14:44:30.454670+0000","last_peered":"2024-09-13T14:44:30.454670+0000","last_clean":"2024-09-13T14:44:30.454670+0000","last_became_active":"2024-09-13T14:40:22.949002+0000","last_became_peered":"2024-09-13T14:40:22.949002+0000","last_unstale":"2024-09-13T14:44:30.454670+0000","last_undegraded":"2024-09-13T14:44:30.454670+0000","last_fullsized":"2024-09-13T14:44:30.454670+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:28.602942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:28.602942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:29:46.037596+0000","scrub_duration":0.00945176,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":577,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.858147+0000","last_change":"2024-09-13T14:38:13.978679+0000","last_active":"2024-09-13T14:44:38.858147+0000","last_peered":"2024-09-13T14:44:38.858147+0000","last_clean":"2024-09-13T14:44:38.858147+0000","last_became_active":"2024-09-13T14:38:13.978171+0000","last_became_peered":"2024-09-13T14:38:13.978171+0000","last_unstale":"2024-09-13T14:44:38.858147+0000","last_undegraded":"2024-09-13T14:44:38.858147+0000","last_fullsized":"2024-09-13T14:44:38.858147+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:57.691198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":160,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.857952+0000","last_change":"2024-09-13T14:39:42.769691+0000","last_active":"2024-09-13T14:44:38.857952+0000","last_peered":"2024-09-13T14:44:38.857952+0000","last_clean":"2024-09-13T14:44:38.857952+0000","last_became_active":"2024-09-13T14:39:42.769266+0000","last_became_peered":"2024-09-13T14:39:42.769266+0000","last_unstale":"2024-09-13T14:44:38.857952+0000","last_undegraded":"2024-09-13T14:44:38.857952+0000","last_fullsized":"2024-09-13T14:44:38.857952+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:11.637732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":186,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.858438+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:44:38.858438+0000","last_peered":"2024-09-13T14:44:38.858438+0000","last_clean":"2024-09-13T14:44:38.858438+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:44:38.858438+0000","last_undegraded":"2024-09-13T14:44:38.858438+0000","last_fullsized":"2024-09-13T14:44:38.858438+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"153'3866","reported_seq":3893,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:41.862287+0000","last_change":"2024-09-13T14:43:58.301790+0000","last_active":"2024-09-13T14:44:41.862287+0000","last_peered":"2024-09-13T14:44:41.862287+0000","last_clean":"2024-09-13T14:44:41.862287+0000","last_became_active":"2024-09-13T14:43:58.301504+0000","last_became_peered":"2024-09-13T14:43:58.301504+0000","last_unstale":"2024-09-13T14:44:41.862287+0000","last_undegraded":"2024-09-13T14:44:41.862287+0000","last_fullsized":"2024-09-13T14:44:41.862287+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":3866,"log_dups_size":0,"ondisk_log_size":3866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:05:44.303022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253362176,"num_objects":3866,"num_object_clones":0,"num_object_copies":11598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3866,"num_write_kb":247424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":53,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.448671+0000","last_change":"2024-09-13T14:40:26.896691+0000","last_active":"2024-09-13T14:44:30.448671+0000","last_peered":"2024-09-13T14:44:30.448671+0000","last_clean":"2024-09-13T14:44:30.448671+0000","last_became_active":"2024-09-13T14:40:22.949772+0000","last_became_peered":"2024-09-13T14:40:22.949772+0000","last_unstale":"2024-09-13T14:44:30.448671+0000","last_undegraded":"2024-09-13T14:44:30.448671+0000","last_fullsized":"2024-09-13T14:44:30.448671+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:26.896613+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:26.896613+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:01.615433+0000","scrub_duration":0.004120828,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":743,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.448987+0000","last_change":"2024-09-13T14:39:42.805055+0000","last_active":"2024-09-13T14:44:30.448987+0000","last_peered":"2024-09-13T14:44:30.448987+0000","last_clean":"2024-09-13T14:44:30.448987+0000","last_became_active":"2024-09-13T14:39:42.762456+0000","last_became_peered":"2024-09-13T14:39:42.762456+0000","last_unstale":"2024-09-13T14:44:30.448987+0000","last_undegraded":"2024-09-13T14:44:30.448987+0000","last_fullsized":"2024-09-13T14:44:30.448987+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:54:40.839991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":128,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.447336+0000","last_change":"2024-09-13T14:39:42.775159+0000","last_active":"2024-09-13T14:44:30.447336+0000","last_peered":"2024-09-13T14:44:30.447336+0000","last_clean":"2024-09-13T14:44:30.447336+0000","last_became_active":"2024-09-13T14:39:42.774841+0000","last_became_peered":"2024-09-13T14:39:42.774841+0000","last_unstale":"2024-09-13T14:44:30.447336+0000","last_undegraded":"2024-09-13T14:44:30.447336+0000","last_fullsized":"2024-09-13T14:44:30.447336+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:35:45.579397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.858713+0000","last_change":"2024-09-13T14:38:13.981294+0000","last_active":"2024-09-13T14:44:38.858713+0000","last_peered":"2024-09-13T14:44:38.858713+0000","last_clean":"2024-09-13T14:44:38.858713+0000","last_became_active":"2024-09-13T14:38:13.980975+0000","last_became_peered":"2024-09-13T14:38:13.980975+0000","last_unstale":"2024-09-13T14:44:38.858713+0000","last_undegraded":"2024-09-13T14:44:38.858713+0000","last_fullsized":"2024-09-13T14:44:38.858713+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:17.545179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"153'3963","reported_seq":3990,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:43.465055+0000","last_change":"2024-09-13T14:43:58.301421+0000","last_active":"2024-09-13T14:44:43.465055+0000","last_peered":"2024-09-13T14:44:43.465055+0000","last_clean":"2024-09-13T14:44:43.465055+0000","last_became_active":"2024-09-13T14:43:58.301132+0000","last_became_peered":"2024-09-13T14:43:58.301132+0000","last_unstale":"2024-09-13T14:44:43.465055+0000","last_undegraded":"2024-09-13T14:44:43.465055+0000","last_fullsized":"2024-09-13T14:44:43.465055+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":3963,"log_dups_size":0,"ondisk_log_size":3963,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:58.812457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259719168,"num_objects":3963,"num_object_clones":0,"num_object_copies":11889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3963,"num_write_kb":253632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":92,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.857910+0000","last_change":"2024-09-13T14:40:23.093425+0000","last_active":"2024-09-13T14:44:38.857910+0000","last_peered":"2024-09-13T14:44:38.857910+0000","last_clean":"2024-09-13T14:44:38.857910+0000","last_became_active":"2024-09-13T14:40:22.947358+0000","last_became_peered":"2024-09-13T14:40:22.947358+0000","last_unstale":"2024-09-13T14:44:38.857910+0000","last_undegraded":"2024-09-13T14:44:38.857910+0000","last_fullsized":"2024-09-13T14:44:38.857910+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.093356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.093356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:21:39.048340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":134,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.451162+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:44:30.451162+0000","last_peered":"2024-09-13T14:44:30.451162+0000","last_clean":"2024-09-13T14:44:30.451162+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:44:30.451162+0000","last_undegraded":"2024-09-13T14:44:30.451162+0000","last_fullsized":"2024-09-13T14:44:30.451162+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":670,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.858866+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:44:38.858866+0000","last_peered":"2024-09-13T14:44:38.858866+0000","last_clean":"2024-09-13T14:44:38.858866+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:44:38.858866+0000","last_undegraded":"2024-09-13T14:44:38.858866+0000","last_fullsized":"2024-09-13T14:44:38.858866+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":142,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.858367+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:44:38.858367+0000","last_peered":"2024-09-13T14:44:38.858367+0000","last_clean":"2024-09-13T14:44:38.858367+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:44:38.858367+0000","last_undegraded":"2024-09-13T14:44:38.858367+0000","last_fullsized":"2024-09-13T14:44:38.858367+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"153'3926","reported_seq":3953,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:42.191387+0000","last_change":"2024-09-13T14:43:58.299661+0000","last_active":"2024-09-13T14:44:42.191387+0000","last_peered":"2024-09-13T14:44:42.191387+0000","last_clean":"2024-09-13T14:44:42.191387+0000","last_became_active":"2024-09-13T14:43:58.299408+0000","last_became_peered":"2024-09-13T14:43:58.299408+0000","last_unstale":"2024-09-13T14:44:42.191387+0000","last_undegraded":"2024-09-13T14:44:42.191387+0000","last_fullsized":"2024-09-13T14:44:42.191387+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":3926,"log_dups_size":0,"ondisk_log_size":3926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:21:13.142607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257294336,"num_objects":3926,"num_object_clones":0,"num_object_copies":11778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3926,"num_write_kb":251264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":239,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.860278+0000","last_change":"2024-09-13T14:40:23.334441+0000","last_active":"2024-09-13T14:44:38.860278+0000","last_peered":"2024-09-13T14:44:38.860278+0000","last_clean":"2024-09-13T14:44:38.860278+0000","last_became_active":"2024-09-13T14:40:22.952908+0000","last_became_peered":"2024-09-13T14:40:22.952908+0000","last_unstale":"2024-09-13T14:44:38.860278+0000","last_undegraded":"2024-09-13T14:44:38.860278+0000","last_fullsized":"2024-09-13T14:44:38.860278+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.334379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.334379+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:28:12.746426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":153,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.858797+0000","last_change":"2024-09-13T14:37:40.921861+0000","last_active":"2024-09-13T14:44:38.858797+0000","last_peered":"2024-09-13T14:44:38.858797+0000","last_clean":"2024-09-13T14:44:38.858797+0000","last_became_active":"2024-09-13T14:37:40.921587+0000","last_became_peered":"2024-09-13T14:37:40.921587+0000","last_unstale":"2024-09-13T14:44:38.858797+0000","last_undegraded":"2024-09-13T14:44:38.858797+0000","last_fullsized":"2024-09-13T14:44:38.858797+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:24.619352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":366,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.455393+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:44:30.455393+0000","last_peered":"2024-09-13T14:44:30.455393+0000","last_clean":"2024-09-13T14:44:30.455393+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:44:30.455393+0000","last_undegraded":"2024-09-13T14:44:30.455393+0000","last_fullsized":"2024-09-13T14:44:30.455393+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":155,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.860069+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:44:38.860069+0000","last_peered":"2024-09-13T14:44:38.860069+0000","last_clean":"2024-09-13T14:44:38.860069+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:44:38.860069+0000","last_undegraded":"2024-09-13T14:44:38.860069+0000","last_fullsized":"2024-09-13T14:44:38.860069+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"153'3824","reported_seq":3840,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:41.713497+0000","last_change":"2024-09-13T14:43:58.300122+0000","last_active":"2024-09-13T14:44:41.713497+0000","last_peered":"2024-09-13T14:44:41.713497+0000","last_clean":"2024-09-13T14:44:41.713497+0000","last_became_active":"2024-09-13T14:43:58.299830+0000","last_became_peered":"2024-09-13T14:43:58.299830+0000","last_unstale":"2024-09-13T14:44:41.713497+0000","last_undegraded":"2024-09-13T14:44:41.713497+0000","last_fullsized":"2024-09-13T14:44:41.713497+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":3824,"log_dups_size":0,"ondisk_log_size":3824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:18.285498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250609664,"num_objects":3824,"num_object_clones":0,"num_object_copies":11472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3824,"num_write_kb":244736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":54,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.447055+0000","last_change":"2024-09-13T14:40:30.500108+0000","last_active":"2024-09-13T14:44:30.447055+0000","last_peered":"2024-09-13T14:44:30.447055+0000","last_clean":"2024-09-13T14:44:30.447055+0000","last_became_active":"2024-09-13T14:40:22.949882+0000","last_became_peered":"2024-09-13T14:40:22.949882+0000","last_unstale":"2024-09-13T14:44:30.447055+0000","last_undegraded":"2024-09-13T14:44:30.447055+0000","last_fullsized":"2024-09-13T14:44:30.447055+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:40:30.500046+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:22:58.480202+0000","scrub_duration":0.00248832,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":169,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.858123+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:44:38.858123+0000","last_peered":"2024-09-13T14:44:38.858123+0000","last_clean":"2024-09-13T14:44:38.858123+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:44:38.858123+0000","last_undegraded":"2024-09-13T14:44:38.858123+0000","last_fullsized":"2024-09-13T14:44:38.858123+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":491,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.858168+0000","last_change":"2024-09-13T14:39:44.990216+0000","last_active":"2024-09-13T14:44:38.858168+0000","last_peered":"2024-09-13T14:44:38.858168+0000","last_clean":"2024-09-13T14:44:38.858168+0000","last_became_active":"2024-09-13T14:39:44.989433+0000","last_became_peered":"2024-09-13T14:39:44.989433+0000","last_unstale":"2024-09-13T14:44:38.858168+0000","last_undegraded":"2024-09-13T14:44:38.858168+0000","last_fullsized":"2024-09-13T14:44:38.858168+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:00.426042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":166,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.864178+0000","last_change":"2024-09-13T14:38:13.979255+0000","last_active":"2024-09-13T14:44:38.864178+0000","last_peered":"2024-09-13T14:44:38.864178+0000","last_clean":"2024-09-13T14:44:38.864178+0000","last_became_active":"2024-09-13T14:38:13.978868+0000","last_became_peered":"2024-09-13T14:38:13.978868+0000","last_unstale":"2024-09-13T14:44:38.864178+0000","last_undegraded":"2024-09-13T14:44:38.864178+0000","last_fullsized":"2024-09-13T14:44:38.864178+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:15:26.003538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"153'3869","reported_seq":3884,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:41.703926+0000","last_change":"2024-09-13T14:43:58.299991+0000","last_active":"2024-09-13T14:44:41.703926+0000","last_peered":"2024-09-13T14:44:41.703926+0000","last_clean":"2024-09-13T14:44:41.703926+0000","last_became_active":"2024-09-13T14:43:58.299386+0000","last_became_peered":"2024-09-13T14:43:58.299386+0000","last_unstale":"2024-09-13T14:44:41.703926+0000","last_undegraded":"2024-09-13T14:44:41.703926+0000","last_fullsized":"2024-09-13T14:44:41.703926+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":3869,"log_dups_size":0,"ondisk_log_size":3869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:24:28.041082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253558784,"num_objects":3869,"num_object_clones":0,"num_object_copies":11607,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3869,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3869,"num_write_kb":247616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":98,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.856373+0000","last_change":"2024-09-13T14:40:24.124750+0000","last_active":"2024-09-13T14:44:38.856373+0000","last_peered":"2024-09-13T14:44:38.856373+0000","last_clean":"2024-09-13T14:44:38.856373+0000","last_became_active":"2024-09-13T14:40:22.958820+0000","last_became_peered":"2024-09-13T14:40:22.958820+0000","last_unstale":"2024-09-13T14:44:38.856373+0000","last_undegraded":"2024-09-13T14:44:38.856373+0000","last_fullsized":"2024-09-13T14:44:38.856373+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:24.124531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:24.124531+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:16:56.750282+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":177,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.860322+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:44:38.860322+0000","last_peered":"2024-09-13T14:44:38.860322+0000","last_clean":"2024-09-13T14:44:38.860322+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:44:38.860322+0000","last_undegraded":"2024-09-13T14:44:38.860322+0000","last_fullsized":"2024-09-13T14:44:38.860322+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1070,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.455129+0000","last_change":"2024-09-13T14:39:44.990389+0000","last_active":"2024-09-13T14:44:30.455129+0000","last_peered":"2024-09-13T14:44:30.455129+0000","last_clean":"2024-09-13T14:44:30.455129+0000","last_became_active":"2024-09-13T14:39:44.989805+0000","last_became_peered":"2024-09-13T14:39:44.989805+0000","last_unstale":"2024-09-13T14:44:30.455129+0000","last_undegraded":"2024-09-13T14:44:30.455129+0000","last_fullsized":"2024-09-13T14:44:30.455129+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:15.392445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.446729+0000","last_change":"2024-09-13T14:39:42.769954+0000","last_active":"2024-09-13T14:44:30.446729+0000","last_peered":"2024-09-13T14:44:30.446729+0000","last_clean":"2024-09-13T14:44:30.446729+0000","last_became_active":"2024-09-13T14:39:42.769635+0000","last_became_peered":"2024-09-13T14:39:42.769635+0000","last_unstale":"2024-09-13T14:44:30.446729+0000","last_undegraded":"2024-09-13T14:44:30.446729+0000","last_fullsized":"2024-09-13T14:44:30.446729+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:51.588071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":127,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.447268+0000","last_change":"2024-09-13T14:39:42.775357+0000","last_active":"2024-09-13T14:44:30.447268+0000","last_peered":"2024-09-13T14:44:30.447268+0000","last_clean":"2024-09-13T14:44:30.447268+0000","last_became_active":"2024-09-13T14:39:42.775041+0000","last_became_peered":"2024-09-13T14:39:42.775041+0000","last_unstale":"2024-09-13T14:44:30.447268+0000","last_undegraded":"2024-09-13T14:44:30.447268+0000","last_fullsized":"2024-09-13T14:44:30.447268+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:44.768910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":167,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.859874+0000","last_change":"2024-09-13T14:39:42.787151+0000","last_active":"2024-09-13T14:44:38.859874+0000","last_peered":"2024-09-13T14:44:38.859874+0000","last_clean":"2024-09-13T14:44:38.859874+0000","last_became_active":"2024-09-13T14:39:42.771712+0000","last_became_peered":"2024-09-13T14:39:42.771712+0000","last_unstale":"2024-09-13T14:44:38.859874+0000","last_undegraded":"2024-09-13T14:44:38.859874+0000","last_fullsized":"2024-09-13T14:44:38.859874+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:54.216663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":160,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.858224+0000","last_change":"2024-09-13T14:39:42.772974+0000","last_active":"2024-09-13T14:44:38.858224+0000","last_peered":"2024-09-13T14:44:38.858224+0000","last_clean":"2024-09-13T14:44:38.858224+0000","last_became_active":"2024-09-13T14:39:42.771449+0000","last_became_peered":"2024-09-13T14:39:42.771449+0000","last_unstale":"2024-09-13T14:44:38.858224+0000","last_undegraded":"2024-09-13T14:44:38.858224+0000","last_fullsized":"2024-09-13T14:44:38.858224+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:46.441079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":133,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.455135+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:44:30.455135+0000","last_peered":"2024-09-13T14:44:30.455135+0000","last_clean":"2024-09-13T14:44:30.455135+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:44:30.455135+0000","last_undegraded":"2024-09-13T14:44:30.455135+0000","last_fullsized":"2024-09-13T14:44:30.455135+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":117,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.449243+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:44:30.449243+0000","last_peered":"2024-09-13T14:44:30.449243+0000","last_clean":"2024-09-13T14:44:30.449243+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:44:30.449243+0000","last_undegraded":"2024-09-13T14:44:30.449243+0000","last_fullsized":"2024-09-13T14:44:30.449243+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":129,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.853614+0000","last_change":"2024-09-13T14:39:42.767764+0000","last_active":"2024-09-13T14:44:38.853614+0000","last_peered":"2024-09-13T14:44:38.853614+0000","last_clean":"2024-09-13T14:44:38.853614+0000","last_became_active":"2024-09-13T14:39:42.767450+0000","last_became_peered":"2024-09-13T14:39:42.767450+0000","last_unstale":"2024-09-13T14:44:38.853614+0000","last_undegraded":"2024-09-13T14:44:38.853614+0000","last_fullsized":"2024-09-13T14:44:38.853614+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:47.729382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.860854+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:44:38.860854+0000","last_peered":"2024-09-13T14:44:38.860854+0000","last_clean":"2024-09-13T14:44:38.860854+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:44:38.860854+0000","last_undegraded":"2024-09-13T14:44:38.860854+0000","last_fullsized":"2024-09-13T14:44:38.860854+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":178,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.864277+0000","last_change":"2024-09-13T14:39:42.772266+0000","last_active":"2024-09-13T14:44:38.864277+0000","last_peered":"2024-09-13T14:44:38.864277+0000","last_clean":"2024-09-13T14:44:38.864277+0000","last_became_active":"2024-09-13T14:39:42.771880+0000","last_became_peered":"2024-09-13T14:39:42.771880+0000","last_unstale":"2024-09-13T14:44:38.864277+0000","last_undegraded":"2024-09-13T14:44:38.864277+0000","last_fullsized":"2024-09-13T14:44:38.864277+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:41:01.160105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":107,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.449413+0000","last_change":"2024-09-13T14:39:42.773890+0000","last_active":"2024-09-13T14:44:30.449413+0000","last_peered":"2024-09-13T14:44:30.449413+0000","last_clean":"2024-09-13T14:44:30.449413+0000","last_became_active":"2024-09-13T14:39:42.773523+0000","last_became_peered":"2024-09-13T14:39:42.773523+0000","last_unstale":"2024-09-13T14:44:30.449413+0000","last_undegraded":"2024-09-13T14:44:30.449413+0000","last_fullsized":"2024-09-13T14:44:30.449413+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:13.156261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":189,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.864078+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:44:38.864078+0000","last_peered":"2024-09-13T14:44:38.864078+0000","last_clean":"2024-09-13T14:44:38.864078+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:44:38.864078+0000","last_undegraded":"2024-09-13T14:44:38.864078+0000","last_fullsized":"2024-09-13T14:44:38.864078+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":166,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.860484+0000","last_change":"2024-09-13T14:38:13.979604+0000","last_active":"2024-09-13T14:44:38.860484+0000","last_peered":"2024-09-13T14:44:38.860484+0000","last_clean":"2024-09-13T14:44:38.860484+0000","last_became_active":"2024-09-13T14:38:13.979288+0000","last_became_peered":"2024-09-13T14:38:13.979288+0000","last_unstale":"2024-09-13T14:44:38.860484+0000","last_undegraded":"2024-09-13T14:44:38.860484+0000","last_fullsized":"2024-09-13T14:44:38.860484+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:21.031527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":153,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.858537+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:44:38.858537+0000","last_peered":"2024-09-13T14:44:38.858537+0000","last_clean":"2024-09-13T14:44:38.858537+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:44:38.858537+0000","last_undegraded":"2024-09-13T14:44:38.858537+0000","last_fullsized":"2024-09-13T14:44:38.858537+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":185,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.854820+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:44:38.854820+0000","last_peered":"2024-09-13T14:44:38.854820+0000","last_clean":"2024-09-13T14:44:38.854820+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:44:38.854820+0000","last_undegraded":"2024-09-13T14:44:38.854820+0000","last_fullsized":"2024-09-13T14:44:38.854820+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":136,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.447751+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:44:30.447751+0000","last_peered":"2024-09-13T14:44:30.447751+0000","last_clean":"2024-09-13T14:44:30.447751+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:44:30.447751+0000","last_undegraded":"2024-09-13T14:44:30.447751+0000","last_fullsized":"2024-09-13T14:44:30.447751+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":191,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.857252+0000","last_change":"2024-09-13T14:38:13.976950+0000","last_active":"2024-09-13T14:44:38.857252+0000","last_peered":"2024-09-13T14:44:38.857252+0000","last_clean":"2024-09-13T14:44:38.857252+0000","last_became_active":"2024-09-13T14:38:13.976639+0000","last_became_peered":"2024-09-13T14:38:13.976639+0000","last_unstale":"2024-09-13T14:44:38.857252+0000","last_undegraded":"2024-09-13T14:44:38.857252+0000","last_fullsized":"2024-09-13T14:44:38.857252+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:24.126890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":157,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.859976+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:44:38.859976+0000","last_peered":"2024-09-13T14:44:38.859976+0000","last_clean":"2024-09-13T14:44:38.859976+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:44:38.859976+0000","last_undegraded":"2024-09-13T14:44:38.859976+0000","last_fullsized":"2024-09-13T14:44:38.859976+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":134,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.447203+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:44:30.447203+0000","last_peered":"2024-09-13T14:44:30.447203+0000","last_clean":"2024-09-13T14:44:30.447203+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:44:30.447203+0000","last_undegraded":"2024-09-13T14:44:30.447203+0000","last_fullsized":"2024-09-13T14:44:30.447203+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":119,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.449184+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:44:30.449184+0000","last_peered":"2024-09-13T14:44:30.449184+0000","last_clean":"2024-09-13T14:44:30.449184+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:44:30.449184+0000","last_undegraded":"2024-09-13T14:44:30.449184+0000","last_fullsized":"2024-09-13T14:44:30.449184+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":134,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.447669+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:44:30.447669+0000","last_peered":"2024-09-13T14:44:30.447669+0000","last_clean":"2024-09-13T14:44:30.447669+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:44:30.447669+0000","last_undegraded":"2024-09-13T14:44:30.447669+0000","last_fullsized":"2024-09-13T14:44:30.447669+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":144,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.451163+0000","last_change":"2024-09-13T14:39:42.778413+0000","last_active":"2024-09-13T14:44:30.451163+0000","last_peered":"2024-09-13T14:44:30.451163+0000","last_clean":"2024-09-13T14:44:30.451163+0000","last_became_active":"2024-09-13T14:39:42.766624+0000","last_became_peered":"2024-09-13T14:39:42.766624+0000","last_unstale":"2024-09-13T14:44:30.451163+0000","last_undegraded":"2024-09-13T14:44:30.451163+0000","last_fullsized":"2024-09-13T14:44:30.451163+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:13.435604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.448852+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:44:30.448852+0000","last_peered":"2024-09-13T14:44:30.448852+0000","last_clean":"2024-09-13T14:44:30.448852+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:44:30.448852+0000","last_undegraded":"2024-09-13T14:44:30.448852+0000","last_fullsized":"2024-09-13T14:44:30.448852+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":185,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.853548+0000","last_change":"2024-09-13T14:39:42.767451+0000","last_active":"2024-09-13T14:44:38.853548+0000","last_peered":"2024-09-13T14:44:38.853548+0000","last_clean":"2024-09-13T14:44:38.853548+0000","last_became_active":"2024-09-13T14:39:42.767086+0000","last_became_peered":"2024-09-13T14:39:42.767086+0000","last_unstale":"2024-09-13T14:44:38.853548+0000","last_undegraded":"2024-09-13T14:44:38.853548+0000","last_fullsized":"2024-09-13T14:44:38.853548+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:02.674069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":120,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:44:30.446717+0000","last_change":"2024-09-13T14:39:42.768653+0000","last_active":"2024-09-13T14:44:30.446717+0000","last_peered":"2024-09-13T14:44:30.446717+0000","last_clean":"2024-09-13T14:44:30.446717+0000","last_became_active":"2024-09-13T14:39:42.768333+0000","last_became_peered":"2024-09-13T14:39:42.768333+0000","last_unstale":"2024-09-13T14:44:30.446717+0000","last_undegraded":"2024-09-13T14:44:30.446717+0000","last_fullsized":"2024-09-13T14:44:30.446717+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:25.655793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":169,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-09-13T14:44:38.860773+0000","last_change":"2024-09-13T14:37:40.924860+0000","last_active":"2024-09-13T14:44:38.860773+0000","last_peered":"2024-09-13T14:44:38.860773+0000","last_clean":"2024-09-13T14:44:38.860773+0000","last_became_active":"2024-09-13T14:37:40.924522+0000","last_became_peered":"2024-09-13T14:37:40.924522+0000","last_unstale":"2024-09-13T14:44:38.860773+0000","last_undegraded":"2024-09-13T14:44:38.860773+0000","last_fullsized":"2024-09-13T14:44:38.860773+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:10.340511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4057530368,"num_objects":61913,"num_object_clones":0,"num_object_copies":185739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":61913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61913,"num_write_kb":3962432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12064456704,"data_stored":12064456704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":61913,"ondisk_log_size":61913,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":9648,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397729,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2280332,"kb_used_data":1974660,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91464820,"statfs":{"total":95995035648,"available":93659975680,"internally_reserved":0,"allocated":2022051840,"data_stored":2021119465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,0,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":184683593958,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790187,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1524160,"kb_used_data":1226808,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":92220992,"statfs":{"total":95995035648,"available":94434295808,"internally_reserved":0,"allocated":1256251392,"data_stored":1255327318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986416,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1515584,"kb_used_data":1201592,"kb_used_omap":1,"kb_used_meta":313982,"kb_avail":92229568,"statfs":{"total":95995035648,"available":94443077632,"internally_reserved":0,"allocated":1230430208,"data_stored":1229505321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":321517912},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,4,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598864,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2594044,"kb_used_data":2427696,"kb_used_omap":6,"kb_used_meta":166329,"kb_avail":91151108,"statfs":{"total":95995035648,"available":93338734592,"internally_reserved":0,"allocated":2485960704,"data_stored":2485044261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":170321888},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762390,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2594364,"kb_used_data":2319732,"kb_used_omap":8,"kb_used_meta":274615,"kb_avail":91150788,"statfs":{"total":95995035648,"available":93338406912,"internally_reserved":0,"allocated":2375405568,"data_stored":2374481758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":281206218},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,4,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":94,"seq":403726925917,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1645816,"kb_used_data":1491952,"kb_used_omap":7,"kb_used_meta":153848,"kb_avail":92099336,"statfs":{"total":95995035648,"available":94309720064,"internally_reserved":0,"allocated":1527758848,"data_stored":1526842054,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":157541065},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122157,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1240656,"kb_used_data":1038216,"kb_used_omap":7,"kb_used_meta":202424,"kb_avail":92504496,"statfs":{"total":95995035648,"available":94724603904,"internally_reserved":0,"allocated":1063133184,"data_stored":1062197240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":207282767},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1061879808,"data_stored":1061879808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1530134528,"data_stored":1530134528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2406547456,"data_stored":2406547456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2525626368,"data_stored":2525626368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1245249536,"data_stored":1245249536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1266679808,"data_stored":1266679808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2028339200,"data_stored":2028339200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:44:47.724 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 11 2024-09-13T14:44:47.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.725 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:47.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:47.826 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:47.858 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:44:47.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.885 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:47.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:47.929 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 39 16 69650 69634 111.581 76.4375 0.017734 0.00895728 2024-09-13T14:44:47.951 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:44:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:44:47] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-13T14:44:47.986 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:48.014 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:48.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:48.116 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:48.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.150 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:48.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:48.251 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:48.319 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:48.323 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:44:48.324 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:48.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:48.330 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:48.344 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:48.362 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:44:48.362 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:48.394 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:44:48.394 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:48.425 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:48.427 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:44:48.428 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:48.452 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:48.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:48.458 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:44:48.458 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:48.488 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:44:48.488 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:48.518 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:44:48.518 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:48.548 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:44:48.549 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:48.554 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:48.580 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:48.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:48.682 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:48.726 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:48.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:48.828 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:48.857 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:48.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:48.929 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:44:48.929180+0000 min lat: 0.00106401 max lat: 0.202978 avg lat: 0.00903842 2024-09-13T14:44:48.929 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:44:48.929 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 40 16 70792 70776 110.576 71.375 0.00801892 0.00903842 2024-09-13T14:44:48.959 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:48.988 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:48.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:49.090 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:49.129 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:49.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:49.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:48 smithi204 ceph-mon[87071]: pgmap v440: 99 pgs: 99 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-13T14:44:49.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:48 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3844102363' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:49.232 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:49.270 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:49.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:48 smithi096 ceph-mon[98901]: pgmap v440: 99 pgs: 99 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-13T14:44:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:48 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3844102363' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:48 smithi096 ceph-mon[102928]: pgmap v440: 99 pgs: 99 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-13T14:44:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:48 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3844102363' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:49.372 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:49.404 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:49.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:49.506 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:49.543 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:49.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:49.645 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:49.672 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:49.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:49.774 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:49.809 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:49.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:49.911 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:49.930 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 41 16 72064 72048 109.815 79.5 0.0116014 0.00910066 2024-09-13T14:44:49.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.947 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:49.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:49.999 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:49.999 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:50.047 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:50.080 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:50.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:50.182 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:50.223 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:50.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:50.324 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:50.352 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:50.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:50.366 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:44:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:44:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:50.454 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:50.490 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:50.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:50.592 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:50.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.624 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:50.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:50.706 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:44:50.725 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:50.758 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:50.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:50.860 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:50.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.904 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:50.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:50.930 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 42 16 73349 73333 109.113 80.3125 0.0103419 0.00915853 2024-09-13T14:44:50.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:50 smithi204 ceph-mon[87071]: pgmap v441: 99 pgs: 99 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-09-13T14:44:50.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:50 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:51.005 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:51.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.055 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:51.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:51.099 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:51.157 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:51.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.189 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:51.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:51.208 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:50 smithi096 ceph-mon[98901]: pgmap v441: 99 pgs: 99 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-09-13T14:44:51.208 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:50 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:51.208 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:50 smithi096 ceph-mon[102928]: pgmap v441: 99 pgs: 99 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-09-13T14:44:51.208 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:50 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:51.290 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:51.319 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:51.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:51.421 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:51.459 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:51.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:51.561 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:51.597 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:51.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:51.700 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:51.730 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:51.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:51.831 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:51.852 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:51.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:51.930 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 43 16 74480 74464 108.219 70.6875 0.0146479 0.00923512 2024-09-13T14:44:51.955 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:51.983 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:51.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:52.085 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:52.114 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:52.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:52.125 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:44:52.215 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:52.241 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:52.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:52.344 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:52.388 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:52.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:52.490 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:52.517 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:52.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:52.619 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:52.651 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:52.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:52.753 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:52.783 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:52.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:52.879 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:52.880 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:52.885 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:52.919 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:52.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:52.930 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 44 16 75623 75607 107.383 71.4375 0.00809255 0.00930694 2024-09-13T14:44:53.021 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:53.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.049 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:53.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:53.150 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:53.168 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:52 smithi096 ceph-mon[98901]: pgmap v442: 99 pgs: 99 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:44:53.169 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:52 smithi096 ceph-mon[102928]: pgmap v442: 99 pgs: 99 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:44:53.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:52 smithi204 ceph-mon[87071]: pgmap v442: 99 pgs: 99 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:44:53.219 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:53.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:53.320 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:53.348 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:53.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:53.452 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:53.512 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:53.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:53.596 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:44:53.613 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:53.656 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:53.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:53.757 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:53.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.793 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:53.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:53.894 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:53.930 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 45 16 76696 76680 106.487 67.0625 0.00828524 0.00938397 2024-09-13T14:44:53.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.939 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:53.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:54.040 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:54.066 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:54.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:54.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:53 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:44:54.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:53 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:44:54.168 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:54.210 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:54.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:54.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:53 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:44:54.255 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:54.313 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:54.338 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:54.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:54.440 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:54.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.481 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:54.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:54.582 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:54.622 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:54.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:54.724 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:54.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.769 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:54.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:54.842 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:54.869 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:54.908 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:44:54.909 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:54.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:54.912 INFO:teuthology.orchestra.run.smithi096.stderr:set pool 1 pgp_num to 11 2024-09-13T14:44:54.930 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 46 16 77689 77673 105.521 62.0625 0.00742647 0.00947074 2024-09-13T14:44:55.011 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:55.013 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:54 smithi096 ceph-mon[98901]: pgmap v443: 99 pgs: 99 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T14:44:55.013 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:54 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1383423898' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:55.013 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:54 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:55.013 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:54 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1823769778' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:44:55.013 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:54 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:44:55.013 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:54 smithi096 ceph-mon[102928]: pgmap v443: 99 pgs: 99 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T14:44:55.013 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:54 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1383423898' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:55.013 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:54 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:55.013 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:54 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1823769778' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:44:55.013 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:54 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:44:55.046 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:55.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:55.148 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:55.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.188 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:55.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:54 smithi204 ceph-mon[87071]: pgmap v443: 99 pgs: 99 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T14:44:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:54 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1383423898' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:54 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:54 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1823769778' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:44:55.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:54 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:44:55.289 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:55.322 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:55.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:55.425 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:55.465 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:55.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:55.567 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:55.609 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:55.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:55.711 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:55.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.737 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:55.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:55.838 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:55.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.874 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:55.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:55.931 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 47 16 78900 78884 104.886 75.6875 0.0186697 0.00952836 2024-09-13T14:44:55.975 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:56.014 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:56.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:56.072 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:55 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:56.072 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:55 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:44:56.072 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:55 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:44:56.072 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:55 smithi204 ceph-mon[87071]: osdmap e154: 8 total, 8 up, 7 in 2024-09-13T14:44:56.115 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:56.167 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:56.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:56.269 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:56.298 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:56.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:56.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:55 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:56.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:55 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:44:56.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:55 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:44:56.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:55 smithi096 ceph-mon[98901]: osdmap e154: 8 total, 8 up, 7 in 2024-09-13T14:44:56.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:55 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:56.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:55 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:44:56.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:55 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:44:56.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:55 smithi096 ceph-mon[102928]: osdmap e154: 8 total, 8 up, 7 in 2024-09-13T14:44:56.400 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:56.427 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:56.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:56.528 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:56.558 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:56.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:56.660 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:56.684 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:56.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:56.786 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:56.819 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:56.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:56.921 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:56.931 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 48 16 80101 80085 104.265 75.0625 0.0191637 0.00958477 2024-09-13T14:44:56.956 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:56.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:57.058 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:57.106 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:57.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:57.204 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:56 smithi204 ceph-mon[87071]: pgmap v445: 99 pgs: 99 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:44:57.208 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:57.243 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:56 smithi096 ceph-mon[98901]: pgmap v445: 99 pgs: 99 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:44:57.243 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:56 smithi096 ceph-mon[102928]: pgmap v445: 99 pgs: 99 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:44:57.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.261 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:57.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:57.324 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:57.324 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:57.363 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:57.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.415 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:57.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:57.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:57.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.541 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:57.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:57.647 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:57.675 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:57.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:57.777 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:57.802 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:57.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:57.903 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:57.907 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:44:57.908 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:57.930 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 49 16 81111 81095 103.425 63.125 0.0383581 0.00966348 2024-09-13T14:44:57.940 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:57.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:57.943 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:44:57.943 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:57.978 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:44:57.979 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:58.017 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:44:58.018 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:58.042 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:58.054 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:44:58.054 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:58.081 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:58.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:58.083 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:44:58.083 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:58.118 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:44:58.118 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:58.118 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:57 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3516080963' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:58.118 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:57 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3516080963' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:58.119 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:44:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:44:57] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.51.0" 2024-09-13T14:44:58.150 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:44:58.151 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:58.183 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:58.193 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:57 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3516080963' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:58.209 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:58.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:58.225 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:44:58.311 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:58.336 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:58.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:58.439 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:58.468 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:58.521 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:44:58.569 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:58.596 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:58.700 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:58.725 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:58.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:58.827 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:58.882 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:58.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:58.931 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 50 16 81994 81978 102.46 55.1875 0.028829 0.00975455 2024-09-13T14:44:58.985 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:59.013 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:59.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:59.116 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:59.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.159 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:59.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:59.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:44:58 smithi204 ceph-mon[87071]: pgmap v446: 99 pgs: 99 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:44:59.232 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:59.259 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:59.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.311 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:59.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:59.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:44:58 smithi096 ceph-mon[98901]: pgmap v446: 99 pgs: 99 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:44:59.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:44:58 smithi096 ceph-mon[102928]: pgmap v446: 99 pgs: 99 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:44:59.411 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:59.438 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:59.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:59.540 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:59.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.572 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:59.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:59.674 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:59.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.724 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:59.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:59.825 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:59.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.856 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:44:59.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:59.931 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 51 16 82933 82917 101.602 58.6875 0.0219349 0.00983682 2024-09-13T14:44:59.958 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:59.985 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:44:59.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:00.087 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:00.144 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:00.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:00.245 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:00.273 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:00.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:00.375 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:00.414 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:00.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:00.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:45:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:45:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:00.516 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:00.559 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:00.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:00.661 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:00.689 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:00.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:00.790 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:00.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.829 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:00.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:00.910 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:45:00.930 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:00.933 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 52 16 84016 84000 100.949 67.6875 0.0071111 0.00990069 2024-09-13T14:45:00.984 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:00.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:01.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:00 smithi096 ceph-mon[98901]: pgmap v447: 99 pgs: 99 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:45:01.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:00 smithi096 ceph-mon[102928]: pgmap v447: 99 pgs: 99 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:45:01.078 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:00 smithi204 ceph-mon[87071]: pgmap v447: 99 pgs: 99 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:45:01.086 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:01.119 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:01.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:01.131 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:45:01.221 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:01.246 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:01.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:01.348 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:01.413 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:01.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:01.515 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:01.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.569 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:01.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:01.670 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:01.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.710 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:01.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:01.811 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:01.842 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:01.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:01.931 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 53 16 84993 84977 100.196 61.0625 0.0217383 0.00997437 2024-09-13T14:45:01.944 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:01.947 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:45:01.947 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:45:01.947 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-09-13T14:45:01.947 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 1000 0.05 2024-09-13T14:45:01.983 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:45:02.007 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:02.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.022 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:02.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:02.123 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:02.150 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:02.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:02.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:01 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/766382992' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:02.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:01 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:02.252 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:02.279 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:02.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:02.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:01 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/766382992' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:02.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:01 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:02.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:01 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/766382992' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:02.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:01 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:02.380 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:02.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.427 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:02.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:02.528 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:02.554 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:02.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:02.656 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:02.696 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:02.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:02.798 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:02.841 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:02.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:02.931 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 54 16 86019 86003 99.5284 64.125 0.0113918 0.010041 2024-09-13T14:45:02.944 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:02.993 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:02.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:03.095 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:03.125 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:03.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:03.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:02 smithi204 ceph-mon[87071]: pgmap v448: 99 pgs: 99 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 72 MiB/s wr, 1.14k op/s 2024-09-13T14:45:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:02 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:45:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:02 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:45:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:02 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:02 smithi204 ceph-mon[87071]: osdmap e155: 8 total, 8 up, 7 in 2024-09-13T14:45:03.227 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:03.253 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:03.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:03.269 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:02 smithi096 ceph-mon[98901]: pgmap v448: 99 pgs: 99 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 72 MiB/s wr, 1.14k op/s 2024-09-13T14:45:03.269 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:02 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:45:03.269 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:02 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:45:03.269 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:02 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:03.269 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:02 smithi096 ceph-mon[98901]: osdmap e155: 8 total, 8 up, 7 in 2024-09-13T14:45:03.269 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:02 smithi096 ceph-mon[102928]: pgmap v448: 99 pgs: 99 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 72 MiB/s wr, 1.14k op/s 2024-09-13T14:45:03.269 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:02 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:45:03.269 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:02 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:45:03.269 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:02 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:03.269 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:02 smithi096 ceph-mon[102928]: osdmap e155: 8 total, 8 up, 7 in 2024-09-13T14:45:03.354 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:03.402 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:03.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:03.504 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:03.530 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:03.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:03.631 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:03.662 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:03.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:03.746 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:45:03.746 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T14:45:03.746 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T14:45:03.746 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:45:03.746 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.746 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:68069 5.4 5:20652590:::benchmark_data_smithi096_138841_object68068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:45:03.747 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:25.511751+0000", 2024-09-13T14:45:03.747 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 278.23066556800001, 2024-09-13T14:45:03.747 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.166519957, 2024-09-13T14:45:03.747 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.747 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.747 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:45:03.747 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:45:03.747 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:45:03.747 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 68069 2024-09-13T14:45:03.747 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.747 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.747 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.747 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.748 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.511751+0000", 2024-09-13T14:45:03.748 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.748 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.748 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.748 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.748 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.511751+0000", 2024-09-13T14:45:03.748 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.748 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.748 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.748 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.748 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.511750+0000", 2024-09-13T14:45:03.748 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:45:03.748 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.749 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.749 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.749 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.511839+0000", 2024-09-13T14:45:03.749 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8355000000000004e-05 2024-09-13T14:45:03.749 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.749 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.749 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.749 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.511853+0000", 2024-09-13T14:45:03.749 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4687e-05 2024-09-13T14:45:03.749 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.749 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.749 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.750 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.511859+0000", 2024-09-13T14:45:03.750 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5099999999999998e-06 2024-09-13T14:45:03.750 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.750 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.750 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.750 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.513682+0000", 2024-09-13T14:45:03.750 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001823437 2024-09-13T14:45:03.750 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.750 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.750 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.750 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.513803+0000", 2024-09-13T14:45:03.750 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000121128 2024-09-13T14:45:03.750 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.750 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.751 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:45:03.751 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.513923+0000", 2024-09-13T14:45:03.751 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119542 2024-09-13T14:45:03.751 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.751 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.751 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:45:03.751 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.518669+0000", 2024-09-13T14:45:03.751 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0047461439999999999 2024-09-13T14:45:03.751 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.751 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.751 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.751 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.519128+0000", 2024-09-13T14:45:03.751 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000459004 2024-09-13T14:45:03.752 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.752 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.752 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.752 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.678206+0000", 2024-09-13T14:45:03.752 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15907775399999999 2024-09-13T14:45:03.752 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.752 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.752 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.752 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.678244+0000", 2024-09-13T14:45:03.752 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8247000000000002e-05 2024-09-13T14:45:03.752 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.752 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.752 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.753 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.678271+0000", 2024-09-13T14:45:03.753 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7084e-05 2024-09-13T14:45:03.753 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.753 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.753 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.753 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.753 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.753 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:68077 5.4 5:2c136659:::benchmark_data_smithi096_138841_object68076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:45:03.753 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:25.515483+0000", 2024-09-13T14:45:03.754 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 278.22693417099998, 2024-09-13T14:45:03.754 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.162940803, 2024-09-13T14:45:03.754 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.754 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.754 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:45:03.754 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:45:03.754 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:45:03.754 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 68077 2024-09-13T14:45:03.754 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.754 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.754 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.754 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.754 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.515483+0000", 2024-09-13T14:45:03.755 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.755 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.755 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.755 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.755 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.515483+0000", 2024-09-13T14:45:03.755 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.755 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.755 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.755 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.755 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.515481+0000", 2024-09-13T14:45:03.755 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:03.755 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.756 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.756 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.756 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.515519+0000", 2024-09-13T14:45:03.756 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8167000000000002e-05 2024-09-13T14:45:03.756 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.756 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.756 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.756 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.515531+0000", 2024-09-13T14:45:03.756 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.207e-05 2024-09-13T14:45:03.756 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.756 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.756 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.757 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.515537+0000", 2024-09-13T14:45:03.757 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2920000000000003e-06 2024-09-13T14:45:03.757 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.757 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.757 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.757 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.516197+0000", 2024-09-13T14:45:03.757 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00066015900000000001 2024-09-13T14:45:03.757 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.757 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.757 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.757 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.516307+0000", 2024-09-13T14:45:03.758 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010998 2024-09-13T14:45:03.758 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.758 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.758 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:45:03.758 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.516439+0000", 2024-09-13T14:45:03.758 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013221700000000001 2024-09-13T14:45:03.758 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.758 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.758 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:45:03.758 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.518865+0000", 2024-09-13T14:45:03.758 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002426283 2024-09-13T14:45:03.758 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.758 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.759 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.759 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.519241+0000", 2024-09-13T14:45:03.759 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00037589999999999998 2024-09-13T14:45:03.759 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.759 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.759 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.759 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.678330+0000", 2024-09-13T14:45:03.759 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15908873300000001 2024-09-13T14:45:03.759 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.759 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.759 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.759 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.678378+0000", 2024-09-13T14:45:03.759 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8155000000000002e-05 2024-09-13T14:45:03.759 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.760 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.760 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.760 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.678423+0000", 2024-09-13T14:45:03.760 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5364000000000001e-05 2024-09-13T14:45:03.760 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.760 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.760 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.760 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.760 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.760 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:68092 5.4 5:2f0335d4:::benchmark_data_smithi096_138841_object68091:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:45:03.760 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:25.524527+0000", 2024-09-13T14:45:03.760 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 278.21789006, 2024-09-13T14:45:03.760 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16264743700000001, 2024-09-13T14:45:03.761 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.761 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.761 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:45:03.761 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:45:03.761 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:45:03.762 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 68092 2024-09-13T14:45:03.762 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.762 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.762 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.762 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.762 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.524527+0000", 2024-09-13T14:45:03.762 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.762 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.762 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.762 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.762 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.524527+0000", 2024-09-13T14:45:03.763 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.763 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.763 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.763 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.763 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.524525+0000", 2024-09-13T14:45:03.763 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:03.763 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.763 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.763 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.763 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.524574+0000", 2024-09-13T14:45:03.763 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8387999999999998e-05 2024-09-13T14:45:03.764 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.764 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.764 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.764 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.524586+0000", 2024-09-13T14:45:03.764 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1883e-05 2024-09-13T14:45:03.764 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.764 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.764 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.764 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.524590+0000", 2024-09-13T14:45:03.764 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5369999999999999e-06 2024-09-13T14:45:03.764 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.765 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.765 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.765 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.524636+0000", 2024-09-13T14:45:03.765 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5289e-05 2024-09-13T14:45:03.765 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.765 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.765 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.765 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.524727+0000", 2024-09-13T14:45:03.765 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1606000000000004e-05 2024-09-13T14:45:03.765 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.765 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.765 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:45:03.765 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.524819+0000", 2024-09-13T14:45:03.766 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1841000000000004e-05 2024-09-13T14:45:03.766 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.766 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.766 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:45:03.766 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.526600+0000", 2024-09-13T14:45:03.766 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001780565 2024-09-13T14:45:03.766 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.766 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.766 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.766 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.526945+0000", 2024-09-13T14:45:03.766 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00034550599999999998 2024-09-13T14:45:03.766 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.766 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.767 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.767 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.687093+0000", 2024-09-13T14:45:03.767 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16014787799999999 2024-09-13T14:45:03.767 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.767 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.767 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.767 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.687141+0000", 2024-09-13T14:45:03.767 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7586000000000001e-05 2024-09-13T14:45:03.767 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.767 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.767 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.767 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.687174+0000", 2024-09-13T14:45:03.767 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3553999999999998e-05 2024-09-13T14:45:03.767 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.768 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.768 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.768 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.768 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.768 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:34135 6.3 6:c2de75f9:::benchmark_data_smithi096_167087_object34134:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:45:03.768 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.699103+0000", 2024-09-13T14:45:03.768 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 41.043314101999997, 2024-09-13T14:45:03.768 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19865819200000001, 2024-09-13T14:45:03.768 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.768 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.768 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:45:03.769 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:45:03.769 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:45:03.769 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 34135 2024-09-13T14:45:03.769 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.769 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.769 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.769 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.769 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699103+0000", 2024-09-13T14:45:03.769 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.769 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.769 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.769 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.770 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699103+0000", 2024-09-13T14:45:03.770 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.770 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.770 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.770 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.770 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699102+0000", 2024-09-13T14:45:03.770 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:45:03.770 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.770 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.770 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.770 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699154+0000", 2024-09-13T14:45:03.770 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2114000000000003e-05 2024-09-13T14:45:03.770 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.770 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.771 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.771 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699161+0000", 2024-09-13T14:45:03.771 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1899999999999998e-06 2024-09-13T14:45:03.771 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.771 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.771 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.771 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699165+0000", 2024-09-13T14:45:03.771 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.597e-06 2024-09-13T14:45:03.771 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.771 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.771 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.771 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699198+0000", 2024-09-13T14:45:03.771 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3531000000000001e-05 2024-09-13T14:45:03.772 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.772 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.772 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.772 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699264+0000", 2024-09-13T14:45:03.772 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5819000000000001e-05 2024-09-13T14:45:03.772 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.772 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.772 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:45:03.772 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699341+0000", 2024-09-13T14:45:03.772 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6790000000000002e-05 2024-09-13T14:45:03.772 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.772 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.772 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:45:03.772 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.701095+0000", 2024-09-13T14:45:03.773 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001754434 2024-09-13T14:45:03.773 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.773 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.773 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.773 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897558+0000", 2024-09-13T14:45:03.773 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19646306199999999 2024-09-13T14:45:03.773 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.773 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.773 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.773 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897605+0000", 2024-09-13T14:45:03.773 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7012000000000003e-05 2024-09-13T14:45:03.773 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.773 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.773 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.774 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897648+0000", 2024-09-13T14:45:03.774 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3058e-05 2024-09-13T14:45:03.774 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.774 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.774 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.774 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897761+0000", 2024-09-13T14:45:03.774 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011252999999999999 2024-09-13T14:45:03.774 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.774 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.775 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.775 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.775 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.775 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:34139 6.3 6:cd715ee8:::benchmark_data_smithi096_167087_object34138:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:45:03.775 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.699339+0000", 2024-09-13T14:45:03.775 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 41.043077857, 2024-09-13T14:45:03.775 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19895058299999999, 2024-09-13T14:45:03.775 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.775 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.775 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:45:03.775 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:45:03.775 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:45:03.775 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 34139 2024-09-13T14:45:03.776 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.776 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.776 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.776 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.776 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699339+0000", 2024-09-13T14:45:03.776 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.776 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.776 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.776 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.776 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699339+0000", 2024-09-13T14:45:03.776 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.776 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.776 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.777 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.777 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699338+0000", 2024-09-13T14:45:03.777 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:45:03.777 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.777 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.777 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.777 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699447+0000", 2024-09-13T14:45:03.777 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000109077 2024-09-13T14:45:03.777 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.777 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.778 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.778 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699456+0000", 2024-09-13T14:45:03.778 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3520000000000007e-06 2024-09-13T14:45:03.778 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.778 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.778 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.778 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699460+0000", 2024-09-13T14:45:03.778 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6920000000000001e-06 2024-09-13T14:45:03.778 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.778 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.778 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.778 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.700283+0000", 2024-09-13T14:45:03.779 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000822742 2024-09-13T14:45:03.779 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.779 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.779 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.779 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.700351+0000", 2024-09-13T14:45:03.779 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8039000000000004e-05 2024-09-13T14:45:03.779 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.779 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.779 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:45:03.779 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.700433+0000", 2024-09-13T14:45:03.779 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2398999999999995e-05 2024-09-13T14:45:03.779 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.779 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.780 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:45:03.780 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.701174+0000", 2024-09-13T14:45:03.780 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00074037599999999995 2024-09-13T14:45:03.780 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.780 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.780 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.780 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897584+0000", 2024-09-13T14:45:03.780 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19641035100000001 2024-09-13T14:45:03.780 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.780 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.780 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.780 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.898214+0000", 2024-09-13T14:45:03.780 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00063014600000000005 2024-09-13T14:45:03.781 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.781 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.781 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.781 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.898255+0000", 2024-09-13T14:45:03.781 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1083e-05 2024-09-13T14:45:03.781 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.781 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.781 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.781 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.898289+0000", 2024-09-13T14:45:03.781 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4069000000000002e-05 2024-09-13T14:45:03.781 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.781 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.781 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.782 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.782 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.782 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:34136 6.c e150/145 6:3e63118a:::benchmark_data_smithi096_167087_object34135:head v 150'2081, mlcod=150'2079)", 2024-09-13T14:45:03.782 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.703607+0000", 2024-09-13T14:45:03.782 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 41.038809569999998, 2024-09-13T14:45:03.782 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19718830400000001, 2024-09-13T14:45:03.782 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.782 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.782 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.782 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.782 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.782 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703607+0000", 2024-09-13T14:45:03.782 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.783 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.783 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.783 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.783 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703607+0000", 2024-09-13T14:45:03.783 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.783 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.783 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.783 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.783 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703606+0000", 2024-09-13T14:45:03.783 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:03.784 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.784 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.784 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.784 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703652+0000", 2024-09-13T14:45:03.784 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6789000000000002e-05 2024-09-13T14:45:03.784 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.784 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.784 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.785 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703674+0000", 2024-09-13T14:45:03.785 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1297e-05 2024-09-13T14:45:03.785 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.785 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.785 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.785 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703683+0000", 2024-09-13T14:45:03.785 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5559999999999996e-06 2024-09-13T14:45:03.785 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.785 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.785 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.786 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897235+0000", 2024-09-13T14:45:03.786 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19355149599999999 2024-09-13T14:45:03.786 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.786 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.786 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.786 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897275+0000", 2024-09-13T14:45:03.786 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0275000000000003e-05 2024-09-13T14:45:03.786 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.786 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.786 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.786 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.900770+0000", 2024-09-13T14:45:03.787 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0034945810000000001 2024-09-13T14:45:03.787 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.787 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.787 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.787 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.900795+0000", 2024-09-13T14:45:03.787 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5803e-05 2024-09-13T14:45:03.787 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.787 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.787 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.787 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.787 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.787 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.45167.0:34135 6.3 e150/145 ondisk, result = 0)", 2024-09-13T14:45:03.788 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.704116+0000", 2024-09-13T14:45:03.788 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 41.03830061, 2024-09-13T14:45:03.788 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19344439799999999, 2024-09-13T14:45:03.788 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.788 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:45:03.788 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.788 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.788 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.788 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704116+0000", 2024-09-13T14:45:03.788 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.788 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.788 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.788 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.789 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704116+0000", 2024-09-13T14:45:03.789 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.789 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.789 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.789 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.789 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704115+0000", 2024-09-13T14:45:03.789 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:45:03.789 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.789 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.789 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.789 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704119+0000", 2024-09-13T14:45:03.789 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9960000000000004e-06 2024-09-13T14:45:03.789 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.790 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.790 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.790 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897303+0000", 2024-09-13T14:45:03.790 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19318347899999999 2024-09-13T14:45:03.790 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.790 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.790 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.790 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897312+0000", 2024-09-13T14:45:03.790 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8969999999999997e-06 2024-09-13T14:45:03.790 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.791 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.791 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.791 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897545+0000", 2024-09-13T14:45:03.791 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00023321799999999999 2024-09-13T14:45:03.791 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.792 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.792 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.792 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897552+0000", 2024-09-13T14:45:03.792 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.396e-06 2024-09-13T14:45:03.792 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.792 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.792 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.792 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897560+0000", 2024-09-13T14:45:03.792 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0050000000000004e-06 2024-09-13T14:45:03.792 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.792 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.793 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.793 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.793 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.793 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:34141 6.b e150/145 6:d45fd98b:::benchmark_data_smithi096_167087_object34140:head v 150'2166, mlcod=150'2163)", 2024-09-13T14:45:03.793 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.704766+0000", 2024-09-13T14:45:03.793 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 41.037650366999998, 2024-09-13T14:45:03.793 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.197159529, 2024-09-13T14:45:03.793 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.793 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.793 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.793 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.793 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.793 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704766+0000", 2024-09-13T14:45:03.794 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.794 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.794 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.794 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.794 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704766+0000", 2024-09-13T14:45:03.794 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.794 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.794 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.794 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.794 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704766+0000", 2024-09-13T14:45:03.794 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:45:03.794 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.794 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.795 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.795 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704832+0000", 2024-09-13T14:45:03.795 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5964999999999996e-05 2024-09-13T14:45:03.795 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.795 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.795 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.795 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897456+0000", 2024-09-13T14:45:03.795 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.192624657 2024-09-13T14:45:03.795 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.795 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.795 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.795 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897466+0000", 2024-09-13T14:45:03.796 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5510000000000005e-06 2024-09-13T14:45:03.796 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.796 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.796 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.796 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897783+0000", 2024-09-13T14:45:03.796 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00031745699999999999 2024-09-13T14:45:03.796 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.796 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.796 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.796 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897800+0000", 2024-09-13T14:45:03.796 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6619000000000001e-05 2024-09-13T14:45:03.796 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.796 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.797 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.797 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.901888+0000", 2024-09-13T14:45:03.797 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0040879230000000003 2024-09-13T14:45:03.797 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.797 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.797 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.797 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.901926+0000", 2024-09-13T14:45:03.797 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7832e-05 2024-09-13T14:45:03.797 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.797 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.797 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.797 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.797 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.798 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:35700 6.b e150/145 6:db998901:::benchmark_data_smithi096_167087_object35699:head v 150'2254, mlcod=150'2252)", 2024-09-13T14:45:03.798 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:23.878355+0000", 2024-09-13T14:45:03.798 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 39.864061726999999, 2024-09-13T14:45:03.798 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.163311281, 2024-09-13T14:45:03.798 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.798 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.798 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.798 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.798 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.798 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.878355+0000", 2024-09-13T14:45:03.798 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.798 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.798 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.799 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.799 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.878355+0000", 2024-09-13T14:45:03.799 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.799 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.799 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.799 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.799 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.878354+0000", 2024-09-13T14:45:03.799 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:45:03.799 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.799 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.799 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.799 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.878405+0000", 2024-09-13T14:45:03.800 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0309999999999998e-05 2024-09-13T14:45:03.800 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.800 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.800 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.800 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.878415+0000", 2024-09-13T14:45:03.800 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0042e-05 2024-09-13T14:45:03.800 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.800 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.800 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.800 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.878420+0000", 2024-09-13T14:45:03.800 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9540000000000003e-06 2024-09-13T14:45:03.800 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.800 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.801 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.801 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.878467+0000", 2024-09-13T14:45:03.801 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7765000000000001e-05 2024-09-13T14:45:03.801 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.801 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.801 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.801 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.878485+0000", 2024-09-13T14:45:03.801 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7578999999999999e-05 2024-09-13T14:45:03.801 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.801 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.801 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.801 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041633+0000", 2024-09-13T14:45:03.801 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16314788199999999 2024-09-13T14:45:03.802 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.802 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.802 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.802 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041666+0000", 2024-09-13T14:45:03.802 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3385e-05 2024-09-13T14:45:03.802 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.802 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.802 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.802 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.802 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.802 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:35702 6.b e150/145 6:d22ffcf1:::benchmark_data_smithi096_167087_object35701:head v 150'2255, mlcod=150'2252)", 2024-09-13T14:45:03.802 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:23.879183+0000", 2024-09-13T14:45:03.802 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 39.863233887, 2024-09-13T14:45:03.803 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.162503021, 2024-09-13T14:45:03.803 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.803 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.803 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.803 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.803 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.803 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879183+0000", 2024-09-13T14:45:03.803 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.803 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.803 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.803 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.803 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879183+0000", 2024-09-13T14:45:03.803 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.803 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.804 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.804 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.804 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879182+0000", 2024-09-13T14:45:03.804 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:45:03.804 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.804 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.804 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.804 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879238+0000", 2024-09-13T14:45:03.804 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5962999999999999e-05 2024-09-13T14:45:03.804 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.804 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.804 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.804 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879246+0000", 2024-09-13T14:45:03.805 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.225e-06 2024-09-13T14:45:03.805 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.805 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.805 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.805 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879250+0000", 2024-09-13T14:45:03.805 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.771e-06 2024-09-13T14:45:03.805 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.805 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.805 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.805 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879659+0000", 2024-09-13T14:45:03.805 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00040838300000000001 2024-09-13T14:45:03.805 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.805 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.806 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.806 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879670+0000", 2024-09-13T14:45:03.806 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1938e-05 2024-09-13T14:45:03.806 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.806 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.806 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.806 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041669+0000", 2024-09-13T14:45:03.806 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16199898300000001 2024-09-13T14:45:03.806 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.806 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.806 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.806 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041686+0000", 2024-09-13T14:45:03.806 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6368e-05 2024-09-13T14:45:03.807 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.807 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.807 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.807 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.807 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.807 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:35708 6.6 e150/145 6:653b67ef:::benchmark_data_smithi096_167087_object35707:head v 150'2235, mlcod=150'2233)", 2024-09-13T14:45:03.807 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:23.879256+0000", 2024-09-13T14:45:03.807 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 39.863160442999998, 2024-09-13T14:45:03.807 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16249878000000001, 2024-09-13T14:45:03.807 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.807 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.807 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.807 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.807 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.807 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879256+0000", 2024-09-13T14:45:03.808 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.808 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.808 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.808 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.808 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879256+0000", 2024-09-13T14:45:03.808 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.808 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.808 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.808 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.808 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879256+0000", 2024-09-13T14:45:03.808 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:45:03.808 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.809 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.809 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.809 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879354+0000", 2024-09-13T14:45:03.809 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7880000000000005e-05 2024-09-13T14:45:03.809 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.809 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.809 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.809 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879359+0000", 2024-09-13T14:45:03.809 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:03.810 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1030000000000001e-06 2024-09-13T14:45:03.811 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.811 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.811 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.811 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879362+0000", 2024-09-13T14:45:03.811 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8250000000000001e-06 2024-09-13T14:45:03.811 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.811 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.811 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.811 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879405+0000", 2024-09-13T14:45:03.811 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.32e-05 2024-09-13T14:45:03.811 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.812 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.812 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.812 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.879423+0000", 2024-09-13T14:45:03.812 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8309999999999999e-05 2024-09-13T14:45:03.812 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.812 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.812 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.812 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041732+0000", 2024-09-13T14:45:03.812 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.162308701 2024-09-13T14:45:03.812 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.812 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.812 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.812 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041755+0000", 2024-09-13T14:45:03.813 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.3020999999999999e-05 2024-09-13T14:45:03.813 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.813 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.813 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.813 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.813 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.813 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:35712 6.3 6:ca714cda:::benchmark_data_smithi096_167087_object35711:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:45:03.813 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:23.880208+0000", 2024-09-13T14:45:03.813 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 39.862208803999998, 2024-09-13T14:45:03.813 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16539719, 2024-09-13T14:45:03.813 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.813 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.814 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:45:03.814 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:45:03.814 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:45:03.814 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 35712 2024-09-13T14:45:03.814 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.814 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.814 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.814 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.814 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880208+0000", 2024-09-13T14:45:03.814 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.814 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.814 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.815 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.815 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880208+0000", 2024-09-13T14:45:03.815 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.815 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.815 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.815 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.815 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880206+0000", 2024-09-13T14:45:03.815 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:03.815 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.815 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.815 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.815 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880264+0000", 2024-09-13T14:45:03.815 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7732999999999998e-05 2024-09-13T14:45:03.816 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.816 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.816 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.816 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880277+0000", 2024-09-13T14:45:03.816 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2296999999999999e-05 2024-09-13T14:45:03.816 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.816 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.816 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.816 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880282+0000", 2024-09-13T14:45:03.816 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6640000000000003e-06 2024-09-13T14:45:03.816 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.816 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.816 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.816 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880343+0000", 2024-09-13T14:45:03.817 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1199999999999997e-05 2024-09-13T14:45:03.817 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.817 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.817 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.817 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880458+0000", 2024-09-13T14:45:03.817 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011454 2024-09-13T14:45:03.817 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.817 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.817 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:45:03.817 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041479+0000", 2024-09-13T14:45:03.817 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.161021468 2024-09-13T14:45:03.817 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.817 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.818 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:45:03.818 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.044883+0000", 2024-09-13T14:45:03.818 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0034040139999999999 2024-09-13T14:45:03.818 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.818 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.818 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.818 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.044964+0000", 2024-09-13T14:45:03.818 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1007999999999994e-05 2024-09-13T14:45:03.818 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.818 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.818 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.818 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.045534+0000", 2024-09-13T14:45:03.818 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000569919 2024-09-13T14:45:03.818 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.819 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.819 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.819 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.045575+0000", 2024-09-13T14:45:03.819 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0642000000000002e-05 2024-09-13T14:45:03.819 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.819 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.819 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.819 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.045605+0000", 2024-09-13T14:45:03.819 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0131999999999999e-05 2024-09-13T14:45:03.819 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.819 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.819 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.819 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.820 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.820 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:35713 6.1 e150/145 6:8c5626c2:::benchmark_data_smithi096_167087_object35712:head v 150'2272, mlcod=150'2270)", 2024-09-13T14:45:03.820 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:23.880746+0000", 2024-09-13T14:45:03.820 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 39.861670580000002, 2024-09-13T14:45:03.820 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.164313708, 2024-09-13T14:45:03.820 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.820 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.820 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.820 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.820 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.820 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880746+0000", 2024-09-13T14:45:03.820 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.820 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.821 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.821 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.821 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880746+0000", 2024-09-13T14:45:03.821 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.821 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.821 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.821 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.821 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880745+0000", 2024-09-13T14:45:03.821 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:45:03.821 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.821 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.821 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.821 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.880827+0000", 2024-09-13T14:45:03.822 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1975e-05 2024-09-13T14:45:03.822 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.822 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.822 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.822 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041227+0000", 2024-09-13T14:45:03.822 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.160400195 2024-09-13T14:45:03.822 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.822 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.822 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.822 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041242+0000", 2024-09-13T14:45:03.822 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4313e-05 2024-09-13T14:45:03.822 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.822 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.823 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.823 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041327+0000", 2024-09-13T14:45:03.823 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5792999999999997e-05 2024-09-13T14:45:03.823 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.823 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.823 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.823 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041356+0000", 2024-09-13T14:45:03.823 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8659000000000001e-05 2024-09-13T14:45:03.823 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.823 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.823 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.823 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.045024+0000", 2024-09-13T14:45:03.823 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0036679080000000001 2024-09-13T14:45:03.823 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.824 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.824 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.824 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.045060+0000", 2024-09-13T14:45:03.824 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5803999999999998e-05 2024-09-13T14:45:03.824 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.824 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.824 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.824 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.824 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.824 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:35705 6.b e150/145 6:dda0ee43:::benchmark_data_smithi096_167087_object35704:head v 150'2256, mlcod=150'2252)", 2024-09-13T14:45:03.824 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:23.883018+0000", 2024-09-13T14:45:03.824 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 39.859398732000002, 2024-09-13T14:45:03.824 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16198644500000001, 2024-09-13T14:45:03.825 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.825 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.825 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.825 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.825 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.825 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.883018+0000", 2024-09-13T14:45:03.825 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.825 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.825 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.825 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.825 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.883018+0000", 2024-09-13T14:45:03.825 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.825 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.826 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.826 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.826 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.883017+0000", 2024-09-13T14:45:03.826 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:45:03.826 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.826 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.826 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.826 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:23.883076+0000", 2024-09-13T14:45:03.826 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8260000000000001e-05 2024-09-13T14:45:03.826 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.826 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.826 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.826 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041741+0000", 2024-09-13T14:45:03.826 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15866539499999999 2024-09-13T14:45:03.827 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.827 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.827 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.827 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041748+0000", 2024-09-13T14:45:03.827 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.968e-06 2024-09-13T14:45:03.827 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.827 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.827 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.827 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041862+0000", 2024-09-13T14:45:03.827 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000113501 2024-09-13T14:45:03.827 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.827 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.827 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.828 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.041881+0000", 2024-09-13T14:45:03.828 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9738e-05 2024-09-13T14:45:03.828 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.828 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.828 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.828 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.044969+0000", 2024-09-13T14:45:03.828 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0030877869999999998 2024-09-13T14:45:03.828 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.828 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.828 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.828 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.045004+0000", 2024-09-13T14:45:03.828 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5385000000000001e-05 2024-09-13T14:45:03.829 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.829 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.829 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.829 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.829 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.829 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:36153 6.8 6:10d61fe3:::benchmark_data_smithi096_167087_object36152:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:45:03.829 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:24.256244+0000", 2024-09-13T14:45:03.829 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 39.486172568000001, 2024-09-13T14:45:03.829 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.159125076, 2024-09-13T14:45:03.829 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.829 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.829 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:45:03.829 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:45:03.829 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:45:03.830 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 36153 2024-09-13T14:45:03.830 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.830 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.830 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.830 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.830 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.256244+0000", 2024-09-13T14:45:03.830 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.830 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.830 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.830 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.830 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.256244+0000", 2024-09-13T14:45:03.830 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.830 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.831 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.831 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.831 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.256244+0000", 2024-09-13T14:45:03.831 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:45:03.831 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.831 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.831 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.831 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.256266+0000", 2024-09-13T14:45:03.831 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1994e-05 2024-09-13T14:45:03.831 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.831 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.831 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.831 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.256270+0000", 2024-09-13T14:45:03.831 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5600000000000004e-06 2024-09-13T14:45:03.832 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.832 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.832 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.832 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.256273+0000", 2024-09-13T14:45:03.832 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.604e-06 2024-09-13T14:45:03.832 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.832 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.832 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.832 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.256308+0000", 2024-09-13T14:45:03.832 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4742e-05 2024-09-13T14:45:03.832 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.832 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.832 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.833 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.256412+0000", 2024-09-13T14:45:03.833 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000104188 2024-09-13T14:45:03.833 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.833 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.833 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:45:03.833 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.256528+0000", 2024-09-13T14:45:03.833 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011586699999999999 2024-09-13T14:45:03.833 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.833 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.833 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:45:03.833 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.258053+0000", 2024-09-13T14:45:03.833 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0015252569999999999 2024-09-13T14:45:03.833 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.834 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.834 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.834 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.265959+0000", 2024-09-13T14:45:03.834 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0079056260000000007 2024-09-13T14:45:03.834 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.834 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.834 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.834 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.415295+0000", 2024-09-13T14:45:03.834 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.14933611099999999 2024-09-13T14:45:03.834 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.834 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.834 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.834 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.415335+0000", 2024-09-13T14:45:03.834 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0630000000000002e-05 2024-09-13T14:45:03.835 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.835 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.835 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.835 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.415369+0000", 2024-09-13T14:45:03.835 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3883000000000001e-05 2024-09-13T14:45:03.835 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.835 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.835 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.835 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.835 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.835 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:36167 6.8 6:1100d541:::benchmark_data_smithi096_167087_object36166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:45:03.835 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:24.262660+0000", 2024-09-13T14:45:03.836 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 39.479756283999997, 2024-09-13T14:45:03.836 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.152904657, 2024-09-13T14:45:03.836 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.836 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.836 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:45:03.836 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:45:03.836 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:45:03.836 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 36167 2024-09-13T14:45:03.836 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.836 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.836 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.836 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.836 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.262660+0000", 2024-09-13T14:45:03.836 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.837 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.837 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.837 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.837 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.262660+0000", 2024-09-13T14:45:03.837 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.837 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.837 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.837 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.837 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.262660+0000", 2024-09-13T14:45:03.837 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:45:03.837 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.837 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.837 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.838 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.262712+0000", 2024-09-13T14:45:03.838 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2057e-05 2024-09-13T14:45:03.838 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.838 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.838 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.838 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.262723+0000", 2024-09-13T14:45:03.838 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1008e-05 2024-09-13T14:45:03.838 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.838 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.838 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.838 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.262726+0000", 2024-09-13T14:45:03.838 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8750000000000002e-06 2024-09-13T14:45:03.838 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.838 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.839 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.839 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.262763+0000", 2024-09-13T14:45:03.839 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6498999999999997e-05 2024-09-13T14:45:03.839 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.839 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.839 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.839 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.262868+0000", 2024-09-13T14:45:03.839 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000105544 2024-09-13T14:45:03.839 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.839 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.839 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:45:03.839 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.263002+0000", 2024-09-13T14:45:03.840 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013336899999999999 2024-09-13T14:45:03.840 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.840 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.840 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:45:03.840 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.264206+0000", 2024-09-13T14:45:03.840 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001204568 2024-09-13T14:45:03.840 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.840 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.840 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.840 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.413833+0000", 2024-09-13T14:45:03.840 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.149626747 2024-09-13T14:45:03.840 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.840 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.841 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:03.841 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.415480+0000", 2024-09-13T14:45:03.841 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0016465410000000001 2024-09-13T14:45:03.841 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.841 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.841 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.841 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.415532+0000", 2024-09-13T14:45:03.841 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.219e-05 2024-09-13T14:45:03.841 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.841 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.841 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.841 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:24.415565+0000", 2024-09-13T14:45:03.841 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3167999999999998e-05 2024-09-13T14:45:03.842 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.842 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.842 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.842 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.842 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.842 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:37471 6.9 e150/145 6:92dc42f2:::benchmark_data_smithi096_167087_object37470:head v 150'2391, mlcod=150'2389)", 2024-09-13T14:45:03.842 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:25.099939+0000", 2024-09-13T14:45:03.842 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 38.642477929999998, 2024-09-13T14:45:03.842 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16909158199999999, 2024-09-13T14:45:03.842 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.842 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.842 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.842 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.842 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.843 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.099939+0000", 2024-09-13T14:45:03.843 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.843 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.843 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.843 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.843 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.099939+0000", 2024-09-13T14:45:03.843 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.843 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.843 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.843 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.843 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.099938+0000", 2024-09-13T14:45:03.843 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:45:03.843 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.843 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.844 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.844 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.099998+0000", 2024-09-13T14:45:03.844 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9787999999999997e-05 2024-09-13T14:45:03.844 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.844 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.844 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.844 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100005+0000", 2024-09-13T14:45:03.844 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5900000000000002e-06 2024-09-13T14:45:03.844 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.844 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.844 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.844 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100009+0000", 2024-09-13T14:45:03.845 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5839999999999999e-06 2024-09-13T14:45:03.845 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.845 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.845 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.845 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100043+0000", 2024-09-13T14:45:03.845 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3726000000000002e-05 2024-09-13T14:45:03.845 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.845 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.845 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.845 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100060+0000", 2024-09-13T14:45:03.845 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6731999999999998e-05 2024-09-13T14:45:03.845 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.845 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.845 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.846 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.268975+0000", 2024-09-13T14:45:03.846 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16891590300000001 2024-09-13T14:45:03.846 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.846 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.846 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.846 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269030+0000", 2024-09-13T14:45:03.846 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4947000000000001e-05 2024-09-13T14:45:03.846 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.846 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.846 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.846 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.846 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.846 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:37473 6.6 e150/145 6:6dedaf54:::benchmark_data_smithi096_167087_object37472:head v 150'2324, mlcod=150'2322)", 2024-09-13T14:45:03.846 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:25.100530+0000", 2024-09-13T14:45:03.847 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 38.641886317999997, 2024-09-13T14:45:03.847 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.17121808399999999, 2024-09-13T14:45:03.847 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.847 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.847 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.847 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.847 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.847 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100530+0000", 2024-09-13T14:45:03.847 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.847 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.847 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.847 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.847 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100530+0000", 2024-09-13T14:45:03.848 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.848 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.848 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.848 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.848 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100530+0000", 2024-09-13T14:45:03.848 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:45:03.848 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.848 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.848 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.848 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100650+0000", 2024-09-13T14:45:03.848 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011967 2024-09-13T14:45:03.848 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.848 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.849 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.849 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100656+0000", 2024-09-13T14:45:03.849 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1380000000000004e-06 2024-09-13T14:45:03.849 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.849 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.849 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.849 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100659+0000", 2024-09-13T14:45:03.849 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9249999999999999e-06 2024-09-13T14:45:03.849 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.849 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.849 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.849 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100689+0000", 2024-09-13T14:45:03.849 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0114999999999999e-05 2024-09-13T14:45:03.850 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.850 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.850 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.850 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100704+0000", 2024-09-13T14:45:03.850 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5502000000000002e-05 2024-09-13T14:45:03.850 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.850 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.850 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.850 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271712+0000", 2024-09-13T14:45:03.850 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.17100812600000001 2024-09-13T14:45:03.850 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.850 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.850 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.851 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271748+0000", 2024-09-13T14:45:03.851 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6117e-05 2024-09-13T14:45:03.851 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.851 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.851 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.851 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.851 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.851 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:37477 6.b e150/145 6:d0ca37d0:::benchmark_data_smithi096_167087_object37476:head v 150'2365, mlcod=150'2363)", 2024-09-13T14:45:03.851 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:25.101869+0000", 2024-09-13T14:45:03.851 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 38.640547931999997, 2024-09-13T14:45:03.851 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.169716322, 2024-09-13T14:45:03.851 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.851 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.851 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.852 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.852 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.852 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.101869+0000", 2024-09-13T14:45:03.852 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.852 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.852 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.852 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.852 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.101869+0000", 2024-09-13T14:45:03.852 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.852 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.852 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.852 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.853 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.101868+0000", 2024-09-13T14:45:03.853 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:45:03.853 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.853 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.853 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.853 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.101944+0000", 2024-09-13T14:45:03.853 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6200999999999998e-05 2024-09-13T14:45:03.853 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.853 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.853 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.853 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269038+0000", 2024-09-13T14:45:03.853 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.167093726 2024-09-13T14:45:03.853 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.853 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.854 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.854 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269044+0000", 2024-09-13T14:45:03.854 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2729999999999999e-06 2024-09-13T14:45:03.854 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.854 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.854 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.854 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269122+0000", 2024-09-13T14:45:03.854 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7550000000000001e-05 2024-09-13T14:45:03.854 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.854 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.854 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.854 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269141+0000", 2024-09-13T14:45:03.854 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9026999999999999e-05 2024-09-13T14:45:03.855 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.855 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.855 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.855 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271551+0000", 2024-09-13T14:45:03.855 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0024101700000000001 2024-09-13T14:45:03.855 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.855 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.855 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.855 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271585+0000", 2024-09-13T14:45:03.855 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3859000000000002e-05 2024-09-13T14:45:03.855 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.855 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.855 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.856 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.856 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.856 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:37481 6.1 e150/145 6:86b51d11:::benchmark_data_smithi096_167087_object37480:head v 150'2387, mlcod=150'2385)", 2024-09-13T14:45:03.856 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:25.103719+0000", 2024-09-13T14:45:03.856 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 38.638697295999997, 2024-09-13T14:45:03.856 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16811326800000001, 2024-09-13T14:45:03.856 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:45:03.856 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:03.856 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:45:03.856 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.856 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:45:03.856 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.103719+0000", 2024-09-13T14:45:03.856 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.857 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.857 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.857 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:45:03.857 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.103719+0000", 2024-09-13T14:45:03.857 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:45:03.857 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.857 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.857 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:45:03.857 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.103719+0000", 2024-09-13T14:45:03.857 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:45:03.857 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.857 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.857 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:45:03.857 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.103768+0000", 2024-09-13T14:45:03.858 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9070000000000003e-05 2024-09-13T14:45:03.858 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.858 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.858 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:45:03.858 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.268962+0000", 2024-09-13T14:45:03.858 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.165193749 2024-09-13T14:45:03.858 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.858 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.858 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:45:03.858 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.268973+0000", 2024-09-13T14:45:03.858 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1504e-05 2024-09-13T14:45:03.858 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.858 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.858 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:45:03.859 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269062+0000", 2024-09-13T14:45:03.859 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8554999999999995e-05 2024-09-13T14:45:03.859 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.859 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.859 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:45:03.859 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269087+0000", 2024-09-13T14:45:03.859 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5772999999999999e-05 2024-09-13T14:45:03.859 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.859 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.859 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:45:03.859 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271798+0000", 2024-09-13T14:45:03.859 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0027102630000000001 2024-09-13T14:45:03.859 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:45:03.859 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:45:03.860 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:45:03.860 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271833+0000", 2024-09-13T14:45:03.860 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5077000000000003e-05 2024-09-13T14:45:03.860 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.860 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.860 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.860 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:45:03.860 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:45:03.860 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:45:03.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.895 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:03.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:03.931 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 55 16 87111 87095 98.9596 68.25 0.0266865 0.0100998 2024-09-13T14:45:03.996 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:04.033 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:04.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:04.135 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:04.143 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:04.196 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:04.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:04.298 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:04.344 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:04.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:04.446 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:04.482 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:04.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:04.505 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:04 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:04.505 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:04 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:04.505 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:04 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:04.505 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:04 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:04.585 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:04.612 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:04.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:04.714 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:04.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:04 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:04.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:04 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:04.754 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:04.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:04.757 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:45:04.858 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:04.863 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:45:04.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.906 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:04.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:04.931 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 56 15 88135 88120 98.3364 64.0625 0.0226585 0.0101635 2024-09-13T14:45:05.006 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:05.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.081 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:05.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:05.182 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:05.246 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:05.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:05.348 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:05.391 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:05.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:05.493 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:05.521 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:05.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:05.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:05 smithi096 ceph-mon[98901]: pgmap v450: 99 pgs: 99 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:45:05.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:05 smithi096 ceph-mon[102928]: pgmap v450: 99 pgs: 99 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:45:05.623 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:05.660 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:05.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:05 smithi204 ceph-mon[87071]: pgmap v450: 99 pgs: 99 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:45:05.762 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:05.805 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:05.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:05.907 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:05.931 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 57 16 89125 89109 97.6956 61.8125 0.00874289 0.0102297 2024-09-13T14:45:05.943 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:05.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:06.045 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:06.071 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:06.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:06.173 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:06.199 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:06.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:06.301 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:06.327 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:06.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:06.429 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:06.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.480 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:06.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:06.581 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:06.606 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:06.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:06.708 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:06.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.781 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:06.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:06.882 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:06.923 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:06.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:06.931 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 58 16 90158 90142 97.1243 64.5625 0.015626 0.0102906 2024-09-13T14:45:07.025 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:07.049 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:07.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:06 smithi096 ceph-mon[98901]: pgmap v451: 99 pgs: 99 active+clean; 5.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:45:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:07.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:07.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:06 smithi096 ceph-mon[102928]: pgmap v451: 99 pgs: 99 active+clean; 5.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:45:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:07.151 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:07.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.193 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:07.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:07.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:06 smithi204 ceph-mon[87071]: pgmap v451: 99 pgs: 99 active+clean; 5.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:45:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:07.295 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:07.346 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:07.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:07.448 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:07.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.476 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:07.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:07.577 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:07.604 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:07.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:07.706 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:07.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.735 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:07.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:07.837 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:07.866 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:07.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:07.932 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 59 16 91222 91206 96.6052 66.5 0.0204001 0.0103463 2024-09-13T14:45:07.968 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:07.994 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:07.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:08.096 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:08.124 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:08.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:08.213 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:45:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:45:07] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-13T14:45:08.226 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:08.263 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:08.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:08.366 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:08.391 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:08.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:08.493 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:08.520 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:08.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:08.622 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:08.664 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:08.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:08.766 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:08.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.812 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:08.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:08.848 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:45:08.848 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:45:08.884 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:45:08.884 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:45:08.913 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:08.917 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:45:08.917 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:45:08.932 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:45:08.932376+0000 min lat: 0.00106401 max lat: 0.202978 avg lat: 0.0104057 2024-09-13T14:45:08.932 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:45:08.932 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 60 16 92241 92225 96.0558 63.6875 0.0206198 0.0104057 2024-09-13T14:45:08.939 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:08.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:08.959 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:45:08.959 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:45:08.988 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:45:08.989 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:45:09.018 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:45:09.018 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:45:09.040 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:09.048 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:45:09.048 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:45:09.078 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:45:09.078 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:45:09.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.084 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:09.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:09.102 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:08 smithi096 ceph-mon[98901]: pgmap v452: 99 pgs: 99 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:45:09.102 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:08 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2403717637' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:09.102 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:08 smithi096 ceph-mon[102928]: pgmap v452: 99 pgs: 99 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:45:09.102 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:08 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2403717637' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:09.186 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:09.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:08 smithi204 ceph-mon[87071]: pgmap v452: 99 pgs: 99 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:45:09.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:08 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2403717637' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:09.221 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:09.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:09.323 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:09.348 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:09.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:09.451 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:09.472 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:09.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:09.574 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:09.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.605 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:09.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:09.706 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:09.732 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:09.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:09.834 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:09.863 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:09.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:09.932 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 61 16 93173 93157 95.436 58.25 0.0106952 0.0104731 2024-09-13T14:45:09.965 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:09.989 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:09.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:10.091 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:10.118 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:10.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:10.221 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:10.247 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:10.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:10.350 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:10.381 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:10.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:10.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:45:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:45:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:10.484 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:10.510 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:10.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:10.612 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:10.649 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:10.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:10.751 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:10.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.790 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:10.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:10.891 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:10.929 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:10.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:10.932 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 62 16 94172 94156 94.9037 62.4375 0.019633 0.0105319 2024-09-13T14:45:11.031 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:11.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.071 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:11.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:11.172 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:11.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.201 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:11.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:11.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:10 smithi204 ceph-mon[87071]: pgmap v453: 99 pgs: 99 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:45:11.303 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:11.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:10 smithi096 ceph-mon[98901]: pgmap v453: 99 pgs: 99 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:45:11.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:10 smithi096 ceph-mon[102928]: pgmap v453: 99 pgs: 99 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:45:11.328 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:11.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:11.430 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:11.457 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:11.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:11.559 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:11.595 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:11.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:11.698 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:11.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.748 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:11.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:11.849 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:11.868 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:11.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.902 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:11.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:11.933 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 63 16 95147 95131 94.3642 60.9375 0.0250503 0.0105894 2024-09-13T14:45:12.003 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:12.056 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:12.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:12.158 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:12.181 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:12.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:12.189 INFO:teuthology.orchestra.run.smithi096.stdout:moved 0 / 297 (0%) 2024-09-13T14:45:12.189 INFO:teuthology.orchestra.run.smithi096.stdout:avg 42.4286 2024-09-13T14:45:12.190 INFO:teuthology.orchestra.run.smithi096.stdout:stddev 3.99489 -> 3.99489 (expected baseline 6.03053) 2024-09-13T14:45:12.190 INFO:teuthology.orchestra.run.smithi096.stdout:min osd.5 with 37 -> 37 pgs (0.872054 -> 0.872054 * mean) 2024-09-13T14:45:12.190 INFO:teuthology.orchestra.run.smithi096.stdout:max osd.3 with 50 -> 50 pgs (1.17845 -> 1.17845 * mean) 2024-09-13T14:45:12.190 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:45:12.190 INFO:teuthology.orchestra.run.smithi096.stdout:oload 1000 2024-09-13T14:45:12.190 INFO:teuthology.orchestra.run.smithi096.stdout:max_change 0.05 2024-09-13T14:45:12.190 INFO:teuthology.orchestra.run.smithi096.stdout:max_change_osds 4 2024-09-13T14:45:12.190 INFO:teuthology.orchestra.run.smithi096.stdout:average_utilization 0.0296 2024-09-13T14:45:12.190 INFO:teuthology.orchestra.run.smithi096.stdout:overload_utilization 0.2961 2024-09-13T14:45:12.190 INFO:teuthology.orchestra.run.smithi096.stderr:no change 2024-09-13T14:45:12.193 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:12.283 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:12.310 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:12.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:12.412 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:12.457 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:12.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:12.559 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:12.585 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:12.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:12.687 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:12.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.857 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:12.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:12.933 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 64 16 96088 96072 93.8087 58.8125 0.0116532 0.0106543 2024-09-13T14:45:12.958 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:12.990 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:12.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:13.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:12 smithi096 ceph-mon[98901]: pgmap v454: 99 pgs: 99 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T14:45:13.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:12 smithi096 ceph-mon[98901]: from='client.45245 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:45:13.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:12 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:45:13.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:12 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:45:13.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:12 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:13.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:12 smithi096 ceph-mon[102928]: pgmap v454: 99 pgs: 99 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T14:45:13.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:12 smithi096 ceph-mon[102928]: from='client.45245 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:45:13.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:12 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:45:13.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:12 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:45:13.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:12 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:13.092 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:13.120 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:13.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:13.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:12 smithi204 ceph-mon[87071]: pgmap v454: 99 pgs: 99 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T14:45:13.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:12 smithi204 ceph-mon[87071]: from='client.45245 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:45:13.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:12 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:45:13.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:12 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:45:13.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:12 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:13.222 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:13.244 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:13.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:13.345 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:13.379 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:13.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:13.481 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:13.512 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:13.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:13.598 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:45:13.614 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:13.639 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:13.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:13.741 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:13.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.784 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:13.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:13.885 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:13.934 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:13.934 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 65 16 97010 96994 93.2511 57.625 0.0171393 0.0107183 2024-09-13T14:45:13.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:14.036 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:14.079 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:14.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:14.181 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:14.219 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:14.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:14.321 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:14.359 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:14.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:14.461 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:14.482 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:14.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:14.584 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:14.611 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:14.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:14.713 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:14.740 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:14.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:14.842 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:14.869 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:14.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:14.933 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 66 16 97948 97932 92.7264 58.625 0.0161039 0.0107794 2024-09-13T14:45:14.971 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:14.996 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:14.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:15.098 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:15.124 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:15.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:15.142 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:14 smithi096 ceph-mon[98901]: pgmap v455: 99 pgs: 99 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s 2024-09-13T14:45:15.142 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:14 smithi096 ceph-mon[102928]: pgmap v455: 99 pgs: 99 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s 2024-09-13T14:45:15.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:14 smithi204 ceph-mon[87071]: pgmap v455: 99 pgs: 99 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s 2024-09-13T14:45:15.226 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:15.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.266 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:15.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:15.367 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:15.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.393 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:15.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:15.494 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:15.519 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:15.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:15.620 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:15.646 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:15.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:15.748 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:15.774 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:15.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:15.876 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:15.900 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:15.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:15.933 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 67 16 98943 98927 92.2703 62.1875 0.0235862 0.0108321 2024-09-13T14:45:16.002 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:16.028 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:16.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:16.131 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:16.172 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:16.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:16.274 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:16.308 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:16.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:16.410 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:16.455 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:16.557 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:16.581 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:16.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:16.603 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-13T14:45:16.603 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 110 0.05 2024-09-13T14:45:16.683 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:16.706 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:16.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:16.807 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:16.830 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:16.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:16.932 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:16.934 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 68 15 99896 99881 91.7896 59.625 0.0149209 0.0108895 2024-09-13T14:45:16.958 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:17.060 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:17.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:16 smithi096 ceph-mon[98901]: pgmap v456: 99 pgs: 99 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 55 MiB/s wr, 872 op/s 2024-09-13T14:45:17.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:16 smithi096 ceph-mon[102928]: pgmap v456: 99 pgs: 99 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 55 MiB/s wr, 872 op/s 2024-09-13T14:45:17.086 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:17.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:17.188 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:17.212 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:17.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:16 smithi204 ceph-mon[87071]: pgmap v456: 99 pgs: 99 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 55 MiB/s wr, 872 op/s 2024-09-13T14:45:17.314 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:17.356 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:17.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:17.458 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:17.492 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:17.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:17.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:17.628 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:17.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:17.730 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:17.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.776 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:17.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:17.877 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:17.936 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 69 16 100857 100841 91.3266 60 0.0149965 0.0109433 2024-09-13T14:45:17.947 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:17.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:18.049 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:18.089 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:18.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:18.191 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:18.229 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:18.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:18.271 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:45:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:45:17] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-13T14:45:18.335 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:18.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.373 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:18.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:18.397 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:18.438 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:45:18.439 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:45:18.474 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:18.503 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:18.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:18.605 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:18.630 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:18.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:18.732 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:18.773 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:18.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:18.875 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:18.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.903 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:18.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:18.922 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:18.936 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 70 16 101750 101734 90.8192 55.8125 0.0193229 0.0110051 2024-09-13T14:45:19.005 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:19.032 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:19.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:19.134 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:19.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.161 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:19.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:19.195 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:18 smithi096 ceph-mon[98901]: pgmap v457: 99 pgs: 99 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:45:19.195 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:18 smithi096 ceph-mon[102928]: pgmap v457: 99 pgs: 99 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:45:19.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:18 smithi204 ceph-mon[87071]: pgmap v457: 99 pgs: 99 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:45:19.263 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:19.307 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:19.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:19.409 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:19.437 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:19.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:19.539 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:19.577 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:19.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:19.679 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:19.718 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:19.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:19.752 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:45:19.823 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:19.867 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:45:19.897 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:19.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:19.936 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 71 15 102680 102665 90.3596 58.1875 0.0178704 0.0110609 2024-09-13T14:45:19.962 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:19 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/294530527' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:19.962 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:19 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:19.963 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:19 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/294530527' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:19.963 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:19 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:20.000 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:20.049 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:20.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:20.119 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:19 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/294530527' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:20.119 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:19 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:20.151 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:20.178 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:20.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:20.280 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:20.310 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:20.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:20.411 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:20.448 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:20.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:20.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:45:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:45:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:20.550 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:20.580 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:20.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:20.682 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:20.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.721 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:20.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:20.823 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:20.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.855 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:20.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:20.936 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 72 16 103650 103634 89.9457 60.5625 0.00939635 0.011113 2024-09-13T14:45:20.956 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:20.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.994 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:20.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:21.095 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:21.121 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:21.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:21.143 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:20 smithi096 ceph-mon[98901]: pgmap v458: 99 pgs: 99 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 55 MiB/s wr, 887 op/s 2024-09-13T14:45:21.143 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:20 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:21.143 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:20 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:45:21.144 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:20 smithi096 ceph-mon[98901]: osdmap e156: 8 total, 8 up, 7 in 2024-09-13T14:45:21.144 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:20 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:21.144 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:20 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:21.144 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:20 smithi096 ceph-mon[102928]: pgmap v458: 99 pgs: 99 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 55 MiB/s wr, 887 op/s 2024-09-13T14:45:21.144 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:20 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:21.144 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:20 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:45:21.144 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:20 smithi096 ceph-mon[102928]: osdmap e156: 8 total, 8 up, 7 in 2024-09-13T14:45:21.144 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:20 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:21.144 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:20 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:21.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:20 smithi204 ceph-mon[87071]: pgmap v458: 99 pgs: 99 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 55 MiB/s wr, 887 op/s 2024-09-13T14:45:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:20 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:20 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:45:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:20 smithi204 ceph-mon[87071]: osdmap e156: 8 total, 8 up, 7 in 2024-09-13T14:45:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:20 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:20 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:21.223 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:21.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.282 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:21.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:21.383 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:21.427 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:21.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:21.530 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:21.559 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:21.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:21.661 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:21.698 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:21.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:21.800 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:21.827 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:21.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:21.929 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:21.936 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 73 16 104617 104601 89.5414 60.4375 0.0156162 0.0111625 2024-09-13T14:45:21.966 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:21.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:22.068 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:22.094 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:22.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:22.197 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:22.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.249 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:22.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:22.350 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:22.378 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:22.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:22.481 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:22.507 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:22.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:22.609 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:22.634 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:22.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:22.736 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:22.765 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:22.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:22.852 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:22.867 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:22.929 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:22.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:22.937 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 74 16 105572 105556 89.1373 59.6875 0.0186582 0.0112132 2024-09-13T14:45:23.032 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:23.074 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:23.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:23.144 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:45:23.175 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:23.195 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:22 smithi096 ceph-mon[98901]: pgmap v460: 99 pgs: 99 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 61 MiB/s wr, 968 op/s 2024-09-13T14:45:23.195 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:22 smithi096 ceph-mon[102928]: pgmap v460: 99 pgs: 99 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 61 MiB/s wr, 968 op/s 2024-09-13T14:45:23.203 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:23.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:23.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:22 smithi204 ceph-mon[87071]: pgmap v460: 99 pgs: 99 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 61 MiB/s wr, 968 op/s 2024-09-13T14:45:23.305 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:23.329 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:23.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:23.433 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:23.457 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:23.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:23.559 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:23.586 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:23.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:23.688 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:23.711 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:23.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:23.813 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:23.838 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:23.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:23.937 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 75 16 106431 106415 88.6646 53.6875 0.0154099 0.0112723 2024-09-13T14:45:23.940 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:23.966 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:23.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:24.068 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:24.094 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:24.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:24.196 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:24.220 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:24.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:24.300 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:45:24.300 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:45:24.322 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:24.338 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:45:24.338 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:45:24.347 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:24.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:24.374 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:45:24.374 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:45:24.405 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:45:24.405 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:45:24.435 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:45:24.436 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:45:24.449 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:24.465 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:45:24.466 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:45:24.490 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:24.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:24.504 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:45:24.504 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:45:24.537 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:45:24.538 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:45:24.592 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:24.625 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:24.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:24.727 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:24.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.770 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:24.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:24.871 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:24.893 INFO:teuthology.orchestra.run.smithi096.stdout:moved 3 / 297 (1.0101%) 2024-09-13T14:45:24.893 INFO:teuthology.orchestra.run.smithi096.stdout:avg 42.4286 2024-09-13T14:45:24.893 INFO:teuthology.orchestra.run.smithi096.stdout:stddev 3.99489 -> 4.33778 (expected baseline 6.03053) 2024-09-13T14:45:24.893 INFO:teuthology.orchestra.run.smithi096.stdout:min osd.5 with 37 -> 37 pgs (0.872054 -> 0.872054 * mean) 2024-09-13T14:45:24.893 INFO:teuthology.orchestra.run.smithi096.stdout:max osd.3 with 50 -> 50 pgs (1.17845 -> 1.17845 * mean) 2024-09-13T14:45:24.893 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:45:24.893 INFO:teuthology.orchestra.run.smithi096.stdout:oload 110 2024-09-13T14:45:24.893 INFO:teuthology.orchestra.run.smithi096.stdout:max_change 0.05 2024-09-13T14:45:24.893 INFO:teuthology.orchestra.run.smithi096.stdout:max_change_osds 4 2024-09-13T14:45:24.893 INFO:teuthology.orchestra.run.smithi096.stdout:average_utilization 0.0333 2024-09-13T14:45:24.893 INFO:teuthology.orchestra.run.smithi096.stdout:overload_utilization 0.0366 2024-09-13T14:45:24.893 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2 weight 1.0000 -> 0.9500 2024-09-13T14:45:24.894 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3 weight 1.0000 -> 0.9500 2024-09-13T14:45:24.894 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7 weight 1.0000 -> 0.9500 2024-09-13T14:45:24.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.913 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:24.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:24.937 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 76 16 107328 107312 88.2356 56.0625 0.0138862 0.0113279 2024-09-13T14:45:25.014 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:25.072 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:25.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:25.174 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:25.186 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:24 smithi096 ceph-mon[98901]: pgmap v461: 99 pgs: 99 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-13T14:45:25.186 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:24 smithi096 ceph-mon[98901]: from='client.35379 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:45:25.186 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:24 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260','7':'62260'}"} : dispatch 2024-09-13T14:45:25.186 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:24 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4002245186' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:25.186 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:24 smithi096 ceph-mon[102928]: pgmap v461: 99 pgs: 99 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-13T14:45:25.186 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:24 smithi096 ceph-mon[102928]: from='client.35379 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:45:25.186 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:24 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260','7':'62260'}"} : dispatch 2024-09-13T14:45:25.187 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:24 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4002245186' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:24 smithi204 ceph-mon[87071]: pgmap v461: 99 pgs: 99 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-13T14:45:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:24 smithi204 ceph-mon[87071]: from='client.35379 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:45:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:24 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260','7':'62260'}"} : dispatch 2024-09-13T14:45:25.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:24 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4002245186' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:25.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.221 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:25.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:25.322 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:25.347 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:25.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:25.449 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:25.489 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:25.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:25.590 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:25.619 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:25.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:25.720 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:25.760 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:25.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:25.862 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:25.882 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-13T14:45:25.883 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 110 0.05 2024-09-13T14:45:25.886 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:25.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:25.939 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 77 16 108236 108220 87.8242 56.75 0.0318793 0.0113801 2024-09-13T14:45:25.988 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:26.013 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:26.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:26.115 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:26.154 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:26.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:26.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:25 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260','7':'62260'}"}]': finished 2024-09-13T14:45:26.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:25 smithi204 ceph-mon[87071]: osdmap e157: 8 total, 8 up, 7 in 2024-09-13T14:45:26.256 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:26.293 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:25 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260','7':'62260'}"}]': finished 2024-09-13T14:45:26.293 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:25 smithi096 ceph-mon[98901]: osdmap e157: 8 total, 8 up, 7 in 2024-09-13T14:45:26.294 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:25 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260','7':'62260'}"}]': finished 2024-09-13T14:45:26.294 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:25 smithi096 ceph-mon[102928]: osdmap e157: 8 total, 8 up, 7 in 2024-09-13T14:45:26.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.319 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:26.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:26.420 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:26.447 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:26.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:26.549 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:26.618 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:26.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:26.719 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:26.736 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:26.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.771 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:26.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:26.872 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:26.891 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:45:26.892 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:45:26.892 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:45:26.892 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:45:26.892 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:45:26.932 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:26.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:26.939 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 78 16 109083 109067 87.3768 52.9375 0.0233226 0.0114383 2024-09-13T14:45:27.034 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:27.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:26 smithi096 ceph-mon[98901]: pgmap v463: 99 pgs: 99 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 44 MiB/s wr, 699 op/s 2024-09-13T14:45:27.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:26 smithi096 ceph-mon[98901]: osdmap e158: 8 total, 8 up, 7 in 2024-09-13T14:45:27.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:26 smithi096 ceph-mon[102928]: pgmap v463: 99 pgs: 99 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 44 MiB/s wr, 699 op/s 2024-09-13T14:45:27.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:26 smithi096 ceph-mon[102928]: osdmap e158: 8 total, 8 up, 7 in 2024-09-13T14:45:27.083 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:27.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:27.185 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:27.212 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:27.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:27.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:26 smithi204 ceph-mon[87071]: pgmap v463: 99 pgs: 99 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 44 MiB/s wr, 699 op/s 2024-09-13T14:45:27.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:26 smithi204 ceph-mon[87071]: osdmap e158: 8 total, 8 up, 7 in 2024-09-13T14:45:27.315 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:27.341 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:27.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:27.443 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:27.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.481 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:27.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:27.581 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:27.608 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:27.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:27.710 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:27.749 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:27.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:27.775 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:27.852 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:27.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.911 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:27.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:27.940 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 79 16 110031 110015 87.02 59.25 0.0149335 0.0114862 2024-09-13T14:45:28.012 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:28.038 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:28.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:28.139 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:28.169 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:28.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:28.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:27 smithi204 ceph-mon[87071]: 1.2 scrub starts 2024-09-13T14:45:28.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:27 smithi204 ceph-mon[87071]: osdmap e159: 8 total, 8 up, 7 in 2024-09-13T14:45:28.219 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:27 smithi096 ceph-mon[98901]: 1.2 scrub starts 2024-09-13T14:45:28.219 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:27 smithi096 ceph-mon[98901]: osdmap e159: 8 total, 8 up, 7 in 2024-09-13T14:45:28.219 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:27 smithi096 ceph-mon[102928]: 1.2 scrub starts 2024-09-13T14:45:28.219 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:27 smithi096 ceph-mon[102928]: osdmap e159: 8 total, 8 up, 7 in 2024-09-13T14:45:28.220 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:45:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:45:27] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.51.0" 2024-09-13T14:45:28.260 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:45:28.271 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:28.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:45:28.354 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:28.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:28.457 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:28.481 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:28.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:28.583 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:28.605 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:28.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:28.707 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:28.735 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:28.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:28.837 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:28.901 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:28.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:28.942 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:45:28.941955+0000 min lat: 0.00106401 max lat: 0.202978 avg lat: 0.011541 2024-09-13T14:45:28.942 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:45:28.942 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 80 16 110894 110878 86.605 53.9375 0.0223053 0.011541 2024-09-13T14:45:29.002 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:29.045 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:29.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:29.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:28 smithi096 ceph-mon[98901]: pgmap v466: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:45:29.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:28 smithi096 ceph-mon[98901]: osdmap e160: 8 total, 8 up, 7 in 2024-09-13T14:45:29.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:28 smithi096 ceph-mon[102928]: pgmap v466: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:45:29.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:28 smithi096 ceph-mon[102928]: osdmap e160: 8 total, 8 up, 7 in 2024-09-13T14:45:29.147 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:29.171 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:29.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:29.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:28 smithi204 ceph-mon[87071]: pgmap v466: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:45:29.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:28 smithi204 ceph-mon[87071]: osdmap e160: 8 total, 8 up, 7 in 2024-09-13T14:45:29.273 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:29.298 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:29.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:29.400 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:29.426 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:29.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:29.527 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:29.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.573 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:29.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:29.674 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:29.714 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:29.718 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:29.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:29.821 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:29.847 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:29.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:29.942 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 81 16 111828 111812 86.2565 58.375 0.0193339 0.0115885 2024-09-13T14:45:29.948 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:29.974 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:29.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:30.076 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:30.104 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:30.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:30.206 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:30.251 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:30.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:30.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:29 smithi096 ceph-mon[98901]: osdmap e161: 8 total, 8 up, 7 in 2024-09-13T14:45:30.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:29 smithi096 ceph-mon[102928]: osdmap e161: 8 total, 8 up, 7 in 2024-09-13T14:45:30.353 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:30.380 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:30.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:30.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:29 smithi204 ceph-mon[87071]: osdmap e161: 8 total, 8 up, 7 in 2024-09-13T14:45:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:45:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:45:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:30.482 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:30.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.525 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:30.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:30.626 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:30.650 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:30.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:30.753 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:30.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.810 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:30.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:30.840 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:30.911 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:30.935 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:30.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:30.942 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 82 16 112673 112657 85.8484 52.8125 0.0184118 0.0116437 2024-09-13T14:45:31.037 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:31.094 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:31.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:31.167 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:30 smithi096 ceph-mon[98901]: pgmap v469: 99 pgs: 99 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 17 KiB/s rd, 70 MiB/s wr, 1.14k op/s; 13 KiB/s, 0 objects/s recovering 2024-09-13T14:45:31.167 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:30 smithi096 ceph-mon[102928]: pgmap v469: 99 pgs: 99 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 17 KiB/s rd, 70 MiB/s wr, 1.14k op/s; 13 KiB/s, 0 objects/s recovering 2024-09-13T14:45:31.196 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:31.242 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:31.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:31.344 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:31.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.384 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:31.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:31.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:30 smithi204 ceph-mon[87071]: pgmap v469: 99 pgs: 99 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 17 KiB/s rd, 70 MiB/s wr, 1.14k op/s; 13 KiB/s, 0 objects/s recovering 2024-09-13T14:45:31.485 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:31.510 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:31.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:31.611 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:31.635 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:31.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:31.736 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:31.759 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:31.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:31.861 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:31.892 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:31.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:31.942 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 83 16 113408 113392 85.3675 45.9375 0.0217139 0.0117079 2024-09-13T14:45:31.990 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:45:31.994 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:32.038 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:32 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3384307927' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:32.038 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:32 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:32.038 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:31 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3384307927' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:32.039 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:31 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:32.044 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:32.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:32.147 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:32.171 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:32.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:32.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:31 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3384307927' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:32.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:31 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:32.273 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:32.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.318 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:32.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:32.420 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:32.444 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:32.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:32.546 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:32.574 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:32.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:32.677 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:32.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.718 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:32.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:32.819 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:32.855 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:32.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:32.942 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 84 16 114168 114152 84.9167 47.5 0.0240414 0.0117699 2024-09-13T14:45:32.957 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:32.981 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:32.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:33.011 INFO:teuthology.orchestra.run.smithi096.stdout:moved 5 / 297 (1.6835%) 2024-09-13T14:45:33.011 INFO:teuthology.orchestra.run.smithi096.stdout:avg 42.4286 2024-09-13T14:45:33.011 INFO:teuthology.orchestra.run.smithi096.stdout:stddev 4.33778 -> 4.37059 (expected baseline 6.03053) 2024-09-13T14:45:33.011 INFO:teuthology.orchestra.run.smithi096.stdout:min osd.5 with 37 -> 38 pgs (0.872054 -> 0.895623 * mean) 2024-09-13T14:45:33.011 INFO:teuthology.orchestra.run.smithi096.stdout:max osd.3 with 50 -> 48 pgs (1.17845 -> 1.13131 * mean) 2024-09-13T14:45:33.011 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:45:33.012 INFO:teuthology.orchestra.run.smithi096.stdout:oload 110 2024-09-13T14:45:33.012 INFO:teuthology.orchestra.run.smithi096.stdout:max_change 0.05 2024-09-13T14:45:33.012 INFO:teuthology.orchestra.run.smithi096.stdout:max_change_osds 4 2024-09-13T14:45:33.012 INFO:teuthology.orchestra.run.smithi096.stdout:average_utilization 0.0355 2024-09-13T14:45:33.012 INFO:teuthology.orchestra.run.smithi096.stdout:overload_utilization 0.0391 2024-09-13T14:45:33.012 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2 weight 0.9500 -> 0.9000 2024-09-13T14:45:33.012 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3 weight 0.9500 -> 0.9000 2024-09-13T14:45:33.012 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7 weight 0.9500 -> 0.9000 2024-09-13T14:45:33.084 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:33.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.127 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:33.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:33.228 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:33.250 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:33.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:33.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:33 smithi096 ceph-mon[102928]: pgmap v470: 99 pgs: 99 active+clean; 6.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 12 KiB/s rd, 62 MiB/s wr, 1.01k op/s; 9.0 KiB/s, 0 objects/s recovering 2024-09-13T14:45:33.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:33 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:33.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:33 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:33.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:33 smithi096 ceph-mon[102928]: osdmap e162: 8 total, 8 up, 7 in 2024-09-13T14:45:33.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:33 smithi096 ceph-mon[102928]: from='client.35391 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:45:33.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'2':'58984','3':'58984','7':'58984'}"} : dispatch 2024-09-13T14:45:33.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:33 smithi096 ceph-mon[98901]: pgmap v470: 99 pgs: 99 active+clean; 6.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 12 KiB/s rd, 62 MiB/s wr, 1.01k op/s; 9.0 KiB/s, 0 objects/s recovering 2024-09-13T14:45:33.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:33 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:33.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:33 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:33.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:33 smithi096 ceph-mon[98901]: osdmap e162: 8 total, 8 up, 7 in 2024-09-13T14:45:33.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:33 smithi096 ceph-mon[98901]: from='client.35391 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:45:33.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'2':'58984','3':'58984','7':'58984'}"} : dispatch 2024-09-13T14:45:33.352 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:33.373 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:33.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:33 smithi204 ceph-mon[87071]: pgmap v470: 99 pgs: 99 active+clean; 6.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 12 KiB/s rd, 62 MiB/s wr, 1.01k op/s; 9.0 KiB/s, 0 objects/s recovering 2024-09-13T14:45:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:33 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:33 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:33 smithi204 ceph-mon[87071]: osdmap e162: 8 total, 8 up, 7 in 2024-09-13T14:45:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:33 smithi204 ceph-mon[87071]: from='client.35391 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:45:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'2':'58984','3':'58984','7':'58984'}"} : dispatch 2024-09-13T14:45:33.475 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:33.496 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:33.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:33.598 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:33.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.621 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:33.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:33.722 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:33.745 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:33.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:33.847 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:33.869 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:33.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:33.942 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 85 16 114364 114348 84.0619 12.25 0.00348551 0.0117777 2024-09-13T14:45:33.971 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:33.993 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:33.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:34.095 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:34.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.117 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:34.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:34.218 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:34.241 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:34.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:34.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:34 smithi096 ceph-mon[102928]: 1.2 scrub starts 2024-09-13T14:45:34.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:34 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'58984','3':'58984','7':'58984'}"}]': finished 2024-09-13T14:45:34.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:34 smithi096 ceph-mon[102928]: osdmap e163: 8 total, 8 up, 7 in 2024-09-13T14:45:34.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:34 smithi096 ceph-mon[98901]: 1.2 scrub starts 2024-09-13T14:45:34.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:34 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'58984','3':'58984','7':'58984'}"}]': finished 2024-09-13T14:45:34.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:34 smithi096 ceph-mon[98901]: osdmap e163: 8 total, 8 up, 7 in 2024-09-13T14:45:34.343 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:34.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.365 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:34.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:34.466 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:34.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:34 smithi204 ceph-mon[87071]: 1.2 scrub starts 2024-09-13T14:45:34.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:34 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'58984','3':'58984','7':'58984'}"}]': finished 2024-09-13T14:45:34.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:34 smithi204 ceph-mon[87071]: osdmap e163: 8 total, 8 up, 7 in 2024-09-13T14:45:34.488 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:34.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:34.590 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:34.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.613 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:34.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:34.714 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:34.738 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:34.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:34.840 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:34.863 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:34.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:34.942 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 86 16 114364 114348 83.0845 0 - 0.0117777 2024-09-13T14:45:34.965 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:34.986 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:34.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:35.087 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:35.109 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:35.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:35.210 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:35.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.234 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:35.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:35.309 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:35 smithi096 ceph-mon[98901]: pgmap v473: 99 pgs: 2 peering, 97 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 12 KiB/s rd, 52 MiB/s wr, 833 op/s; 9.0 KiB/s, 0 objects/s recovering 2024-09-13T14:45:35.309 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:35 smithi096 ceph-mon[98901]: osdmap e164: 8 total, 8 up, 7 in 2024-09-13T14:45:35.310 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:35 smithi096 ceph-mon[102928]: pgmap v473: 99 pgs: 2 peering, 97 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 12 KiB/s rd, 52 MiB/s wr, 833 op/s; 9.0 KiB/s, 0 objects/s recovering 2024-09-13T14:45:35.310 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:35 smithi096 ceph-mon[102928]: osdmap e164: 8 total, 8 up, 7 in 2024-09-13T14:45:35.335 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:35.358 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:35.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:35.460 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:35.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:35 smithi204 ceph-mon[87071]: pgmap v473: 99 pgs: 2 peering, 97 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 12 KiB/s rd, 52 MiB/s wr, 833 op/s; 9.0 KiB/s, 0 objects/s recovering 2024-09-13T14:45:35.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:35 smithi204 ceph-mon[87071]: osdmap e164: 8 total, 8 up, 7 in 2024-09-13T14:45:35.482 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:35.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:35.584 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:35.624 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:35.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:35.726 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:35.751 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:35.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:35.853 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:35.877 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:35.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:35.943 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 87 16 114722 114706 82.3868 11.1875 0.0170874 0.0121316 2024-09-13T14:45:35.980 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:36.001 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:45:36.017 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:36.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:36.032 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:45:36.032 INFO:teuthology.orchestra.run.smithi096.stdout: "size": 20, 2024-09-13T14:45:36.032 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 600, 2024-09-13T14:45:36.032 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:45:36.032 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.032 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:68079 5.3 5:cc82a13d:::benchmark_data_smithi096_138841_object68078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:45:36.032 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.517423+0000", 2024-09-13T14:45:36.032 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 310.29705476599997, 2024-09-13T14:45:36.032 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16008386699999999, 2024-09-13T14:45:36.032 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.032 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.033 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.033 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:45:36.033 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:45:36.033 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:45:36.033 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 68079 2024-09-13T14:45:36.033 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.033 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.033 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.033 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.033 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517423+0000", 2024-09-13T14:45:36.033 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.033 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.034 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.034 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.034 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517423+0000", 2024-09-13T14:45:36.034 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.034 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.034 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.034 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.034 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517424+0000", 2024-09-13T14:45:36.034 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1540000000000001e-06 2024-09-13T14:45:36.034 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.034 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.034 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.034 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517443+0000", 2024-09-13T14:45:36.034 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8576e-05 2024-09-13T14:45:36.035 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.035 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.035 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.035 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517452+0000", 2024-09-13T14:45:36.035 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.4329999999999993e-06 2024-09-13T14:45:36.035 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.035 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.035 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.035 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517458+0000", 2024-09-13T14:45:36.035 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9030000000000001e-06 2024-09-13T14:45:36.035 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.035 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.035 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.036 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.518942+0000", 2024-09-13T14:45:36.036 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0014841469999999999 2024-09-13T14:45:36.036 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.036 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.036 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.036 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519105+0000", 2024-09-13T14:45:36.036 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000162495 2024-09-13T14:45:36.036 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.036 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.036 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 3,5", 2024-09-13T14:45:36.036 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519231+0000", 2024-09-13T14:45:36.036 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012689 2024-09-13T14:45:36.037 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.037 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.037 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.037 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.521772+0000", 2024-09-13T14:45:36.037 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.002540212 2024-09-13T14:45:36.037 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.037 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.037 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:45:36.037 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.521866+0000", 2024-09-13T14:45:36.037 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3863999999999994e-05 2024-09-13T14:45:36.037 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.037 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.037 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.037 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677396+0000", 2024-09-13T14:45:36.038 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.155530265 2024-09-13T14:45:36.038 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.038 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.038 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.038 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677458+0000", 2024-09-13T14:45:36.038 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1790000000000003e-05 2024-09-13T14:45:36.038 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.038 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.038 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.038 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677507+0000", 2024-09-13T14:45:36.038 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9138000000000003e-05 2024-09-13T14:45:36.038 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.038 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.039 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.039 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.039 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.039 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:68081 5.5 5:a2088a5b:::benchmark_data_smithi096_138841_object68080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:45:36.039 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.517472+0000", 2024-09-13T14:45:36.039 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 310.29700531899999, 2024-09-13T14:45:36.039 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.164906097, 2024-09-13T14:45:36.039 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.039 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.039 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.039 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:45:36.039 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:45:36.039 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:45:36.039 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 68081 2024-09-13T14:45:36.040 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.040 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.040 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.040 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.040 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517472+0000", 2024-09-13T14:45:36.040 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.040 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.040 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.040 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.040 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517472+0000", 2024-09-13T14:45:36.040 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.040 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.040 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.040 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.041 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517473+0000", 2024-09-13T14:45:36.041 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7300000000000001e-07 2024-09-13T14:45:36.041 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.041 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.041 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.041 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517489+0000", 2024-09-13T14:45:36.041 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6595999999999998e-05 2024-09-13T14:45:36.041 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.041 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.041 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.041 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517496+0000", 2024-09-13T14:45:36.041 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6590000000000001e-06 2024-09-13T14:45:36.041 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.041 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.042 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.042 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517501+0000", 2024-09-13T14:45:36.042 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8110000000000002e-06 2024-09-13T14:45:36.042 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.042 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.042 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.042 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517713+0000", 2024-09-13T14:45:36.042 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00021189200000000001 2024-09-13T14:45:36.042 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.042 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.042 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.042 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517828+0000", 2024-09-13T14:45:36.042 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000115171 2024-09-13T14:45:36.042 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.043 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.043 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:45:36.043 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517950+0000", 2024-09-13T14:45:36.043 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012224499999999999 2024-09-13T14:45:36.043 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.043 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.043 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:45:36.043 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.681738+0000", 2024-09-13T14:45:36.043 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.163787502 2024-09-13T14:45:36.043 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.043 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.043 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.043 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.681777+0000", 2024-09-13T14:45:36.043 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9407999999999999e-05 2024-09-13T14:45:36.044 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.044 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.044 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.044 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.682179+0000", 2024-09-13T14:45:36.044 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00040160599999999999 2024-09-13T14:45:36.044 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.044 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.044 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.044 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.682331+0000", 2024-09-13T14:45:36.044 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000152631 2024-09-13T14:45:36.044 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.044 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.044 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.044 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.682378+0000", 2024-09-13T14:45:36.045 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7102999999999999e-05 2024-09-13T14:45:36.045 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.045 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.045 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.045 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.045 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.045 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:68082 5.5 5:ab56da9e:::benchmark_data_smithi096_138841_object68081:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:45:36.045 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.517514+0000", 2024-09-13T14:45:36.045 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 310.29696338399998, 2024-09-13T14:45:36.045 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16716524299999999, 2024-09-13T14:45:36.045 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.045 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.045 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.045 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:45:36.046 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:45:36.046 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:45:36.046 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 68082 2024-09-13T14:45:36.046 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.046 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.046 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.046 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.046 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517514+0000", 2024-09-13T14:45:36.046 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.046 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.046 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.046 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.046 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517514+0000", 2024-09-13T14:45:36.046 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.046 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.047 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.047 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.047 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517515+0000", 2024-09-13T14:45:36.047 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9500000000000002e-07 2024-09-13T14:45:36.047 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.047 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.047 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.047 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517532+0000", 2024-09-13T14:45:36.047 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6756e-05 2024-09-13T14:45:36.047 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.047 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.047 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.047 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517538+0000", 2024-09-13T14:45:36.048 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2729999999999999e-06 2024-09-13T14:45:36.048 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.048 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.048 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.048 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517543+0000", 2024-09-13T14:45:36.048 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6319999999999999e-06 2024-09-13T14:45:36.048 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.048 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.048 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.048 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522076+0000", 2024-09-13T14:45:36.048 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0045333719999999999 2024-09-13T14:45:36.048 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.048 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.049 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.049 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677838+0000", 2024-09-13T14:45:36.049 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15576197 2024-09-13T14:45:36.049 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.049 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.049 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:45:36.049 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677976+0000", 2024-09-13T14:45:36.049 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013789 2024-09-13T14:45:36.049 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.049 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.049 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:45:36.049 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.681787+0000", 2024-09-13T14:45:36.049 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0038115979999999998 2024-09-13T14:45:36.050 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.050 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.050 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.050 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.682419+0000", 2024-09-13T14:45:36.050 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000631541 2024-09-13T14:45:36.050 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.050 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.050 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.050 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684445+0000", 2024-09-13T14:45:36.050 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0020261459999999999 2024-09-13T14:45:36.050 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.050 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.050 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.051 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684625+0000", 2024-09-13T14:45:36.051 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00017956600000000001 2024-09-13T14:45:36.051 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.051 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.051 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.051 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684680+0000", 2024-09-13T14:45:36.051 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4904000000000002e-05 2024-09-13T14:45:36.051 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.051 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.051 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.051 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.051 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.051 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:68084 5.2 5:41fe4da9:::benchmark_data_smithi096_138841_object68083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:45:36.051 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.519853+0000", 2024-09-13T14:45:36.052 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 310.29462514900001, 2024-09-13T14:45:36.052 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.18042530400000001, 2024-09-13T14:45:36.052 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.052 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.052 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.052 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:45:36.052 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:45:36.052 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:45:36.052 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 68084 2024-09-13T14:45:36.052 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.052 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.052 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.052 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.052 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519853+0000", 2024-09-13T14:45:36.053 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.053 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.053 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.053 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.053 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519853+0000", 2024-09-13T14:45:36.053 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.053 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.053 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.053 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.053 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519854+0000", 2024-09-13T14:45:36.053 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.446e-06 2024-09-13T14:45:36.053 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.053 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.054 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.054 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519875+0000", 2024-09-13T14:45:36.054 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0798999999999999e-05 2024-09-13T14:45:36.054 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.054 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.054 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.054 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519887+0000", 2024-09-13T14:45:36.054 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1921e-05 2024-09-13T14:45:36.054 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.054 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.054 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.054 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519905+0000", 2024-09-13T14:45:36.054 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7994000000000001e-05 2024-09-13T14:45:36.054 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.055 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.055 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.055 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520192+0000", 2024-09-13T14:45:36.055 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00028749299999999999 2024-09-13T14:45:36.055 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.055 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.055 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.055 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520321+0000", 2024-09-13T14:45:36.055 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012930799999999999 2024-09-13T14:45:36.055 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.055 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.055 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:45:36.056 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520447+0000", 2024-09-13T14:45:36.056 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012507100000000001 2024-09-13T14:45:36.056 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.056 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.056 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.056 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.680495+0000", 2024-09-13T14:45:36.056 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16004827699999999 2024-09-13T14:45:36.056 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.056 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.056 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:45:36.056 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684061+0000", 2024-09-13T14:45:36.056 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0035666180000000001 2024-09-13T14:45:36.056 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.057 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.057 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.057 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.700185+0000", 2024-09-13T14:45:36.057 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.016123723 2024-09-13T14:45:36.057 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.057 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.057 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.057 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.700239+0000", 2024-09-13T14:45:36.057 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3553e-05 2024-09-13T14:45:36.057 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.057 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.057 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.057 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.700278+0000", 2024-09-13T14:45:36.057 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9100999999999997e-05 2024-09-13T14:45:36.058 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.058 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.058 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.058 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.058 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.058 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:68087 5.2 5:44b2e474:::benchmark_data_smithi096_138841_object68086:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:45:36.058 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.519923+0000", 2024-09-13T14:45:36.058 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 310.29455483499999, 2024-09-13T14:45:36.058 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.18477276300000001, 2024-09-13T14:45:36.058 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.058 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.058 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.059 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:45:36.059 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:45:36.059 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:45:36.059 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 68087 2024-09-13T14:45:36.059 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.059 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.059 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.059 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.059 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519923+0000", 2024-09-13T14:45:36.059 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.059 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.059 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.059 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.060 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519923+0000", 2024-09-13T14:45:36.060 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.060 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.060 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.060 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.060 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519923+0000", 2024-09-13T14:45:36.060 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3099999999999998e-07 2024-09-13T14:45:36.060 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.060 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.060 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.060 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519940+0000", 2024-09-13T14:45:36.060 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6365e-05 2024-09-13T14:45:36.060 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.061 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.061 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.061 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519947+0000", 2024-09-13T14:45:36.061 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9489999999999999e-06 2024-09-13T14:45:36.061 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.061 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.061 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.061 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519951+0000", 2024-09-13T14:45:36.061 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7840000000000003e-06 2024-09-13T14:45:36.061 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.061 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.061 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.061 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677824+0000", 2024-09-13T14:45:36.061 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15787304099999999 2024-09-13T14:45:36.062 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.062 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.062 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.062 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677959+0000", 2024-09-13T14:45:36.062 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000134148 2024-09-13T14:45:36.062 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.062 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.062 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:45:36.062 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.678095+0000", 2024-09-13T14:45:36.062 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000136837 2024-09-13T14:45:36.062 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.062 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.062 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.062 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684103+0000", 2024-09-13T14:45:36.062 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0060078400000000004 2024-09-13T14:45:36.063 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.063 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.063 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:45:36.063 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684196+0000", 2024-09-13T14:45:36.063 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2334000000000001e-05 2024-09-13T14:45:36.063 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.063 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.063 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.063 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.704598+0000", 2024-09-13T14:45:36.063 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.020402453000000001 2024-09-13T14:45:36.063 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.063 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.063 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.063 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.704646+0000", 2024-09-13T14:45:36.063 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7812000000000002e-05 2024-09-13T14:45:36.064 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.064 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.064 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.064 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.704696+0000", 2024-09-13T14:45:36.064 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9669000000000002e-05 2024-09-13T14:45:36.064 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.064 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.064 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.064 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.064 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.064 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.35049.0:68081 5.5 e119/107 ondisk, result = 0)", 2024-09-13T14:45:36.064 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.520949+0000", 2024-09-13T14:45:36.064 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 310.29352850399999, 2024-09-13T14:45:36.064 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.160838853, 2024-09-13T14:45:36.065 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.065 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.065 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:45:36.065 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.065 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.065 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.065 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520949+0000", 2024-09-13T14:45:36.065 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.065 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.065 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.065 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.065 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520949+0000", 2024-09-13T14:45:36.065 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.066 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.066 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.066 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.066 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520950+0000", 2024-09-13T14:45:36.066 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9699999999999995e-07 2024-09-13T14:45:36.066 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.066 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.066 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.066 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520953+0000", 2024-09-13T14:45:36.066 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7989999999999998e-06 2024-09-13T14:45:36.066 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.066 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.066 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.066 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520962+0000", 2024-09-13T14:45:36.066 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0659999999999993e-06 2024-09-13T14:45:36.067 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.067 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.067 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.067 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520968+0000", 2024-09-13T14:45:36.067 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0750000000000003e-06 2024-09-13T14:45:36.067 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.067 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.067 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.067 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.681765+0000", 2024-09-13T14:45:36.067 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16079722900000001 2024-09-13T14:45:36.067 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.067 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.067 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.067 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.681772+0000", 2024-09-13T14:45:36.068 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8090000000000004e-06 2024-09-13T14:45:36.068 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.068 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.068 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.068 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.681788+0000", 2024-09-13T14:45:36.068 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6078000000000001e-05 2024-09-13T14:45:36.068 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.068 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.068 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.068 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.068 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.068 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.35049.0:68079 5.3 e119/107 ondisk, result = 0)", 2024-09-13T14:45:36.068 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.522506+0000", 2024-09-13T14:45:36.068 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 310.29197175600001, 2024-09-13T14:45:36.069 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15500866399999999, 2024-09-13T14:45:36.069 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.069 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.069 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:45:36.069 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.069 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.069 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.069 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522506+0000", 2024-09-13T14:45:36.069 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.069 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.069 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.069 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.069 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522506+0000", 2024-09-13T14:45:36.069 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.070 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.070 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.070 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.070 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522507+0000", 2024-09-13T14:45:36.070 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5600000000000004e-07 2024-09-13T14:45:36.070 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.070 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.070 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.070 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522511+0000", 2024-09-13T14:45:36.070 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3200000000000001e-06 2024-09-13T14:45:36.070 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.070 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.070 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.070 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677247+0000", 2024-09-13T14:45:36.071 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15473636299999999 2024-09-13T14:45:36.071 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.071 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.071 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.071 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677263+0000", 2024-09-13T14:45:36.071 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5777000000000002e-05 2024-09-13T14:45:36.071 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.071 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.071 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.071 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677381+0000", 2024-09-13T14:45:36.071 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000117434 2024-09-13T14:45:36.071 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.071 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.071 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.071 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677389+0000", 2024-09-13T14:45:36.072 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8219999999999996e-06 2024-09-13T14:45:36.072 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.072 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.072 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.072 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677515+0000", 2024-09-13T14:45:36.072 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012509199999999999 2024-09-13T14:45:36.072 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.072 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.072 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.072 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.072 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.072 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.35049.0:68084 5.2 e119/107 ondisk, result = 0)", 2024-09-13T14:45:36.072 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.522895+0000", 2024-09-13T14:45:36.072 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 310.291582463, 2024-09-13T14:45:36.073 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15760656300000001, 2024-09-13T14:45:36.073 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.073 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.073 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:45:36.073 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.073 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.073 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.073 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522895+0000", 2024-09-13T14:45:36.073 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.073 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.073 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.073 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.073 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522895+0000", 2024-09-13T14:45:36.074 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.074 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.074 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.074 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.074 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522896+0000", 2024-09-13T14:45:36.074 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3799999999999997e-07 2024-09-13T14:45:36.074 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.074 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.074 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.074 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.522905+0000", 2024-09-13T14:45:36.074 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7100000000000002e-06 2024-09-13T14:45:36.074 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.074 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.075 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.075 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677276+0000", 2024-09-13T14:45:36.075 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.154370968 2024-09-13T14:45:36.075 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.075 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.075 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.075 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677287+0000", 2024-09-13T14:45:36.075 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0641e-05 2024-09-13T14:45:36.075 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.075 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.075 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.075 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.680481+0000", 2024-09-13T14:45:36.075 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0031934720000000002 2024-09-13T14:45:36.075 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.076 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.076 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.076 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.680489+0000", 2024-09-13T14:45:36.076 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0590000000000003e-06 2024-09-13T14:45:36.076 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.076 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.076 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.076 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.680502+0000", 2024-09-13T14:45:36.076 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3175e-05 2024-09-13T14:45:36.076 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.076 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.076 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.076 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.076 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.076 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:155320 5.7 e130/107 5:ebe30544:::benchmark_data_smithi096_138841_object25124:head v 130'9638, mlcod=130'9635)", 2024-09-13T14:45:36.077 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.976910+0000", 2024-09-13T14:45:36.077 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 227.83756736199999, 2024-09-13T14:45:36.077 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.145356561, 2024-09-13T14:45:36.077 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.077 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.077 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.077 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.077 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.077 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.077 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.976910+0000", 2024-09-13T14:45:36.077 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.077 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.077 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.077 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.077 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.976910+0000", 2024-09-13T14:45:36.078 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.078 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.078 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.078 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.078 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.976911+0000", 2024-09-13T14:45:36.078 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.9999999999999995e-07 2024-09-13T14:45:36.078 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.078 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.078 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.078 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.976919+0000", 2024-09-13T14:45:36.078 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0690000000000002e-06 2024-09-13T14:45:36.078 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.078 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.079 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.079 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.976931+0000", 2024-09-13T14:45:36.079 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1506e-05 2024-09-13T14:45:36.079 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.079 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.079 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.079 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.976938+0000", 2024-09-13T14:45:36.079 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1339999999999998e-06 2024-09-13T14:45:36.079 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.079 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.079 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.079 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.977990+0000", 2024-09-13T14:45:36.079 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0010518190000000001 2024-09-13T14:45:36.079 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.080 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.080 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.080 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.978018+0000", 2024-09-13T14:45:36.080 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7798e-05 2024-09-13T14:45:36.080 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.080 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.080 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.080 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.122093+0000", 2024-09-13T14:45:36.080 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.144075693 2024-09-13T14:45:36.080 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.080 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.080 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.080 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.122267+0000", 2024-09-13T14:45:36.081 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000173542 2024-09-13T14:45:36.081 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.081 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.081 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.081 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.081 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.081 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:155322 5.7 e130/107 5:eae81fbc:::benchmark_data_smithi096_138841_object25126:head v 130'9639, mlcod=130'9635)", 2024-09-13T14:45:36.081 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.978364+0000", 2024-09-13T14:45:36.081 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 227.836113442, 2024-09-13T14:45:36.081 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15947017999999999, 2024-09-13T14:45:36.081 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.081 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.081 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.082 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.082 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.082 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.082 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.978364+0000", 2024-09-13T14:45:36.082 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.082 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.082 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.082 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.082 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.978364+0000", 2024-09-13T14:45:36.082 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.082 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.082 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.082 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.082 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.978365+0000", 2024-09-13T14:45:36.083 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4700000000000001e-07 2024-09-13T14:45:36.083 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.083 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.083 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.083 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.978369+0000", 2024-09-13T14:45:36.083 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0860000000000004e-06 2024-09-13T14:45:36.083 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.083 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.083 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.083 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.978379+0000", 2024-09-13T14:45:36.083 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0566e-05 2024-09-13T14:45:36.083 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.083 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.084 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.084 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.978385+0000", 2024-09-13T14:45:36.084 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7169999999999996e-06 2024-09-13T14:45:36.084 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.084 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.084 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.084 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982196+0000", 2024-09-13T14:45:36.084 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0038109509999999999 2024-09-13T14:45:36.084 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.084 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.084 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.084 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982525+0000", 2024-09-13T14:45:36.084 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00032845200000000002 2024-09-13T14:45:36.084 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.085 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.085 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.085 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.137777+0000", 2024-09-13T14:45:36.085 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15525257000000001 2024-09-13T14:45:36.085 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.085 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.085 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.085 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.137834+0000", 2024-09-13T14:45:36.085 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7290999999999997e-05 2024-09-13T14:45:36.085 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.085 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.085 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.085 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.085 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.086 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:155332 5.5 5:af4c7fbd:::benchmark_data_smithi096_138841_object25136:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:45:36.086 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.981878+0000", 2024-09-13T14:45:36.086 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 227.832600114, 2024-09-13T14:45:36.086 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.14758105599999999, 2024-09-13T14:45:36.086 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.086 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.086 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.086 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:45:36.086 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:45:36.086 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:45:36.086 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 155332 2024-09-13T14:45:36.086 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.086 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.086 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.087 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.087 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.981878+0000", 2024-09-13T14:45:36.087 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.087 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.087 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.087 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.087 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.981878+0000", 2024-09-13T14:45:36.087 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.087 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.087 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.087 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.087 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.981879+0000", 2024-09-13T14:45:36.087 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0330000000000001e-06 2024-09-13T14:45:36.088 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.088 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.088 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.088 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.981882+0000", 2024-09-13T14:45:36.088 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6109999999999998e-06 2024-09-13T14:45:36.088 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.088 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.088 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.088 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.981891+0000", 2024-09-13T14:45:36.088 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2070000000000001e-06 2024-09-13T14:45:36.088 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.088 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.088 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.089 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.981898+0000", 2024-09-13T14:45:36.089 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4080000000000003e-06 2024-09-13T14:45:36.089 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.089 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.089 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.089 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982086+0000", 2024-09-13T14:45:36.089 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00018797200000000001 2024-09-13T14:45:36.089 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.089 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.089 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.089 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986738+0000", 2024-09-13T14:45:36.089 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0046517449999999997 2024-09-13T14:45:36.089 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.090 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.090 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:45:36.090 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986844+0000", 2024-09-13T14:45:36.090 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000106773 2024-09-13T14:45:36.090 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.090 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.090 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.090 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129296+0000", 2024-09-13T14:45:36.090 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.142451877 2024-09-13T14:45:36.090 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.090 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.090 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.090 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129341+0000", 2024-09-13T14:45:36.090 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5243e-05 2024-09-13T14:45:36.091 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.091 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.091 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:45:36.091 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129357+0000", 2024-09-13T14:45:36.091 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5495e-05 2024-09-13T14:45:36.091 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.091 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.091 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.091 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129416+0000", 2024-09-13T14:45:36.091 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8953999999999998e-05 2024-09-13T14:45:36.091 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.091 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.091 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.092 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129459+0000", 2024-09-13T14:45:36.092 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2738000000000003e-05 2024-09-13T14:45:36.092 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.092 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.092 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.092 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.092 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.092 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:155335 5.3 5:c44414a0:::benchmark_data_smithi096_138841_object25139:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:45:36.092 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.982035+0000", 2024-09-13T14:45:36.092 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 227.83244215600001, 2024-09-13T14:45:36.092 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.14205281, 2024-09-13T14:45:36.092 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.092 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.093 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.093 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:45:36.093 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:45:36.093 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:45:36.093 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 155335 2024-09-13T14:45:36.093 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.093 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.093 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.093 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.093 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982035+0000", 2024-09-13T14:45:36.093 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.093 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.093 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.094 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.094 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982035+0000", 2024-09-13T14:45:36.094 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.094 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.094 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.094 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.094 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982036+0000", 2024-09-13T14:45:36.094 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9999999999999997e-07 2024-09-13T14:45:36.094 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.094 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.094 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.094 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982040+0000", 2024-09-13T14:45:36.094 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.3519999999999998e-06 2024-09-13T14:45:36.094 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.095 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.095 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.095 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982048+0000", 2024-09-13T14:45:36.095 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6780000000000006e-06 2024-09-13T14:45:36.095 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.095 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.095 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.095 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982054+0000", 2024-09-13T14:45:36.095 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0310000000000004e-06 2024-09-13T14:45:36.095 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.095 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.095 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.095 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982249+0000", 2024-09-13T14:45:36.095 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00019445899999999999 2024-09-13T14:45:36.096 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.096 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.096 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.096 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982377+0000", 2024-09-13T14:45:36.096 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012871200000000001 2024-09-13T14:45:36.096 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.096 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.096 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 3,5", 2024-09-13T14:45:36.096 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982458+0000", 2024-09-13T14:45:36.096 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0708e-05 2024-09-13T14:45:36.096 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.096 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.097 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.097 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989435+0000", 2024-09-13T14:45:36.097 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0069764129999999999 2024-09-13T14:45:36.097 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.097 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.097 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.097 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.119639+0000", 2024-09-13T14:45:36.097 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.130204137 2024-09-13T14:45:36.097 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.097 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.097 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:45:36.097 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124001+0000", 2024-09-13T14:45:36.097 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0043621049999999998 2024-09-13T14:45:36.098 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.098 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.098 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.098 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124047+0000", 2024-09-13T14:45:36.098 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6313000000000002e-05 2024-09-13T14:45:36.098 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.098 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.098 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.098 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124088+0000", 2024-09-13T14:45:36.098 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1202000000000003e-05 2024-09-13T14:45:36.098 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.098 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.098 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.098 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.099 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.099 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:155336 5.3 5:c313698d:::benchmark_data_smithi096_138841_object25140:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:45:36.099 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.982278+0000", 2024-09-13T14:45:36.099 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 227.83219930300001, 2024-09-13T14:45:36.099 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.14239633900000001, 2024-09-13T14:45:36.099 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.099 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.099 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.099 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:45:36.099 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:45:36.099 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:45:36.099 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 155336 2024-09-13T14:45:36.099 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.099 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.100 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.100 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.100 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982278+0000", 2024-09-13T14:45:36.100 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.100 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.100 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.100 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.100 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982278+0000", 2024-09-13T14:45:36.100 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.100 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.100 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.100 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.100 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982279+0000", 2024-09-13T14:45:36.101 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.9399999999999993e-07 2024-09-13T14:45:36.101 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.101 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.101 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.101 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982285+0000", 2024-09-13T14:45:36.101 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2789999999999998e-06 2024-09-13T14:45:36.101 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.101 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.101 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.101 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982296+0000", 2024-09-13T14:45:36.101 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.092e-05 2024-09-13T14:45:36.101 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.101 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.101 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.102 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982303+0000", 2024-09-13T14:45:36.102 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9689999999999997e-06 2024-09-13T14:45:36.102 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.102 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.102 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.102 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.985618+0000", 2024-09-13T14:45:36.102 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.003315597 2024-09-13T14:45:36.102 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.102 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.102 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.102 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.985751+0000", 2024-09-13T14:45:36.102 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013257899999999999 2024-09-13T14:45:36.102 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.102 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.103 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 3,5", 2024-09-13T14:45:36.103 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.985862+0000", 2024-09-13T14:45:36.103 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000110851 2024-09-13T14:45:36.103 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.103 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.103 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.103 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.119699+0000", 2024-09-13T14:45:36.103 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.13383772899999999 2024-09-13T14:45:36.103 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.103 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.103 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:45:36.103 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124097+0000", 2024-09-13T14:45:36.103 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0043978649999999999 2024-09-13T14:45:36.104 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.104 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.104 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.104 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124580+0000", 2024-09-13T14:45:36.104 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00048257600000000002 2024-09-13T14:45:36.104 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.104 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.104 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.104 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124633+0000", 2024-09-13T14:45:36.104 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3705000000000002e-05 2024-09-13T14:45:36.104 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.104 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.104 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.104 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.124675+0000", 2024-09-13T14:45:36.105 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1275e-05 2024-09-13T14:45:36.105 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.105 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.105 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.105 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.105 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.105 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:155325 5.7 e130/107 5:ee3ba1a9:::benchmark_data_smithi096_138841_object25129:head v 130'9640, mlcod=130'9636)", 2024-09-13T14:45:36.105 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.982625+0000", 2024-09-13T14:45:36.105 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 227.83185266800001, 2024-09-13T14:45:36.105 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.155244469, 2024-09-13T14:45:36.105 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.105 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.105 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.106 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.106 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.106 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.106 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982625+0000", 2024-09-13T14:45:36.106 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.106 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.106 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.106 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.106 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982625+0000", 2024-09-13T14:45:36.106 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.106 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.106 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.106 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.106 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982626+0000", 2024-09-13T14:45:36.107 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0800000000000004e-07 2024-09-13T14:45:36.107 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.107 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.107 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.107 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982632+0000", 2024-09-13T14:45:36.107 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6590000000000001e-06 2024-09-13T14:45:36.107 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.107 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.107 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.107 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982641+0000", 2024-09-13T14:45:36.107 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5799999999999992e-06 2024-09-13T14:45:36.107 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.107 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.108 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.108 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.982647+0000", 2024-09-13T14:45:36.108 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7169999999999996e-06 2024-09-13T14:45:36.108 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.108 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.108 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.108 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986298+0000", 2024-09-13T14:45:36.108 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0036512010000000002 2024-09-13T14:45:36.108 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.108 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.108 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.108 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986445+0000", 2024-09-13T14:45:36.108 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00014707699999999999 2024-09-13T14:45:36.108 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.109 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.109 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.109 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.137839+0000", 2024-09-13T14:45:36.109 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15139466800000001 2024-09-13T14:45:36.109 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.109 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.109 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.109 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.137869+0000", 2024-09-13T14:45:36.109 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.9958999999999999e-05 2024-09-13T14:45:36.109 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.109 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.109 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.109 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.110 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.110 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:155327 5.7 e130/107 5:eb2cb127:::benchmark_data_smithi096_138841_object25131:head v 130'9641, mlcod=130'9636)", 2024-09-13T14:45:36.110 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.986380+0000", 2024-09-13T14:45:36.110 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 227.828097297, 2024-09-13T14:45:36.110 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16496414300000001, 2024-09-13T14:45:36.110 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.110 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.110 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.110 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.110 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.110 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.110 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986380+0000", 2024-09-13T14:45:36.110 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.111 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.111 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.111 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.111 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986380+0000", 2024-09-13T14:45:36.111 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.111 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.111 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.111 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.111 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986381+0000", 2024-09-13T14:45:36.111 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3099999999999998e-07 2024-09-13T14:45:36.111 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.111 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.111 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.112 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986386+0000", 2024-09-13T14:45:36.112 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6569999999999996e-06 2024-09-13T14:45:36.112 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.112 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.112 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.112 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986397+0000", 2024-09-13T14:45:36.112 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1121000000000001e-05 2024-09-13T14:45:36.112 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.112 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.112 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.112 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.986402+0000", 2024-09-13T14:45:36.112 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7119999999999997e-06 2024-09-13T14:45:36.112 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.112 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.113 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.113 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.122374+0000", 2024-09-13T14:45:36.113 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.135971707 2024-09-13T14:45:36.113 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.113 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.113 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.113 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.122417+0000", 2024-09-13T14:45:36.113 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3392000000000001e-05 2024-09-13T14:45:36.113 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.113 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.113 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.113 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.151302+0000", 2024-09-13T14:45:36.113 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.028884310999999999 2024-09-13T14:45:36.113 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.114 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.114 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.114 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.151345+0000", 2024-09-13T14:45:36.114 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2712e-05 2024-09-13T14:45:36.114 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.114 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.114 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.114 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.114 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.114 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:155348 5.7 e130/107 5:e352345d:::benchmark_data_smithi096_138841_object25152:head v 130'9642, mlcod=130'9636)", 2024-09-13T14:45:36.114 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.999666+0000", 2024-09-13T14:45:36.114 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 227.81481206199999, 2024-09-13T14:45:36.114 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15170647500000001, 2024-09-13T14:45:36.115 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.115 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.115 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.115 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.115 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.115 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.115 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.999666+0000", 2024-09-13T14:45:36.115 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.115 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.115 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.115 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.115 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.999666+0000", 2024-09-13T14:45:36.115 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.115 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.115 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.116 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.116 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.999667+0000", 2024-09-13T14:45:36.116 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.265e-06 2024-09-13T14:45:36.116 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.116 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.116 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.116 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.999677+0000", 2024-09-13T14:45:36.116 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.876e-06 2024-09-13T14:45:36.116 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.116 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.116 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.116 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120515+0000", 2024-09-13T14:45:36.116 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.120838062 2024-09-13T14:45:36.116 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.117 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.117 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.117 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120520+0000", 2024-09-13T14:45:36.117 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3279999999999996e-06 2024-09-13T14:45:36.117 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.117 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.117 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.117 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.132225+0000", 2024-09-13T14:45:36.117 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011705357 2024-09-13T14:45:36.117 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.117 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.117 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.117 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.132324+0000", 2024-09-13T14:45:36.118 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.8665999999999994e-05 2024-09-13T14:45:36.118 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.118 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.118 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.118 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.151348+0000", 2024-09-13T14:45:36.118 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.019023786000000001 2024-09-13T14:45:36.118 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.118 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.118 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.118 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.151372+0000", 2024-09-13T14:45:36.118 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4134999999999998e-05 2024-09-13T14:45:36.118 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.118 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.118 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.119 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.119 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.119 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:36173 6.9 6:974f41c1:::benchmark_data_smithi096_167087_object36172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:45:36.119 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:44:24.266240+0000", 2024-09-13T14:45:36.119 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 71.548238132999998, 2024-09-13T14:45:36.119 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15562579200000001, 2024-09-13T14:45:36.119 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.119 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.119 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.119 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:45:36.119 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:45:36.119 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:45:36.119 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 36173 2024-09-13T14:45:36.119 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.120 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.120 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.120 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.120 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.266240+0000", 2024-09-13T14:45:36.120 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.120 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.120 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.120 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.120 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.266240+0000", 2024-09-13T14:45:36.120 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.120 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.120 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.120 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.121 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.266242+0000", 2024-09-13T14:45:36.121 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.057e-06 2024-09-13T14:45:36.121 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.121 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.121 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.121 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.266263+0000", 2024-09-13T14:45:36.121 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1339999999999999e-05 2024-09-13T14:45:36.121 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.121 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.121 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.121 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.266275+0000", 2024-09-13T14:45:36.121 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1593999999999999e-05 2024-09-13T14:45:36.121 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.122 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.122 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.122 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.266281+0000", 2024-09-13T14:45:36.122 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8469999999999998e-06 2024-09-13T14:45:36.122 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.122 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.122 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.122 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.266494+0000", 2024-09-13T14:45:36.122 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00021288899999999999 2024-09-13T14:45:36.122 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.122 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.122 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.122 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.266625+0000", 2024-09-13T14:45:36.122 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013092799999999999 2024-09-13T14:45:36.123 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.123 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.123 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:45:36.123 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.266753+0000", 2024-09-13T14:45:36.123 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012771599999999999 2024-09-13T14:45:36.123 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.123 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.123 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:45:36.123 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.269992+0000", 2024-09-13T14:45:36.123 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0032390090000000002 2024-09-13T14:45:36.123 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.123 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.123 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.123 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.270069+0000", 2024-09-13T14:45:36.123 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.7064999999999995e-05 2024-09-13T14:45:36.124 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.124 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.124 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.124 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.421775+0000", 2024-09-13T14:45:36.124 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15170597 2024-09-13T14:45:36.124 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.124 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.124 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.124 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.421828+0000", 2024-09-13T14:45:36.124 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2661000000000003e-05 2024-09-13T14:45:36.124 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.124 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.124 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.125 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.421865+0000", 2024-09-13T14:45:36.125 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7716000000000003e-05 2024-09-13T14:45:36.125 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.125 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.125 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.125 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.125 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.125 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:36175 6.9 6:9b25f95c:::benchmark_data_smithi096_167087_object36174:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:45:36.125 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:44:24.266750+0000", 2024-09-13T14:45:36.125 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 71.547727953000006, 2024-09-13T14:45:36.125 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.155236977, 2024-09-13T14:45:36.125 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.125 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.125 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.126 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:45:36.126 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:45:36.126 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:45:36.126 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 36175 2024-09-13T14:45:36.126 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.126 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.126 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.126 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.126 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.266750+0000", 2024-09-13T14:45:36.126 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.126 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.126 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.126 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.127 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.266750+0000", 2024-09-13T14:45:36.127 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.127 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.127 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.127 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.127 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.266751+0000", 2024-09-13T14:45:36.127 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9899999999999999e-07 2024-09-13T14:45:36.127 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.127 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.127 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.127 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.266770+0000", 2024-09-13T14:45:36.127 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9570999999999999e-05 2024-09-13T14:45:36.127 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.128 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.128 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.128 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.266780+0000", 2024-09-13T14:45:36.128 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0107e-05 2024-09-13T14:45:36.128 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.128 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.128 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.128 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.266787+0000", 2024-09-13T14:45:36.128 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3430000000000001e-06 2024-09-13T14:45:36.128 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.128 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.128 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.128 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.268267+0000", 2024-09-13T14:45:36.128 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0014808270000000001 2024-09-13T14:45:36.129 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.129 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.129 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.129 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.268402+0000", 2024-09-13T14:45:36.129 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013511999999999999 2024-09-13T14:45:36.129 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.129 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.129 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:45:36.129 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.268570+0000", 2024-09-13T14:45:36.129 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00016766400000000001 2024-09-13T14:45:36.129 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.129 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.129 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:45:36.130 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.270000+0000", 2024-09-13T14:45:36.130 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001429914 2024-09-13T14:45:36.130 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.130 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.130 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.130 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.270104+0000", 2024-09-13T14:45:36.130 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000104182 2024-09-13T14:45:36.130 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.130 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.130 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.130 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.421907+0000", 2024-09-13T14:45:36.130 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15180244800000001 2024-09-13T14:45:36.130 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.130 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.131 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.131 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.421952+0000", 2024-09-13T14:45:36.131 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5416e-05 2024-09-13T14:45:36.131 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.131 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.131 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.131 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:24.421987+0000", 2024-09-13T14:45:36.131 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4585999999999997e-05 2024-09-13T14:45:36.131 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.131 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.131 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.131 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.131 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.132 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:37471 6.9 6:92dc42f2:::benchmark_data_smithi096_167087_object37470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:45:36.132 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:44:25.099321+0000", 2024-09-13T14:45:36.132 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 70.715156719999996, 2024-09-13T14:45:36.132 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.169988115, 2024-09-13T14:45:36.132 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.132 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.132 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.132 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:45:36.132 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:45:36.132 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:45:36.132 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 37471 2024-09-13T14:45:36.132 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.132 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.133 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.133 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.133 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:25.099321+0000", 2024-09-13T14:45:36.133 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.133 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.133 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.133 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.133 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:25.099321+0000", 2024-09-13T14:45:36.133 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.133 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.133 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.133 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.133 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:25.099322+0000", 2024-09-13T14:45:36.133 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.525e-06 2024-09-13T14:45:36.134 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.134 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.134 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.134 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:25.099346+0000", 2024-09-13T14:45:36.134 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3988000000000001e-05 2024-09-13T14:45:36.134 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.134 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.134 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.134 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:25.099362+0000", 2024-09-13T14:45:36.134 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5505000000000001e-05 2024-09-13T14:45:36.134 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.134 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.134 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.134 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:25.099371+0000", 2024-09-13T14:45:36.135 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6759999999999996e-06 2024-09-13T14:45:36.135 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.135 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.135 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.135 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:25.099435+0000", 2024-09-13T14:45:36.135 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4332999999999995e-05 2024-09-13T14:45:36.135 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.135 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.135 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.135 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:25.099562+0000", 2024-09-13T14:45:36.135 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012747799999999999 2024-09-13T14:45:36.135 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.135 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.135 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:45:36.136 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:25.099734+0000", 2024-09-13T14:45:36.136 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000171275 2024-09-13T14:45:36.136 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.136 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.136 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:45:36.136 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:25.101677+0000", 2024-09-13T14:45:36.136 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0019433199999999999 2024-09-13T14:45:36.136 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.136 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.136 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.136 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:25.102276+0000", 2024-09-13T14:45:36.136 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00059928499999999997 2024-09-13T14:45:36.136 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.136 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.137 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.137 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:25.269242+0000", 2024-09-13T14:45:36.137 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16696528899999999 2024-09-13T14:45:36.137 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.137 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.137 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.137 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:25.269280+0000", 2024-09-13T14:45:36.137 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.8025999999999998e-05 2024-09-13T14:45:36.137 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.137 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.137 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.137 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:25.269309+0000", 2024-09-13T14:45:36.137 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.9414999999999999e-05 2024-09-13T14:45:36.137 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.138 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.138 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.138 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.138 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.138 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:38742 6.9 6:93f9e3dd:::benchmark_data_smithi096_167087_object38741:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:45:36.138 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:44:26.146732+0000", 2024-09-13T14:45:36.138 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 69.667745792000005, 2024-09-13T14:45:36.138 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.13947226600000001, 2024-09-13T14:45:36.138 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:45:36.138 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:45:36.138 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:36.138 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:45:36.138 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:45:36.139 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:45:36.139 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 38742 2024-09-13T14:45:36.139 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.139 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:45:36.139 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.139 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:45:36.139 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:26.146732+0000", 2024-09-13T14:45:36.139 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.139 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.139 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.139 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:45:36.139 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:26.146732+0000", 2024-09-13T14:45:36.139 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:45:36.139 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.140 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.140 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:45:36.140 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:26.146734+0000", 2024-09-13T14:45:36.140 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8220000000000001e-06 2024-09-13T14:45:36.140 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.140 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.140 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:45:36.140 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:26.146757+0000", 2024-09-13T14:45:36.140 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3295e-05 2024-09-13T14:45:36.140 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.140 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.140 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:45:36.140 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:26.146776+0000", 2024-09-13T14:45:36.140 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.895e-05 2024-09-13T14:45:36.141 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.141 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.141 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:45:36.141 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:26.146785+0000", 2024-09-13T14:45:36.141 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.4520000000000003e-06 2024-09-13T14:45:36.141 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.141 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.141 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:45:36.141 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:26.146846+0000", 2024-09-13T14:45:36.141 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0328999999999998e-05 2024-09-13T14:45:36.141 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.141 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.141 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:45:36.142 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:26.146985+0000", 2024-09-13T14:45:36.142 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013944700000000001 2024-09-13T14:45:36.142 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.142 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.142 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:45:36.142 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:26.147210+0000", 2024-09-13T14:45:36.142 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00022487599999999999 2024-09-13T14:45:36.142 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.142 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.142 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:45:36.142 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:26.150544+0000", 2024-09-13T14:45:36.142 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0033342990000000002 2024-09-13T14:45:36.142 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.142 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.143 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.143 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:26.151910+0000", 2024-09-13T14:45:36.143 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001365727 2024-09-13T14:45:36.143 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.143 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.143 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:36.143 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:26.285600+0000", 2024-09-13T14:45:36.143 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.13369007399999999 2024-09-13T14:45:36.143 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.143 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.143 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:45:36.143 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:26.286157+0000", 2024-09-13T14:45:36.143 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000557366 2024-09-13T14:45:36.143 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:45:36.144 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:45:36.144 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:45:36.144 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:44:26.286204+0000", 2024-09-13T14:45:36.144 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6628999999999997e-05 2024-09-13T14:45:36.144 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.144 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.144 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.144 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:45:36.144 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:45:36.144 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:45:36.144 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:36.170 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:36.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:36.272 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:36.297 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:36.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:36 smithi096 ceph-mon[98901]: osdmap e165: 8 total, 8 up, 7 in 2024-09-13T14:45:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:36 smithi096 ceph-mon[98901]: 6.b starting backfill to osd.4 from (0'0,0'0] MAX to 162'7193 2024-09-13T14:45:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:36.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:36 smithi096 ceph-mon[102928]: osdmap e165: 8 total, 8 up, 7 in 2024-09-13T14:45:36.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:36 smithi096 ceph-mon[102928]: 6.b starting backfill to osd.4 from (0'0,0'0] MAX to 162'7193 2024-09-13T14:45:36.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:36.399 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:36.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.443 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:36.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:36.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:36 smithi204 ceph-mon[87071]: osdmap e165: 8 total, 8 up, 7 in 2024-09-13T14:45:36.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:36 smithi204 ceph-mon[87071]: 6.b starting backfill to osd.4 from (0'0,0'0] MAX to 162'7193 2024-09-13T14:45:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:36.544 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:36.580 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:36.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:36.685 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:36.710 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:36.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:36.812 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:36.838 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:36.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:36.940 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:36.942 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 88 16 115558 115542 82.0443 52.25 0.0206929 0.0121835 2024-09-13T14:45:36.968 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:36.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:36.995 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-09-13T14:45:36.996 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd test-reweight-by-utilization 1000 0.05 2024-09-13T14:45:37.070 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:37.099 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:37.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:37.201 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:37.254 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:37.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:37.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:37 smithi096 ceph-mon[98901]: pgmap v476: 99 pgs: 2 peering, 97 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 44 MiB/s wr, 709 op/s 2024-09-13T14:45:37.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:37 smithi096 ceph-mon[102928]: pgmap v476: 99 pgs: 2 peering, 97 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 44 MiB/s wr, 709 op/s 2024-09-13T14:45:37.357 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:37.408 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:37.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:37.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:37 smithi204 ceph-mon[87071]: pgmap v476: 99 pgs: 2 peering, 97 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 44 MiB/s wr, 709 op/s 2024-09-13T14:45:37.510 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:37.543 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:37.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:37.611 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:45:37.645 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:37.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.715 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:37.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:37.816 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:37.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.871 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:37.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:37.944 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 89 16 116363 116347 81.687 50.3125 0.026781 0.012236 2024-09-13T14:45:37.973 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:38.000 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:38.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:38.102 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:38.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.130 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:38.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:38.231 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:38.258 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:38.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:38.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:45:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:45:37] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.51.0" 2024-09-13T14:45:38.360 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:38.390 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:38.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:38.493 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:38.519 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:38.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:38.621 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:38.650 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:38.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:38.751 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:38.817 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:38.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:38.918 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:38.920 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:38.942 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:38.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:38.943 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Total time run: 90.0082 2024-09-13T14:45:38.944 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Total writes made: 117225 2024-09-13T14:45:38.944 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Write size: 65536 2024-09-13T14:45:38.944 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Object size: 65536 2024-09-13T14:45:38.944 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Bandwidth (MB/sec): 81.3989 2024-09-13T14:45:38.944 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev Bandwidth: 37.9501 2024-09-13T14:45:38.944 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max bandwidth (MB/sec): 204.188 2024-09-13T14:45:38.944 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min bandwidth (MB/sec): 0 2024-09-13T14:45:38.944 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Average IOPS: 1302 2024-09-13T14:45:38.944 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev IOPS: 607.201 2024-09-13T14:45:38.944 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max IOPS: 3267 2024-09-13T14:45:38.944 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min IOPS: 0 2024-09-13T14:45:38.944 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Average Latency(s): 0.0122808 2024-09-13T14:45:38.944 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev Latency(s): 0.0294985 2024-09-13T14:45:38.945 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max latency(s): 2.48616 2024-09-13T14:45:38.945 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min latency(s): 0.00106401 2024-09-13T14:45:38.950 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:45:39.044 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:39.102 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:39.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:39.195 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:39 smithi096 ceph-mon[98901]: pgmap v477: 99 pgs: 1 active+remapped+backfilling, 2 peering, 96 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 12 KiB/s rd, 40 MiB/s wr, 649 op/s; 7111/345897 objects misplaced (2.056%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T14:45:39.196 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:39 smithi096 ceph-mon[102928]: pgmap v477: 99 pgs: 1 active+remapped+backfilling, 2 peering, 96 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 12 KiB/s rd, 40 MiB/s wr, 649 op/s; 7111/345897 objects misplaced (2.056%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T14:45:39.204 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:39.233 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:39.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:39.334 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:39.359 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:39.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:39.461 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:39.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:39 smithi204 ceph-mon[87071]: pgmap v477: 99 pgs: 1 active+remapped+backfilling, 2 peering, 96 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 12 KiB/s rd, 40 MiB/s wr, 649 op/s; 7111/345897 objects misplaced (2.056%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T14:45:39.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.501 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:39.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:39.602 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:39.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.638 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:39.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:39.670 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:39.706 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:45:39.707 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:45:39.738 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:39.745 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:45:39.745 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:45:39.765 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:39.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:39.783 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:45:39.783 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:45:39.813 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:45:39.813 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:45:39.845 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:45:39.845 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:45:39.866 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:39.889 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:45:39.889 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:45:39.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.911 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:39.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:39.923 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:45:39.923 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:45:39.958 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:45:39.958 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:45:40.013 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:40.047 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:40.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:40.149 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:40.177 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:40.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:40.279 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:40.316 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:40.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:40.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:40 smithi096 ceph-mon[98901]: 1.2 scrub starts 2024-09-13T14:45:40.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:40 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2247756784' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:40.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:40 smithi096 ceph-mon[102928]: 1.2 scrub starts 2024-09-13T14:45:40.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:40 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2247756784' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:40.370 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:40.418 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:40.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.469 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:40.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:40 smithi204 ceph-mon[87071]: 1.2 scrub starts 2024-09-13T14:45:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:40 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2247756784' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:40.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:45:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:45:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:40.569 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:40.613 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:40.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:40.715 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:40.742 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:40.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:40.844 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:40.881 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:40.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:40.982 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:41.009 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:41.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:41.111 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:41.137 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:41.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:41.239 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:41.281 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:41.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:41.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:41 smithi096 ceph-mon[98901]: pgmap v478: 99 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 97 active+clean; 7.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 10 KiB/s rd, 27 MiB/s wr, 442 op/s; 7111/347391 objects misplaced (2.047%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:45:41.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:41 smithi096 ceph-mon[102928]: pgmap v478: 99 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 97 active+clean; 7.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 10 KiB/s rd, 27 MiB/s wr, 442 op/s; 7111/347391 objects misplaced (2.047%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:45:41.383 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:41.423 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:41.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:41.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:41 smithi204 ceph-mon[87071]: pgmap v478: 99 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 97 active+clean; 7.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 10 KiB/s rd, 27 MiB/s wr, 442 op/s; 7111/347391 objects misplaced (2.047%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:45:41.524 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:41.549 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:41.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:41.651 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:41.675 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:41.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:41.777 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:41.807 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:41.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:41.909 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:41.942 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:41.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:42.044 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:42.065 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:42.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:42.166 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:42.188 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:42.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:42.289 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:42.325 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:42.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:42.427 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:42.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.453 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:42.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:42.554 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:42.581 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:42.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:42.684 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:42.719 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:42.821 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:42.843 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:42.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:42.945 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:42.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.979 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:42.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:43.080 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:43.105 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:43.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:43.207 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:43.237 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:43.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:43.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:43 smithi096 ceph-mon[98901]: pgmap v479: 99 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 97 active+clean; 7.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 8.6 KiB/s rd, 23 MiB/s wr, 482 op/s; 7111/346761 objects misplaced (2.051%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:45:43.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:43 smithi096 ceph-mon[102928]: pgmap v479: 99 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 97 active+clean; 7.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 8.6 KiB/s rd, 23 MiB/s wr, 482 op/s; 7111/346761 objects misplaced (2.051%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:45:43.339 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:43.373 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:43.473 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:43 smithi204 ceph-mon[87071]: pgmap v479: 99 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 97 active+clean; 7.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 8.6 KiB/s rd, 23 MiB/s wr, 482 op/s; 7111/346761 objects misplaced (2.051%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:45:43.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:43.575 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:43.597 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:43.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:43.699 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:43.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.729 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:43.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:43.831 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:43.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.861 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:43.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:43.962 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:43.988 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:43.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:44.090 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:44.115 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:44.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:44.216 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:44.240 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:44.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:44.344 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:44.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.366 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:44.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:44.467 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:44.494 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:44.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:44.596 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:44.645 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:44.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:44.746 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:44.771 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:44.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:44.873 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:44.905 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:44.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:45.006 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:45.035 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:45.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:45.137 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:45.174 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:45.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:45.276 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:45.301 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:45.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:45.305 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:45 smithi096 ceph-mon[98901]: 1.2 scrub starts 2024-09-13T14:45:45.306 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:45 smithi096 ceph-mon[98901]: pgmap v480: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 97 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 7.2 KiB/s rd, 25 MiB/s wr, 678 op/s; 6420/343638 objects misplaced (1.868%); 5.2 MiB/s, 83 objects/s recovering 2024-09-13T14:45:45.306 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:45 smithi096 ceph-mon[102928]: 1.2 scrub starts 2024-09-13T14:45:45.306 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:45 smithi096 ceph-mon[102928]: pgmap v480: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 97 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 7.2 KiB/s rd, 25 MiB/s wr, 678 op/s; 6420/343638 objects misplaced (1.868%); 5.2 MiB/s, 83 objects/s recovering 2024-09-13T14:45:45.403 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:45.449 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:45.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:45.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:45 smithi204 ceph-mon[87071]: 1.2 scrub starts 2024-09-13T14:45:45.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:45 smithi204 ceph-mon[87071]: pgmap v480: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 97 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 7.2 KiB/s rd, 25 MiB/s wr, 678 op/s; 6420/343638 objects misplaced (1.868%); 5.2 MiB/s, 83 objects/s recovering 2024-09-13T14:45:45.550 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:45.584 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:45.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:45.686 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:45.724 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:45.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:45.826 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:45.851 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:45.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:45.953 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:45.995 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:45.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:46.097 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:46.122 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:46.223 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:46.250 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:46.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:46.352 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:46.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.396 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:46.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:46.497 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:46.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.542 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:46.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:46.643 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:46.669 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:46.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:46.771 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:46.815 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:46.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:46.916 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:46.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.945 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:46.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:47.045 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:47.071 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:47.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:47.173 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:47.211 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:47.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:47.291 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:46 smithi096 ceph-mon[98901]: pgmap v481: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 97 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 6.5 KiB/s rd, 21 MiB/s wr, 621 op/s; 6420/342885 objects misplaced (1.872%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:45:47.291 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:46 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:47.291 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:46 smithi096 ceph-mon[102928]: pgmap v481: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 97 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 6.5 KiB/s rd, 21 MiB/s wr, 621 op/s; 6420/342885 objects misplaced (1.872%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:45:47.291 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:46 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:47.313 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:47.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.372 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:47.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:47.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:46 smithi204 ceph-mon[87071]: pgmap v481: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 97 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 6.5 KiB/s rd, 21 MiB/s wr, 621 op/s; 6420/342885 objects misplaced (1.872%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:45:47.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:46 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:45:47.473 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:47.497 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:47.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:47.599 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:47.627 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:47.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:47.729 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:47.754 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:47.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:47.856 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:47.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.893 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:47.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:47.904 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:45:47.995 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:48.018 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:48.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:48.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:47 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2380558414' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:48.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:45:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:45:47] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.51.0" 2024-09-13T14:45:48.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:48 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2380558414' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:48.120 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:48.142 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:48.244 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:48.247 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:45:48.270 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:48.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:48.372 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:48.398 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:48.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:48.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:47 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2380558414' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:48.500 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:48.539 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:48.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:48.641 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:48.669 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:48.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:48.771 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:48.796 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:48.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:48.898 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:48.923 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:48.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:49.025 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:49.062 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:49.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:49.164 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:49.204 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:49.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:49.306 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:48 smithi096 ceph-mon[98901]: pgmap v482: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 97 active+clean; 6.8 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 5.8 KiB/s rd, 19 MiB/s wr, 837 op/s; 5583/332727 objects misplaced (1.678%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:45:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:48 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:48 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2380558414' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:45:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:48 smithi096 ceph-mon[98901]: osdmap e166: 8 total, 8 up, 7 in 2024-09-13T14:45:49.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:48 smithi096 ceph-mon[102928]: pgmap v482: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 97 active+clean; 6.8 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 5.8 KiB/s rd, 19 MiB/s wr, 837 op/s; 5583/332727 objects misplaced (1.678%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:45:49.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:48 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:48 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2380558414' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:45:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:48 smithi096 ceph-mon[102928]: osdmap e166: 8 total, 8 up, 7 in 2024-09-13T14:45:49.353 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:49.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:49.455 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:48 smithi204 ceph-mon[87071]: pgmap v482: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 97 active+clean; 6.8 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 5.8 KiB/s rd, 19 MiB/s wr, 837 op/s; 5583/332727 objects misplaced (1.678%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:45:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:48 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:48 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2380558414' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:45:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:48 smithi204 ceph-mon[87071]: osdmap e166: 8 total, 8 up, 7 in 2024-09-13T14:45:49.507 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:49.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:49.609 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:49.634 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:49.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:49.667 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:45:49.668 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:45:49.736 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:49.762 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:49.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:49.864 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:49.892 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:49.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:49.994 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:50.022 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:50.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:50.124 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:50.148 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:50.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:50.249 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:50.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.281 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:50.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:50.382 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:50.406 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:50.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:50.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:45:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:45:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:50.508 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:50.532 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:50.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:50.634 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:50.669 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:50.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:50.771 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:50.792 INFO:teuthology.orchestra.run.smithi096.stdout:moved 0 / 297 (0%) 2024-09-13T14:45:50.792 INFO:teuthology.orchestra.run.smithi096.stdout:avg 42.4286 2024-09-13T14:45:50.792 INFO:teuthology.orchestra.run.smithi096.stdout:stddev 4.37059 -> 4.37059 (expected baseline 6.03053) 2024-09-13T14:45:50.792 INFO:teuthology.orchestra.run.smithi096.stdout:min osd.7 with 37 -> 37 pgs (0.872054 -> 0.872054 * mean) 2024-09-13T14:45:50.792 INFO:teuthology.orchestra.run.smithi096.stdout:max osd.3 with 48 -> 48 pgs (1.13131 -> 1.13131 * mean) 2024-09-13T14:45:50.792 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:45:50.792 INFO:teuthology.orchestra.run.smithi096.stdout:oload 1000 2024-09-13T14:45:50.793 INFO:teuthology.orchestra.run.smithi096.stdout:max_change 0.05 2024-09-13T14:45:50.793 INFO:teuthology.orchestra.run.smithi096.stdout:max_change_osds 4 2024-09-13T14:45:50.793 INFO:teuthology.orchestra.run.smithi096.stdout:average_utilization 0.0354 2024-09-13T14:45:50.793 INFO:teuthology.orchestra.run.smithi096.stdout:overload_utilization 0.3544 2024-09-13T14:45:50.793 INFO:teuthology.orchestra.run.smithi096.stderr:no change 2024-09-13T14:45:50.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.811 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:50.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:50.912 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:50.937 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:50.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:50.943 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:45:51.041 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:45:51.041 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:51.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:50 smithi096 ceph-mon[98901]: 1.2 scrub starts 2024-09-13T14:45:51.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:50 smithi096 ceph-mon[98901]: 1.2 scrub ok 2024-09-13T14:45:51.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:50 smithi096 ceph-mon[98901]: pgmap v484: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 8.9 MiB/s wr, 857 op/s; 5583/330204 objects misplaced (1.691%); 7.8 MiB/s, 125 objects/s recovering 2024-09-13T14:45:51.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:50 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:51.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:50 smithi096 ceph-mon[102928]: 1.2 scrub starts 2024-09-13T14:45:51.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:50 smithi096 ceph-mon[102928]: 1.2 scrub ok 2024-09-13T14:45:51.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:50 smithi096 ceph-mon[102928]: pgmap v484: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 8.9 MiB/s wr, 857 op/s; 5583/330204 objects misplaced (1.691%); 7.8 MiB/s, 125 objects/s recovering 2024-09-13T14:45:51.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:50 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:51.090 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:51.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:51.192 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:51.216 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:51.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:51.319 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:51.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.372 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:51.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:50 smithi204 ceph-mon[87071]: 1.2 scrub starts 2024-09-13T14:45:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:50 smithi204 ceph-mon[87071]: 1.2 scrub ok 2024-09-13T14:45:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:50 smithi204 ceph-mon[87071]: pgmap v484: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 8.9 MiB/s wr, 857 op/s; 5583/330204 objects misplaced (1.691%); 7.8 MiB/s, 125 objects/s recovering 2024-09-13T14:45:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:50 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:51.472 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:51.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.514 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:51.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:51.615 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:51.639 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:51.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:51.741 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:51.768 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:51.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:51.857 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:51.870 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:51.888 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-13T14:45:51.889 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 110 1.0 2024-09-13T14:45:51.946 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:51.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:52.048 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:52.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.075 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:52.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:52.176 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:52.201 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:52.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:52.208 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:51 smithi096 ceph-mon[102928]: from='client.54928 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:45:52.211 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:52 smithi096 ceph-mon[98901]: from='client.54928 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:45:52.303 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:52.333 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:52.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:52.435 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:52.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:51 smithi204 ceph-mon[87071]: from='client.54928 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:45:52.473 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:52.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:52.575 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:52.614 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:52.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:52.716 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:52.738 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:52.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:52.840 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:52.865 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:52.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:52.902 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:52.967 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:52.995 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:52.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:53.097 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:53.139 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:53.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:53.240 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:53.266 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:53.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:53.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:53 smithi096 ceph-mon[98901]: pgmap v485: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 5.3 MiB/s wr, 857 op/s; 5583/326139 objects misplaced (1.712%); 7.8 MiB/s, 125 objects/s recovering 2024-09-13T14:45:53.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:53 smithi096 ceph-mon[102928]: pgmap v485: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 5.3 MiB/s wr, 857 op/s; 5583/326139 objects misplaced (1.712%); 7.8 MiB/s, 125 objects/s recovering 2024-09-13T14:45:53.368 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:53.402 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:53.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:53.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:53 smithi204 ceph-mon[87071]: pgmap v485: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 5.3 MiB/s wr, 857 op/s; 5583/326139 objects misplaced (1.712%); 7.8 MiB/s, 125 objects/s recovering 2024-09-13T14:45:53.503 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:53.531 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:53.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:53.633 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:53.659 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:53.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:53.761 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:53.789 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:53.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:53.891 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:53.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.939 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:53.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:54.040 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:54.068 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:54.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:54.170 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:54.193 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:54.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:54.295 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:54.319 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:54.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:54.421 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:54.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.465 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:54.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:54.556 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:54.566 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:54.627 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:54.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:54.729 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:54.754 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:54.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:54.856 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:54.864 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:54.879 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:54.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:54.918 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:45:54.918 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:45:54.954 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:45:54.955 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:45:54.981 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:54.994 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:45:54.995 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:45:55.010 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:55.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:55.037 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:45:55.037 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:45:55.074 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:45:55.074 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:45:55.106 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:45:55.106 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:45:55.112 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:55.150 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:45:55.150 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:45:55.160 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:55.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:55.185 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:45:55.185 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:45:55.247 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:55 smithi096 ceph-mon[98901]: pgmap v486: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 843 op/s; 4740/318270 objects misplaced (1.489%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:45:55.247 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:55 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3754349841' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:55.248 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:55 smithi096 ceph-mon[102928]: pgmap v486: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 843 op/s; 4740/318270 objects misplaced (1.489%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:45:55.248 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:55 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3754349841' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:55.263 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:55.302 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:55.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:55.404 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:55.445 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:55.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:55 smithi204 ceph-mon[87071]: pgmap v486: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 843 op/s; 4740/318270 objects misplaced (1.489%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:45:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:55 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3754349841' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:55.547 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:55.575 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:55.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:55.677 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:55.705 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:55.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:55.807 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:55.837 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:55.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:55.939 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:55.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.987 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:55.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:56.017 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:56 smithi096 ceph-mon[98901]: 1.7 scrub starts 2024-09-13T14:45:56.017 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:56 smithi096 ceph-mon[98901]: 1.7 scrub ok 2024-09-13T14:45:56.089 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:56.114 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:56.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:56.215 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:56.241 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:56.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:56.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:56 smithi096 ceph-mon[102928]: 1.7 scrub starts 2024-09-13T14:45:56.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:56 smithi096 ceph-mon[102928]: 1.7 scrub ok 2024-09-13T14:45:56.343 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:56.369 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:56.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:56.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:56 smithi204 ceph-mon[87071]: 1.7 scrub starts 2024-09-13T14:45:56.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:56 smithi204 ceph-mon[87071]: 1.7 scrub ok 2024-09-13T14:45:56.471 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:56.509 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:56.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:56.611 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:56.654 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:56.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:56.756 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:56.796 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:56.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:56.898 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:56.928 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:56.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:57.030 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:57.069 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:57.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:57.171 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:57.197 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:57.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:57.299 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:57.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:57 smithi096 ceph-mon[102928]: pgmap v487: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.5 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 843 op/s; 4740/317526 objects misplaced (1.493%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:45:57.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:57 smithi096 ceph-mon[98901]: pgmap v487: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.5 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 843 op/s; 4740/317526 objects misplaced (1.493%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:45:57.343 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:57.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:57.445 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:57 smithi204 ceph-mon[87071]: pgmap v487: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.5 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 843 op/s; 4740/317526 objects misplaced (1.493%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:45:57.469 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:57.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:57.570 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:57.609 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:57.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:57.711 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:57.738 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:57.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:57.840 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:57.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.880 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:57.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:57.981 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:58.008 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:58.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:58.110 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:58.136 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:58.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:58.238 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:58.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.275 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:58.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:58.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:45:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:45:57] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.51.0" 2024-09-13T14:45:58.377 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:58.398 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:58.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:58.499 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:58.536 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:58.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:58.638 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:58.678 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:58.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:58.780 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:58.839 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:58.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:58.941 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:58.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.984 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:58.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:59.085 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:59.123 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:59.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:59.190 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:45:59.219 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:45:59 smithi096 ceph-mon[102928]: pgmap v488: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 848 op/s; 3884/307167 objects misplaced (1.264%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:45:59.219 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:45:59 smithi096 ceph-mon[98901]: pgmap v488: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 848 op/s; 3884/307167 objects misplaced (1.264%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:45:59.226 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:59.274 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:59.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:59.376 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:59.402 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:59.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:45:59 smithi204 ceph-mon[87071]: pgmap v488: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 848 op/s; 3884/307167 objects misplaced (1.264%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:45:59.504 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:59.529 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:45:59.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:59.631 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:59.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.685 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:59.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:59.786 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:59.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.838 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:59.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:59.939 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:59.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.973 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:45:59.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:00.074 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:00.094 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:46:00.094 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:46:00.094 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:46:00.094 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:46:00.094 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:46:00.098 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:00.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:00.100 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:00.200 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:00.225 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:00.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:00.327 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:00.366 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:00.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:00.468 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:46:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:46:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:00.498 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:00.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:00.600 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:00.626 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:00.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:00.728 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:00.768 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:00.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:00.882 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:00.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.909 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:00.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:01.010 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:01.055 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:01.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:01.058 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:46:01.157 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:01.196 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:01.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:01.298 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:01.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:01 smithi096 ceph-mon[98901]: pgmap v489: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 794 op/s; 3884/304551 objects misplaced (1.275%); 6.8 MiB/s, 109 objects/s recovering 2024-09-13T14:46:01.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:01 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1373362219' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:01.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:01 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:01.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:01 smithi096 ceph-mon[102928]: pgmap v489: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 794 op/s; 3884/304551 objects misplaced (1.275%); 6.8 MiB/s, 109 objects/s recovering 2024-09-13T14:46:01.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:01 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1373362219' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:01.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:01 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:01.324 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:01.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:01.426 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:01.451 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:01.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:01.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:01 smithi204 ceph-mon[87071]: pgmap v489: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 794 op/s; 3884/304551 objects misplaced (1.275%); 6.8 MiB/s, 109 objects/s recovering 2024-09-13T14:46:01.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:01 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1373362219' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:01.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:01 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:01.553 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:01.580 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:01.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:01.682 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:01.720 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:01.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:01.826 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:01.851 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:01.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:01.923 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:46:01.952 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:02.016 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:02.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:02.079 INFO:teuthology.orchestra.run.smithi096.stdout:moved 29 / 297 (9.76431%) 2024-09-13T14:46:02.079 INFO:teuthology.orchestra.run.smithi096.stdout:avg 42.4286 2024-09-13T14:46:02.079 INFO:teuthology.orchestra.run.smithi096.stdout:stddev 4.37059 -> 7.97957 (expected baseline 6.03053) 2024-09-13T14:46:02.079 INFO:teuthology.orchestra.run.smithi096.stdout:min osd.7 with 37 -> 35 pgs (0.872054 -> 0.824916 * mean) 2024-09-13T14:46:02.079 INFO:teuthology.orchestra.run.smithi096.stdout:max osd.3 with 48 -> 38 pgs (1.13131 -> 0.895623 * mean) 2024-09-13T14:46:02.079 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:46:02.080 INFO:teuthology.orchestra.run.smithi096.stdout:oload 110 2024-09-13T14:46:02.080 INFO:teuthology.orchestra.run.smithi096.stdout:max_change 1 2024-09-13T14:46:02.080 INFO:teuthology.orchestra.run.smithi096.stdout:max_change_osds 4 2024-09-13T14:46:02.080 INFO:teuthology.orchestra.run.smithi096.stdout:average_utilization 0.0331 2024-09-13T14:46:02.080 INFO:teuthology.orchestra.run.smithi096.stdout:overload_utilization 0.0364 2024-09-13T14:46:02.080 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3 weight 0.9000 -> 0.6406 2024-09-13T14:46:02.080 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7 weight 0.9000 -> 0.7773 2024-09-13T14:46:02.080 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2 weight 0.9000 -> 0.6457 2024-09-13T14:46:02.118 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:02.141 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:02.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:02.173 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:02 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:46:02.173 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:02 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:46:02.173 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:02 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:02.173 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:02 smithi096 ceph-mon[98901]: osdmap e167: 8 total, 8 up, 7 in 2024-09-13T14:46:02.173 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:02 smithi096 ceph-mon[98901]: from='client.54934 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:46:02.173 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:02 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'2':'42317','3':'41981','7':'50942'}"} : dispatch 2024-09-13T14:46:02.173 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:02 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:46:02.174 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:02 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:46:02.174 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:02 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:02.174 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:02 smithi096 ceph-mon[102928]: osdmap e167: 8 total, 8 up, 7 in 2024-09-13T14:46:02.174 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:02 smithi096 ceph-mon[102928]: from='client.54934 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:46:02.174 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:02 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'2':'42317','3':'41981','7':'50942'}"} : dispatch 2024-09-13T14:46:02.243 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:02.265 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:02.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:02.366 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:02.388 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:02.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:02.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:02 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:46:02.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:02 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:46:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:02 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:02 smithi204 ceph-mon[87071]: osdmap e167: 8 total, 8 up, 7 in 2024-09-13T14:46:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:02 smithi204 ceph-mon[87071]: from='client.54934 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:46:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:02 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'2':'42317','3':'41981','7':'50942'}"} : dispatch 2024-09-13T14:46:02.490 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:02.511 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:02.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:02.613 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:02.637 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:02.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:02.739 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:02.760 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:02.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:02.862 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:02.885 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:02.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:02.987 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:03.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.009 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:03.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:03.110 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:03.132 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:03.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:03.237 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:03.258 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:03.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:03.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:03 smithi096 ceph-mon[98901]: pgmap v491: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 845 op/s; 3884/300690 objects misplaced (1.292%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:46:03.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:03 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'42317','3':'41981','7':'50942'}"}]': finished 2024-09-13T14:46:03.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:03 smithi096 ceph-mon[98901]: osdmap e168: 8 total, 8 up, 7 in 2024-09-13T14:46:03.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:03 smithi096 ceph-mon[102928]: pgmap v491: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 845 op/s; 3884/300690 objects misplaced (1.292%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:46:03.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:03 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'42317','3':'41981','7':'50942'}"}]': finished 2024-09-13T14:46:03.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:03 smithi096 ceph-mon[102928]: osdmap e168: 8 total, 8 up, 7 in 2024-09-13T14:46:03.360 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:03.381 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:03.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:03.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:03 smithi204 ceph-mon[87071]: pgmap v491: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 845 op/s; 3884/300690 objects misplaced (1.292%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:46:03.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:03 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'42317','3':'41981','7':'50942'}"}]': finished 2024-09-13T14:46:03.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:03 smithi204 ceph-mon[87071]: osdmap e168: 8 total, 8 up, 7 in 2024-09-13T14:46:03.483 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:03.505 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:03.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:03.606 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:03.627 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:03.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:03.729 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:03.751 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:03.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:03.853 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:03.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.877 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:03.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:03.978 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:04.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.010 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:04.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:04.111 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:04.144 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:04.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:04.246 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:04.299 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:04.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:04.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:04 smithi096 ceph-mon[98901]: osdmap e169: 8 total, 8 up, 7 in 2024-09-13T14:46:04.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:04 smithi096 ceph-mon[98901]: osdmap e170: 8 total, 8 up, 7 in 2024-09-13T14:46:04.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:04 smithi096 ceph-mon[102928]: osdmap e169: 8 total, 8 up, 7 in 2024-09-13T14:46:04.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:04 smithi096 ceph-mon[102928]: osdmap e170: 8 total, 8 up, 7 in 2024-09-13T14:46:04.401 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:04.428 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:04.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:04.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:04 smithi204 ceph-mon[87071]: osdmap e169: 8 total, 8 up, 7 in 2024-09-13T14:46:04.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:04 smithi204 ceph-mon[87071]: osdmap e170: 8 total, 8 up, 7 in 2024-09-13T14:46:04.531 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:04.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.560 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:04.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:04.650 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:04.662 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:04.725 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:04.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:04.827 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:04.856 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:04.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:04.958 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:04.980 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:04.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:05.082 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:05.118 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:05.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:05.220 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:05.263 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:05.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:05.365 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:05.394 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:46:05.398 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:05.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:05.439 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[98901]: pgmap v494: 99 pgs: 2 remapped+peering, 1 unknown, 1 active+remapped+backfilling, 7 peering, 88 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 3021/293103 objects misplaced (1.031%); 7.3 MiB/s, 117 objects/s recovering 2024-09-13T14:46:05.440 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[98901]: 6.2 starting backfill to osd.7 from (0'0,0'0] MAX to 167'8615 2024-09-13T14:46:05.440 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[98901]: 6.e starting backfill to osd.5 from (0'0,0'0] MAX to 167'8380 2024-09-13T14:46:05.440 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[98901]: 6.f starting backfill to osd.1 from (0'0,0'0] MAX to 167'8522 2024-09-13T14:46:05.440 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[98901]: 6.0 starting backfill to osd.5 from (0'0,0'0] MAX to 167'8564 2024-09-13T14:46:05.440 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[98901]: 6.9 starting backfill to osd.3 from (0'0,0'0] MAX to 167'8706 2024-09-13T14:46:05.440 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[98901]: 6.1 starting backfill to osd.0 from (0'0,0'0] MAX to 167'8495 2024-09-13T14:46:05.440 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[98901]: 6.4 starting backfill to osd.4 from (0'0,0'0] MAX to 167'8667 2024-09-13T14:46:05.440 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[98901]: 6.a starting backfill to osd.1 from (0'0,0'0] MAX to 167'8673 2024-09-13T14:46:05.440 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[98901]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:46:05.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[102928]: pgmap v494: 99 pgs: 2 remapped+peering, 1 unknown, 1 active+remapped+backfilling, 7 peering, 88 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 3021/293103 objects misplaced (1.031%); 7.3 MiB/s, 117 objects/s recovering 2024-09-13T14:46:05.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[102928]: 6.2 starting backfill to osd.7 from (0'0,0'0] MAX to 167'8615 2024-09-13T14:46:05.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[102928]: 6.e starting backfill to osd.5 from (0'0,0'0] MAX to 167'8380 2024-09-13T14:46:05.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[102928]: 6.f starting backfill to osd.1 from (0'0,0'0] MAX to 167'8522 2024-09-13T14:46:05.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[102928]: 6.0 starting backfill to osd.5 from (0'0,0'0] MAX to 167'8564 2024-09-13T14:46:05.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[102928]: 6.9 starting backfill to osd.3 from (0'0,0'0] MAX to 167'8706 2024-09-13T14:46:05.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[102928]: 6.1 starting backfill to osd.0 from (0'0,0'0] MAX to 167'8495 2024-09-13T14:46:05.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[102928]: 6.4 starting backfill to osd.4 from (0'0,0'0] MAX to 167'8667 2024-09-13T14:46:05.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[102928]: 6.a starting backfill to osd.1 from (0'0,0'0] MAX to 167'8673 2024-09-13T14:46:05.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:05 smithi096 ceph-mon[102928]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:46:05.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:05 smithi204 ceph-mon[87071]: pgmap v494: 99 pgs: 2 remapped+peering, 1 unknown, 1 active+remapped+backfilling, 7 peering, 88 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 3021/293103 objects misplaced (1.031%); 7.3 MiB/s, 117 objects/s recovering 2024-09-13T14:46:05.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:05 smithi204 ceph-mon[87071]: 6.2 starting backfill to osd.7 from (0'0,0'0] MAX to 167'8615 2024-09-13T14:46:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:05 smithi204 ceph-mon[87071]: 6.e starting backfill to osd.5 from (0'0,0'0] MAX to 167'8380 2024-09-13T14:46:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:05 smithi204 ceph-mon[87071]: 6.f starting backfill to osd.1 from (0'0,0'0] MAX to 167'8522 2024-09-13T14:46:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:05 smithi204 ceph-mon[87071]: 6.0 starting backfill to osd.5 from (0'0,0'0] MAX to 167'8564 2024-09-13T14:46:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:05 smithi204 ceph-mon[87071]: 6.9 starting backfill to osd.3 from (0'0,0'0] MAX to 167'8706 2024-09-13T14:46:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:05 smithi204 ceph-mon[87071]: 6.1 starting backfill to osd.0 from (0'0,0'0] MAX to 167'8495 2024-09-13T14:46:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:05 smithi204 ceph-mon[87071]: 6.4 starting backfill to osd.4 from (0'0,0'0] MAX to 167'8667 2024-09-13T14:46:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:05 smithi204 ceph-mon[87071]: 6.a starting backfill to osd.1 from (0'0,0'0] MAX to 167'8673 2024-09-13T14:46:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:05 smithi204 ceph-mon[87071]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:46:05.501 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:05.554 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:05.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:05.656 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:05.680 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:05.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:05.781 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:05.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.824 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:05.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:05.925 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:05.973 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:05.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:06.075 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:06.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.125 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:06.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:06.227 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:06.270 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:06.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:06.372 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:06.400 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:06.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:06.502 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:06.546 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:06.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:06.648 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:06.673 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:06.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:06.775 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:06.811 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:06.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:06.913 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:06.945 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:06.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:07.046 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:07.090 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:07.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:07.192 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:07.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:06 smithi204 ceph-mon[87071]: pgmap v496: 99 pgs: 2 remapped+peering, 1 unknown, 1 active+remapped+backfilling, 7 peering, 88 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 809 op/s; 3021/293103 objects misplaced (1.031%); 9.3 MiB/s, 151 objects/s recovering 2024-09-13T14:46:07.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:46:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:07.228 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:07.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:07.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:06 smithi096 ceph-mon[102928]: pgmap v496: 99 pgs: 2 remapped+peering, 1 unknown, 1 active+remapped+backfilling, 7 peering, 88 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 809 op/s; 3021/293103 objects misplaced (1.031%); 9.3 MiB/s, 151 objects/s recovering 2024-09-13T14:46:07.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:46:07.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:07.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:06 smithi096 ceph-mon[98901]: pgmap v496: 99 pgs: 2 remapped+peering, 1 unknown, 1 active+remapped+backfilling, 7 peering, 88 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 809 op/s; 3021/293103 objects misplaced (1.031%); 9.3 MiB/s, 151 objects/s recovering 2024-09-13T14:46:07.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:46:07.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:07.329 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:07.354 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:07.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:07.456 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:07.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.501 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:07.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:07.602 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:07.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.633 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:07.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:07.735 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:07.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.768 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:07.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:07.869 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:07.898 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:46:07.898 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:46:07.899 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2024-09-13T14:46:07.899 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:46:07.900 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:07.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:08.003 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:08.031 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:08.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:08.133 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:08.161 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:08.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:08.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:07 smithi204 ceph-mon[87071]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:46:08.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:07 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:46:08.263 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:08.291 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:08.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:08.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:07 smithi096 ceph-mon[98901]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:46:08.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:07 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:46:08.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:07 smithi096 ceph-mon[102928]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:46:08.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:07 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:46:08.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:46:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:46:07] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-09-13T14:46:08.393 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:08.419 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:08.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:08.521 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:08.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.553 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:08.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:08.654 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:08.684 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:08.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:08.786 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:08.815 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:08.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:08.917 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:08.949 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:08.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:09.050 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:09.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.095 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:09.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:09.196 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:09.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:08 smithi204 ceph-mon[87071]: pgmap v497: 99 pgs: 3 active+remapped+backfill_wait, 2 remapped+peering, 4 active+remapped+backfilling, 90 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 741 op/s; 35735/287346 objects misplaced (12.436%); 39 MiB/s, 633 objects/s recovering 2024-09-13T14:46:09.237 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:09.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:09.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:08 smithi096 ceph-mon[98901]: pgmap v497: 99 pgs: 3 active+remapped+backfill_wait, 2 remapped+peering, 4 active+remapped+backfilling, 90 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 741 op/s; 35735/287346 objects misplaced (12.436%); 39 MiB/s, 633 objects/s recovering 2024-09-13T14:46:09.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:08 smithi096 ceph-mon[102928]: pgmap v497: 99 pgs: 3 active+remapped+backfill_wait, 2 remapped+peering, 4 active+remapped+backfilling, 90 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 741 op/s; 35735/287346 objects misplaced (12.436%); 39 MiB/s, 633 objects/s recovering 2024-09-13T14:46:09.339 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:09.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.387 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:09.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:09.488 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:09.516 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:09.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:09.618 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:09.660 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:09.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:09.762 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:09.792 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:09.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:09.894 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:09.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.929 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:09.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:09.954 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:46:09.954 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:46:09.993 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:46:09.993 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:46:10.029 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:46:10.030 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:46:10.030 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:10.061 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:10.061 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:46:10.061 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:46:10.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:10.090 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:46:10.090 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:46:10.117 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:46:10.117 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:46:10.151 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:46:10.151 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:46:10.164 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:10.188 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:46:10.189 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:46:10.216 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:10.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:10.318 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:10.363 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:10.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:10.465 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:10.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:46:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:46:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:10.497 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:10.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:10.599 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:10.649 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:10.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:10.751 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:10.781 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:10.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:10.883 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:10.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.936 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:10.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:11.038 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:11.066 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:11.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:11.168 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:11.208 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:11.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:11.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:10 smithi204 ceph-mon[87071]: pgmap v498: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 46981/286227 objects misplaced (16.414%); 36 MiB/s, 572 objects/s recovering 2024-09-13T14:46:11.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:10 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2883293854' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:11.310 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:11.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:10 smithi096 ceph-mon[98901]: pgmap v498: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 46981/286227 objects misplaced (16.414%); 36 MiB/s, 572 objects/s recovering 2024-09-13T14:46:11.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:10 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2883293854' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:11.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:10 smithi096 ceph-mon[102928]: pgmap v498: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 46981/286227 objects misplaced (16.414%); 36 MiB/s, 572 objects/s recovering 2024-09-13T14:46:11.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:10 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2883293854' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:11.337 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:11.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:11.439 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:11.475 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:11.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:11.577 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:11.619 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:11.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:11.690 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:11.720 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:11.749 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:11.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:11.851 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:11.896 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:11.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:11.942 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:11.998 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:12.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.038 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:12.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:12.140 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:12.185 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:12.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:12.287 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:12.334 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:12.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:12.436 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:12.483 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:12.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:12.585 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:12.628 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:12.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:12.729 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:12.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.773 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:12.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:12.875 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:12.928 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:12.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:12.930 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:46:12.930 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:46:13.031 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:13.064 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:13.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:13.165 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:13.204 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:13.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:13.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:12 smithi204 ceph-mon[87071]: pgmap v499: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 44740/282879 objects misplaced (15.816%); 40 MiB/s, 639 objects/s recovering 2024-09-13T14:46:13.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:12 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:46:13.256 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:12 smithi096 ceph-mon[98901]: pgmap v499: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 44740/282879 objects misplaced (15.816%); 40 MiB/s, 639 objects/s recovering 2024-09-13T14:46:13.256 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:12 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:46:13.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:12 smithi096 ceph-mon[102928]: pgmap v499: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 44740/282879 objects misplaced (15.816%); 40 MiB/s, 639 objects/s recovering 2024-09-13T14:46:13.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:12 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:46:13.307 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:13.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.336 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:13.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:13.437 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:13.477 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:13.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:13.578 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:13.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.629 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:13.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:13.680 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:46:13.730 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:13.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.757 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:13.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:13.858 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:13.930 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:13.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:14.032 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:14.088 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:14.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:14.237 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:14.265 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:14.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:14.367 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:14.395 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:14.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:14.497 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:14.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.529 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:14.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:14.630 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:14.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.658 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:14.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:14.759 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:14.781 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:14.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:14.883 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:14.919 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:14.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:15.021 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:15.049 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:15.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:15.151 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:15.156 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:14 smithi096 ceph-mon[98901]: pgmap v500: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 41103/275061 objects misplaced (14.943%); 49 MiB/s, 784 objects/s recovering 2024-09-13T14:46:15.156 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:14 smithi096 ceph-mon[102928]: pgmap v500: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 41103/275061 objects misplaced (14.943%); 49 MiB/s, 784 objects/s recovering 2024-09-13T14:46:15.184 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:15.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:15.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:14 smithi204 ceph-mon[87071]: pgmap v500: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 41103/275061 objects misplaced (14.943%); 49 MiB/s, 784 objects/s recovering 2024-09-13T14:46:15.286 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:15.328 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:15.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:15.430 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:15.451 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:15.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:15.552 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:15.586 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:15.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:15.688 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:15.709 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:15.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:15.811 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:15.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.850 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:15.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:15.951 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:15.976 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:15.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:16.078 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:16.120 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:16.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:16.224 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:16.254 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:16.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:16.323 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:46:16.324 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:46:16.356 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:16.385 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:16.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:16.487 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:16.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.519 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:16.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:16.620 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:16.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.650 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:16.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:16.751 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:16.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.781 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:16.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:16.882 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:16.913 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:16.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:17.015 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.055 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:17.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:17.156 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:17.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.189 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:17.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:16 smithi204 ceph-mon[87071]: pgmap v501: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 41103/275061 objects misplaced (14.943%); 42 MiB/s, 671 objects/s recovering 2024-09-13T14:46:17.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:16 smithi204 ceph-mon[87071]: from='client.35430 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:17.290 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:17.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.320 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:17.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:17.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:16 smithi096 ceph-mon[98901]: pgmap v501: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 41103/275061 objects misplaced (14.943%); 42 MiB/s, 671 objects/s recovering 2024-09-13T14:46:17.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:16 smithi096 ceph-mon[98901]: from='client.35430 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:17.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:16 smithi096 ceph-mon[102928]: pgmap v501: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 41103/275061 objects misplaced (14.943%); 42 MiB/s, 671 objects/s recovering 2024-09-13T14:46:17.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:16 smithi096 ceph-mon[102928]: from='client.35430 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:17.421 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:17.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.458 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:17.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:17.559 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:17.591 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:17.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:17.693 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:17.723 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:17.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:17.824 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:17.875 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:46:17.881 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:17.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:17.983 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:18.010 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:18.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:18.113 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:18.156 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:18.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:18.258 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:18.260 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:46:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:46:17] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-09-13T14:46:18.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.332 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:18.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:18.433 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:18.479 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:18.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:18.581 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:18.611 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:18.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:18.712 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:18.747 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:18.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:18.849 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:18.891 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:18.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:18.923 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":501,"stamp":"2024-09-13T14:46:15.773176+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6011688933,"num_objects":91687,"num_object_clones":0,"num_object_copies":275061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":41103,"num_objects_unfound":0,"num_objects_dirty":91687,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":143700,"num_write_kb":7515024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11314,"num_bytes_recovered":741615613,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111802,"ondisk_log_size":111802,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":306,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":20302868,"kb_used_data":18255388,"kb_used_omap":34,"kb_used_meta":2047005,"kb_avail":635913196,"statfs":{"total":671965249536,"available":651175112704,"internally_reserved":0,"allocated":18693517312,"data_stored":18686110716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35126,"internal_metadata":2096133834},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,2,3,5,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-394133504,"num_objects":-6014,"num_object_clones":0,"num_object_copies":-18042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":38082,"num_objects_unfound":0,"num_objects_dirty":-6014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6014,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7850,"num_bytes_recovered":513736704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.690004"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":206,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.084806+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:46:04.084806+0000","last_peered":"2024-09-13T14:46:04.084806+0000","last_clean":"2024-09-13T14:46:04.084806+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:46:04.084806+0000","last_undegraded":"2024-09-13T14:46:04.084806+0000","last_fullsized":"2024-09-13T14:46:04.084806+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":136,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.085469+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:46:03.085469+0000","last_peered":"2024-09-13T14:46:03.085469+0000","last_clean":"2024-09-13T14:46:03.085469+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:46:03.085469+0000","last_undegraded":"2024-09-13T14:46:03.085469+0000","last_fullsized":"2024-09-13T14:46:03.085469+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":133,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.081614+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:46:03.081614+0000","last_peered":"2024-09-13T14:46:03.081614+0000","last_clean":"2024-09-13T14:46:03.081614+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:46:03.081614+0000","last_undegraded":"2024-09-13T14:46:03.081614+0000","last_fullsized":"2024-09-13T14:46:03.081614+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":206,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.087049+0000","last_change":"2024-09-13T14:46:03.104638+0000","last_active":"2024-09-13T14:46:04.087049+0000","last_peered":"2024-09-13T14:46:04.087049+0000","last_clean":"2024-09-13T14:46:04.087049+0000","last_became_active":"2024-09-13T14:46:03.100486+0000","last_became_peered":"2024-09-13T14:46:03.100486+0000","last_unstale":"2024-09-13T14:46:04.087049+0000","last_undegraded":"2024-09-13T14:46:04.087049+0000","last_fullsized":"2024-09-13T14:46:04.087049+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:32:56.278149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":207,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.086598+0000","last_change":"2024-09-13T14:46:03.103064+0000","last_active":"2024-09-13T14:46:04.086598+0000","last_peered":"2024-09-13T14:46:04.086598+0000","last_clean":"2024-09-13T14:46:04.086598+0000","last_became_active":"2024-09-13T14:46:03.102717+0000","last_became_peered":"2024-09-13T14:46:03.102717+0000","last_unstale":"2024-09-13T14:46:04.086598+0000","last_undegraded":"2024-09-13T14:46:04.086598+0000","last_fullsized":"2024-09-13T14:46:04.086598+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:20:47.402707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":127,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.086160+0000","last_change":"2024-09-13T14:39:42.765301+0000","last_active":"2024-09-13T14:46:03.086160+0000","last_peered":"2024-09-13T14:46:03.086160+0000","last_clean":"2024-09-13T14:46:03.086160+0000","last_became_active":"2024-09-13T14:39:42.764977+0000","last_became_peered":"2024-09-13T14:39:42.764977+0000","last_unstale":"2024-09-13T14:46:03.086160+0000","last_undegraded":"2024-09-13T14:46:03.086160+0000","last_fullsized":"2024-09-13T14:46:03.086160+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:25.223972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.089089+0000","last_change":"2024-09-13T14:46:03.102112+0000","last_active":"2024-09-13T14:46:04.089089+0000","last_peered":"2024-09-13T14:46:04.089089+0000","last_clean":"2024-09-13T14:46:04.089089+0000","last_became_active":"2024-09-13T14:46:03.101801+0000","last_became_peered":"2024-09-13T14:46:03.101801+0000","last_unstale":"2024-09-13T14:46:04.089089+0000","last_undegraded":"2024-09-13T14:46:04.089089+0000","last_fullsized":"2024-09-13T14:46:04.089089+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:41:37.273610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":160,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.112412+0000","last_change":"2024-09-13T14:39:42.781872+0000","last_active":"2024-09-13T14:46:04.112412+0000","last_peered":"2024-09-13T14:46:04.112412+0000","last_clean":"2024-09-13T14:46:04.112412+0000","last_became_active":"2024-09-13T14:39:42.770254+0000","last_became_peered":"2024-09-13T14:39:42.770254+0000","last_unstale":"2024-09-13T14:46:04.112412+0000","last_undegraded":"2024-09-13T14:46:04.112412+0000","last_fullsized":"2024-09-13T14:46:04.112412+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:31.211081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"170'9049","reported_seq":10849,"reported_epoch":170,"state":"active+remapped+backfilling","last_fresh":"2024-09-13T14:46:11.867445+0000","last_change":"2024-09-13T14:46:04.274873+0000","last_active":"2024-09-13T14:46:11.867445+0000","last_peered":"2024-09-13T14:46:11.867445+0000","last_clean":"2024-09-13T14:46:02.050073+0000","last_became_active":"2024-09-13T14:46:04.237372+0000","last_became_peered":"2024-09-13T14:46:04.237372+0000","last_unstale":"2024-09-13T14:46:11.867445+0000","last_undegraded":"2024-09-13T14:46:11.867445+0000","last_fullsized":"2024-09-13T14:46:11.867445+0000","mapping_epoch":169,"log_start":"150'2155","ondisk_log_start":"150'2155","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":6894,"log_dups_size":0,"ondisk_log_size":6894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:51:26.792446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382271488,"num_objects":5833,"num_object_clones":0,"num_object_copies":17499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4980,"num_objects_unfound":0,"num_objects_dirty":5833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9049,"num_write_kb":476224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":878,"num_bytes_recovered":57540608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,2],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":5833}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":190,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.096324+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:46:04.096324+0000","last_peered":"2024-09-13T14:46:04.096324+0000","last_clean":"2024-09-13T14:46:04.096324+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:46:04.096324+0000","last_undegraded":"2024-09-13T14:46:04.096324+0000","last_fullsized":"2024-09-13T14:46:04.096324+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":136,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.104146+0000","last_change":"2024-09-13T14:46:03.103657+0000","last_active":"2024-09-13T14:46:03.104146+0000","last_peered":"2024-09-13T14:46:03.104146+0000","last_clean":"2024-09-13T14:46:03.104146+0000","last_became_active":"2024-09-13T14:46:03.103340+0000","last_became_peered":"2024-09-13T14:46:03.103340+0000","last_unstale":"2024-09-13T14:46:03.104146+0000","last_undegraded":"2024-09-13T14:46:03.104146+0000","last_fullsized":"2024-09-13T14:46:03.104146+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:00:30.925917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"170'9020","reported_seq":20112,"reported_epoch":170,"state":"active+remapped+backfilling","last_fresh":"2024-09-13T14:46:12.407405+0000","last_change":"2024-09-13T14:46:03.103078+0000","last_active":"2024-09-13T14:46:12.407405+0000","last_peered":"2024-09-13T14:46:12.407405+0000","last_clean":"2024-09-13T14:45:33.018667+0000","last_became_active":"2024-09-13T14:45:35.488533+0000","last_became_peered":"2024-09-13T14:45:35.488533+0000","last_unstale":"2024-09-13T14:46:12.407405+0000","last_undegraded":"2024-09-13T14:46:12.407405+0000","last_fullsized":"2024-09-13T14:46:12.407405+0000","mapping_epoch":164,"log_start":"150'2400","ondisk_log_start":"150'2400","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":6620,"log_dups_size":0,"ondisk_log_size":6620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376569856,"num_objects":5746,"num_object_clones":0,"num_object_copies":17238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":858,"num_objects_unfound":0,"num_objects_dirty":5746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9020,"num_write_kb":472512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5507,"num_bytes_recovered":360906752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,7],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3,7","objects":5746}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":136,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.085833+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:46:03.085833+0000","last_peered":"2024-09-13T14:46:03.085833+0000","last_clean":"2024-09-13T14:46:03.085833+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:46:03.085833+0000","last_undegraded":"2024-09-13T14:46:03.085833+0000","last_fullsized":"2024-09-13T14:46:03.085833+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":171,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.113897+0000","last_change":"2024-09-13T14:46:03.107827+0000","last_active":"2024-09-13T14:46:03.113897+0000","last_peered":"2024-09-13T14:46:03.113897+0000","last_clean":"2024-09-13T14:46:03.113897+0000","last_became_active":"2024-09-13T14:46:03.107536+0000","last_became_peered":"2024-09-13T14:46:03.107536+0000","last_unstale":"2024-09-13T14:46:03.113897+0000","last_undegraded":"2024-09-13T14:46:03.113897+0000","last_fullsized":"2024-09-13T14:46:03.113897+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:36:43.604865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"170'8955","reported_seq":8988,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:11.728251+0000","last_change":"2024-09-13T14:43:58.298623+0000","last_active":"2024-09-13T14:46:11.728251+0000","last_peered":"2024-09-13T14:46:11.728251+0000","last_clean":"2024-09-13T14:46:11.728251+0000","last_became_active":"2024-09-13T14:43:58.298319+0000","last_became_peered":"2024-09-13T14:43:58.298319+0000","last_unstale":"2024-09-13T14:46:11.728251+0000","last_undegraded":"2024-09-13T14:46:11.728251+0000","last_fullsized":"2024-09-13T14:46:11.728251+0000","mapping_epoch":145,"log_start":"150'700","ondisk_log_start":"150'700","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":8255,"log_dups_size":0,"ondisk_log_size":8255,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:45.641123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378208256,"num_objects":5771,"num_object_clones":0,"num_object_copies":17313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8955,"num_write_kb":471232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":132,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.085763+0000","last_change":"2024-09-13T14:39:42.774547+0000","last_active":"2024-09-13T14:46:03.085763+0000","last_peered":"2024-09-13T14:46:03.085763+0000","last_clean":"2024-09-13T14:46:03.085763+0000","last_became_active":"2024-09-13T14:39:42.766006+0000","last_became_peered":"2024-09-13T14:39:42.766006+0000","last_unstale":"2024-09-13T14:46:03.085763+0000","last_undegraded":"2024-09-13T14:46:03.085763+0000","last_fullsized":"2024-09-13T14:46:03.085763+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:06:29.632163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":197,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.089796+0000","last_change":"2024-09-13T14:46:03.100861+0000","last_active":"2024-09-13T14:46:04.089796+0000","last_peered":"2024-09-13T14:46:04.089796+0000","last_clean":"2024-09-13T14:46:04.089796+0000","last_became_active":"2024-09-13T14:46:03.100516+0000","last_became_peered":"2024-09-13T14:46:03.100516+0000","last_unstale":"2024-09-13T14:46:04.089796+0000","last_undegraded":"2024-09-13T14:46:04.089796+0000","last_fullsized":"2024-09-13T14:46:04.089796+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:32:24.990047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"170'9140","reported_seq":9206,"reported_epoch":170,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:46:13.577163+0000","last_change":"2024-09-13T14:46:04.250329+0000","last_active":"2024-09-13T14:46:13.577163+0000","last_peered":"2024-09-13T14:46:13.577163+0000","last_clean":"2024-09-13T14:46:02.075870+0000","last_became_active":"2024-09-13T14:46:04.217678+0000","last_became_peered":"2024-09-13T14:46:04.217678+0000","last_unstale":"2024-09-13T14:46:13.577163+0000","last_undegraded":"2024-09-13T14:46:13.577163+0000","last_fullsized":"2024-09-13T14:46:13.577163+0000","mapping_epoch":169,"log_start":"152'3224","ondisk_log_start":"152'3224","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":5916,"log_dups_size":0,"ondisk_log_size":5916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376569856,"num_objects":5746,"num_object_clones":0,"num_object_copies":17238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5746,"num_objects_unfound":0,"num_objects_dirty":5746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9140,"num_write_kb":476352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":5746}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":144,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.081304+0000","last_change":"2024-09-13T14:39:42.765558+0000","last_active":"2024-09-13T14:46:03.081304+0000","last_peered":"2024-09-13T14:46:03.081304+0000","last_clean":"2024-09-13T14:46:03.081304+0000","last_became_active":"2024-09-13T14:39:42.765246+0000","last_became_peered":"2024-09-13T14:39:42.765246+0000","last_unstale":"2024-09-13T14:46:03.081304+0000","last_undegraded":"2024-09-13T14:46:03.081304+0000","last_fullsized":"2024-09-13T14:46:03.081304+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:08.640610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":200,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.086346+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:46:04.086346+0000","last_peered":"2024-09-13T14:46:04.086346+0000","last_clean":"2024-09-13T14:46:04.086346+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:46:04.086346+0000","last_undegraded":"2024-09-13T14:46:04.086346+0000","last_fullsized":"2024-09-13T14:46:04.086346+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"170'8705","reported_seq":8749,"reported_epoch":170,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:46:11.732771+0000","last_change":"2024-09-13T14:46:04.223887+0000","last_active":"2024-09-13T14:46:11.732771+0000","last_peered":"2024-09-13T14:46:11.732771+0000","last_clean":"2024-09-13T14:46:02.000208+0000","last_became_active":"2024-09-13T14:46:04.206067+0000","last_became_peered":"2024-09-13T14:46:04.206067+0000","last_unstale":"2024-09-13T14:46:11.732771+0000","last_undegraded":"2024-09-13T14:46:11.732771+0000","last_fullsized":"2024-09-13T14:46:11.732771+0000","mapping_epoch":169,"log_start":"150'2014","ondisk_log_start":"150'2014","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":6691,"log_dups_size":0,"ondisk_log_size":6691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:20:56.878960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368640000,"num_objects":5625,"num_object_clones":0,"num_object_copies":16875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5625,"num_objects_unfound":0,"num_objects_dirty":5625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8705,"num_write_kb":458560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,2],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":5625}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":307,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.104262+0000","last_change":"2024-09-13T14:46:03.103727+0000","last_active":"2024-09-13T14:46:03.104262+0000","last_peered":"2024-09-13T14:46:03.104262+0000","last_clean":"2024-09-13T14:46:03.104262+0000","last_became_active":"2024-09-13T14:46:03.103343+0000","last_became_peered":"2024-09-13T14:46:03.103343+0000","last_unstale":"2024-09-13T14:46:03.104262+0000","last_undegraded":"2024-09-13T14:46:03.104262+0000","last_fullsized":"2024-09-13T14:46:03.104262+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:07:39.373856+0000","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":203,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.085476+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:46:04.085476+0000","last_peered":"2024-09-13T14:46:04.085476+0000","last_clean":"2024-09-13T14:46:04.085476+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:46:04.085476+0000","last_undegraded":"2024-09-13T14:46:04.085476+0000","last_fullsized":"2024-09-13T14:46:04.085476+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":144,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.080820+0000","last_change":"2024-09-13T14:39:42.775584+0000","last_active":"2024-09-13T14:46:03.080820+0000","last_peered":"2024-09-13T14:46:03.080820+0000","last_clean":"2024-09-13T14:46:03.080820+0000","last_became_active":"2024-09-13T14:39:42.775254+0000","last_became_peered":"2024-09-13T14:39:42.775254+0000","last_unstale":"2024-09-13T14:46:03.080820+0000","last_undegraded":"2024-09-13T14:46:03.080820+0000","last_fullsized":"2024-09-13T14:46:03.080820+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:44.607098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"170'8922","reported_seq":8987,"reported_epoch":170,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:46:12.396849+0000","last_change":"2024-09-13T14:46:04.247402+0000","last_active":"2024-09-13T14:46:12.396849+0000","last_peered":"2024-09-13T14:46:12.396849+0000","last_clean":"2024-09-13T14:46:02.036011+0000","last_became_active":"2024-09-13T14:46:04.221726+0000","last_became_peered":"2024-09-13T14:46:04.221726+0000","last_unstale":"2024-09-13T14:46:12.396849+0000","last_undegraded":"2024-09-13T14:46:12.396849+0000","last_fullsized":"2024-09-13T14:46:12.396849+0000","mapping_epoch":169,"log_start":"150'1322","ondisk_log_start":"150'1322","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":7600,"log_dups_size":0,"ondisk_log_size":7600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374079488,"num_objects":5708,"num_object_clones":0,"num_object_copies":17124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5708,"num_objects_unfound":0,"num_objects_dirty":5708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8922,"num_write_kb":468160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,2],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":5708}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":77,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.079730+0000","last_change":"2024-09-13T14:40:27.868402+0000","last_active":"2024-09-13T14:46:03.079730+0000","last_peered":"2024-09-13T14:46:03.079730+0000","last_clean":"2024-09-13T14:46:03.079730+0000","last_became_active":"2024-09-13T14:40:22.966301+0000","last_became_peered":"2024-09-13T14:40:22.966301+0000","last_unstale":"2024-09-13T14:46:03.079730+0000","last_undegraded":"2024-09-13T14:46:03.079730+0000","last_fullsized":"2024-09-13T14:46:03.079730+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:18.660186+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":126,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.080638+0000","last_change":"2024-09-13T14:39:42.776751+0000","last_active":"2024-09-13T14:46:03.080638+0000","last_peered":"2024-09-13T14:46:03.080638+0000","last_clean":"2024-09-13T14:46:03.080638+0000","last_became_active":"2024-09-13T14:39:42.774855+0000","last_became_peered":"2024-09-13T14:39:42.774855+0000","last_unstale":"2024-09-13T14:46:03.080638+0000","last_undegraded":"2024-09-13T14:46:03.080638+0000","last_fullsized":"2024-09-13T14:46:03.080638+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:11.022013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":151,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.081156+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:46:03.081156+0000","last_peered":"2024-09-13T14:46:03.081156+0000","last_clean":"2024-09-13T14:46:03.081156+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:46:03.081156+0000","last_undegraded":"2024-09-13T14:46:03.081156+0000","last_fullsized":"2024-09-13T14:46:03.081156+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"170'8801","reported_seq":8862,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:12.366738+0000","last_change":"2024-09-13T14:43:58.298605+0000","last_active":"2024-09-13T14:46:12.366738+0000","last_peered":"2024-09-13T14:46:12.366738+0000","last_clean":"2024-09-13T14:46:12.366738+0000","last_became_active":"2024-09-13T14:43:58.298341+0000","last_became_peered":"2024-09-13T14:43:58.298341+0000","last_unstale":"2024-09-13T14:46:12.366738+0000","last_undegraded":"2024-09-13T14:46:12.366738+0000","last_fullsized":"2024-09-13T14:46:12.366738+0000","mapping_epoch":145,"log_start":"150'2100","ondisk_log_start":"150'2100","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":6701,"log_dups_size":0,"ondisk_log_size":6701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:16.656493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370081792,"num_objects":5647,"num_object_clones":0,"num_object_copies":16941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8801,"num_write_kb":462336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":186,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.085775+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:46:04.085775+0000","last_peered":"2024-09-13T14:46:04.085775+0000","last_clean":"2024-09-13T14:46:04.085775+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:46:04.085775+0000","last_undegraded":"2024-09-13T14:46:04.085775+0000","last_fullsized":"2024-09-13T14:46:04.085775+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":144,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.082437+0000","last_change":"2024-09-13T14:39:42.776187+0000","last_active":"2024-09-13T14:46:03.082437+0000","last_peered":"2024-09-13T14:46:03.082437+0000","last_clean":"2024-09-13T14:46:03.082437+0000","last_became_active":"2024-09-13T14:39:42.775751+0000","last_became_peered":"2024-09-13T14:39:42.775751+0000","last_unstale":"2024-09-13T14:46:03.082437+0000","last_undegraded":"2024-09-13T14:46:03.082437+0000","last_fullsized":"2024-09-13T14:46:03.082437+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:35.155717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"170'8869","reported_seq":8902,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:11.846905+0000","last_change":"2024-09-13T14:43:58.298997+0000","last_active":"2024-09-13T14:46:11.846905+0000","last_peered":"2024-09-13T14:46:11.846905+0000","last_clean":"2024-09-13T14:46:11.846905+0000","last_became_active":"2024-09-13T14:43:58.298647+0000","last_became_peered":"2024-09-13T14:43:58.298647+0000","last_unstale":"2024-09-13T14:46:11.846905+0000","last_undegraded":"2024-09-13T14:46:11.846905+0000","last_fullsized":"2024-09-13T14:46:11.846905+0000","mapping_epoch":145,"log_start":"150'1998","ondisk_log_start":"150'1998","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":6871,"log_dups_size":0,"ondisk_log_size":6871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:38:43.997227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372244504,"num_objects":5681,"num_object_clones":0,"num_object_copies":17043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5681,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8869,"num_write_kb":465537,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":70,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.079960+0000","last_change":"2024-09-13T14:40:25.861254+0000","last_active":"2024-09-13T14:46:03.079960+0000","last_peered":"2024-09-13T14:46:03.079960+0000","last_clean":"2024-09-13T14:46:03.079960+0000","last_became_active":"2024-09-13T14:40:22.957614+0000","last_became_peered":"2024-09-13T14:40:22.957614+0000","last_unstale":"2024-09-13T14:46:03.079960+0000","last_undegraded":"2024-09-13T14:46:03.079960+0000","last_fullsized":"2024-09-13T14:46:03.079960+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:53.388190+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.086129+0000","last_change":"2024-09-13T14:46:03.085991+0000","last_active":"2024-09-13T14:46:04.086129+0000","last_peered":"2024-09-13T14:46:04.086129+0000","last_clean":"2024-09-13T14:46:04.086129+0000","last_became_active":"2024-09-13T14:46:03.085652+0000","last_became_peered":"2024-09-13T14:46:03.085652+0000","last_unstale":"2024-09-13T14:46:04.086129+0000","last_undegraded":"2024-09-13T14:46:04.086129+0000","last_fullsized":"2024-09-13T14:46:04.086129+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:39:33.762366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":188,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.089526+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:46:04.089526+0000","last_peered":"2024-09-13T14:46:04.089526+0000","last_clean":"2024-09-13T14:46:04.089526+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:46:04.089526+0000","last_undegraded":"2024-09-13T14:46:04.089526+0000","last_fullsized":"2024-09-13T14:46:04.089526+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"170'8789","reported_seq":8821,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:11.860114+0000","last_change":"2024-09-13T14:43:58.298370+0000","last_active":"2024-09-13T14:46:11.860114+0000","last_peered":"2024-09-13T14:46:11.860114+0000","last_clean":"2024-09-13T14:46:11.860114+0000","last_became_active":"2024-09-13T14:43:58.298041+0000","last_became_peered":"2024-09-13T14:43:58.298041+0000","last_unstale":"2024-09-13T14:46:11.860114+0000","last_undegraded":"2024-09-13T14:46:11.860114+0000","last_fullsized":"2024-09-13T14:46:11.860114+0000","mapping_epoch":145,"log_start":"150'1927","ondisk_log_start":"150'1927","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":6862,"log_dups_size":0,"ondisk_log_size":6862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:28:59.876959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373489664,"num_objects":5699,"num_object_clones":0,"num_object_copies":17097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8789,"num_write_kb":463616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":288,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.085496+0000","last_change":"2024-09-13T14:40:22.639864+0000","last_active":"2024-09-13T14:46:03.085496+0000","last_peered":"2024-09-13T14:46:03.085496+0000","last_clean":"2024-09-13T14:46:03.085496+0000","last_became_active":"2024-09-13T14:40:20.926503+0000","last_became_peered":"2024-09-13T14:40:20.926503+0000","last_unstale":"2024-09-13T14:46:03.085496+0000","last_undegraded":"2024-09-13T14:46:03.085496+0000","last_fullsized":"2024-09-13T14:46:03.085496+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:22.639760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:22.639760+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:08.523563+0000","scrub_duration":0.0051120610000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":696,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.080195+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:46:03.080195+0000","last_peered":"2024-09-13T14:46:03.080195+0000","last_clean":"2024-09-13T14:46:03.080195+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:46:03.080195+0000","last_undegraded":"2024-09-13T14:46:03.080195+0000","last_fullsized":"2024-09-13T14:46:03.080195+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":41,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.084128+0000","last_change":"2024-09-13T14:46:03.083838+0000","last_active":"2024-09-13T14:46:03.084128+0000","last_peered":"2024-09-13T14:46:03.084128+0000","last_clean":"2024-09-13T14:46:03.084128+0000","last_became_active":"2024-09-13T14:46:03.083541+0000","last_became_peered":"2024-09-13T14:46:03.083541+0000","last_unstale":"2024-09-13T14:46:03.084128+0000","last_undegraded":"2024-09-13T14:46:03.084128+0000","last_fullsized":"2024-09-13T14:46:03.084128+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:41:06.495295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":76,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.081017+0000","last_change":"2024-09-13T14:39:42.772096+0000","last_active":"2024-09-13T14:46:03.081017+0000","last_peered":"2024-09-13T14:46:03.081017+0000","last_clean":"2024-09-13T14:46:03.081017+0000","last_became_active":"2024-09-13T14:39:42.771762+0000","last_became_peered":"2024-09-13T14:39:42.771762+0000","last_unstale":"2024-09-13T14:46:03.081017+0000","last_undegraded":"2024-09-13T14:46:03.081017+0000","last_fullsized":"2024-09-13T14:46:03.081017+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:16.172571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"170'9045","reported_seq":9111,"reported_epoch":170,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:46:12.269571+0000","last_change":"2024-09-13T14:46:04.243368+0000","last_active":"2024-09-13T14:46:12.269571+0000","last_peered":"2024-09-13T14:46:12.269571+0000","last_clean":"2024-09-13T14:46:02.067035+0000","last_became_active":"2024-09-13T14:46:04.204753+0000","last_became_peered":"2024-09-13T14:46:04.204753+0000","last_unstale":"2024-09-13T14:46:12.269571+0000","last_undegraded":"2024-09-13T14:46:12.269571+0000","last_fullsized":"2024-09-13T14:46:12.269571+0000","mapping_epoch":169,"log_start":"152'3412","ondisk_log_start":"152'3412","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":5633,"log_dups_size":0,"ondisk_log_size":5633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380698624,"num_objects":5809,"num_object_clones":0,"num_object_copies":17427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5809,"num_objects_unfound":0,"num_objects_dirty":5809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9045,"num_write_kb":475328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,3],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":5809}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":319,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.087762+0000","last_change":"2024-09-13T14:40:23.784403+0000","last_active":"2024-09-13T14:46:04.087762+0000","last_peered":"2024-09-13T14:46:04.087762+0000","last_clean":"2024-09-13T14:46:04.087762+0000","last_became_active":"2024-09-13T14:40:22.951246+0000","last_became_peered":"2024-09-13T14:40:22.951246+0000","last_unstale":"2024-09-13T14:46:04.087762+0000","last_undegraded":"2024-09-13T14:46:04.087762+0000","last_fullsized":"2024-09-13T14:46:04.087762+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.784263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.784263+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:52.491054+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":381,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.085827+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:46:03.085827+0000","last_peered":"2024-09-13T14:46:03.085827+0000","last_clean":"2024-09-13T14:46:03.085827+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:46:03.085827+0000","last_undegraded":"2024-09-13T14:46:03.085827+0000","last_fullsized":"2024-09-13T14:46:03.085827+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":215,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.087253+0000","last_change":"2024-09-13T14:46:03.108667+0000","last_active":"2024-09-13T14:46:04.087253+0000","last_peered":"2024-09-13T14:46:04.087253+0000","last_clean":"2024-09-13T14:46:04.087253+0000","last_became_active":"2024-09-13T14:46:03.107794+0000","last_became_peered":"2024-09-13T14:46:03.107794+0000","last_unstale":"2024-09-13T14:46:04.087253+0000","last_undegraded":"2024-09-13T14:46:04.087253+0000","last_fullsized":"2024-09-13T14:46:04.087253+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:24.876890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":218,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.086752+0000","last_change":"2024-09-13T14:46:03.097285+0000","last_active":"2024-09-13T14:46:04.086752+0000","last_peered":"2024-09-13T14:46:04.086752+0000","last_clean":"2024-09-13T14:46:04.086752+0000","last_became_active":"2024-09-13T14:46:03.096974+0000","last_became_peered":"2024-09-13T14:46:03.096974+0000","last_unstale":"2024-09-13T14:46:04.086752+0000","last_undegraded":"2024-09-13T14:46:04.086752+0000","last_fullsized":"2024-09-13T14:46:04.086752+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:00:14.884033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"170'8836","reported_seq":8869,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:11.744190+0000","last_change":"2024-09-13T14:43:58.299024+0000","last_active":"2024-09-13T14:46:11.744190+0000","last_peered":"2024-09-13T14:46:11.744190+0000","last_clean":"2024-09-13T14:46:11.744190+0000","last_became_active":"2024-09-13T14:43:58.298323+0000","last_became_peered":"2024-09-13T14:43:58.298323+0000","last_unstale":"2024-09-13T14:46:11.744190+0000","last_undegraded":"2024-09-13T14:46:11.744190+0000","last_fullsized":"2024-09-13T14:46:11.744190+0000","mapping_epoch":145,"log_start":"149'500","ondisk_log_start":"149'500","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":8336,"log_dups_size":0,"ondisk_log_size":8336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:32.680552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379322368,"num_objects":5788,"num_object_clones":0,"num_object_copies":17364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8836,"num_write_kb":467968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":209,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.086843+0000","last_change":"2024-09-13T14:46:03.101176+0000","last_active":"2024-09-13T14:46:04.086843+0000","last_peered":"2024-09-13T14:46:04.086843+0000","last_clean":"2024-09-13T14:46:04.086843+0000","last_became_active":"2024-09-13T14:46:03.100119+0000","last_became_peered":"2024-09-13T14:46:03.100119+0000","last_unstale":"2024-09-13T14:46:04.086843+0000","last_undegraded":"2024-09-13T14:46:04.086843+0000","last_fullsized":"2024-09-13T14:46:04.086843+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:45:49.425578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:45:49.425578+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:43:56.276195+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":643,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.097210+0000","last_change":"2024-09-13T14:46:03.135856+0000","last_active":"2024-09-13T14:46:04.097210+0000","last_peered":"2024-09-13T14:46:04.097210+0000","last_clean":"2024-09-13T14:46:04.097210+0000","last_became_active":"2024-09-13T14:46:03.107831+0000","last_became_peered":"2024-09-13T14:46:03.107831+0000","last_unstale":"2024-09-13T14:46:04.097210+0000","last_undegraded":"2024-09-13T14:46:04.097210+0000","last_fullsized":"2024-09-13T14:46:04.097210+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:39:49.725532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":195,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.086231+0000","last_change":"2024-09-13T14:39:42.769691+0000","last_active":"2024-09-13T14:46:04.086231+0000","last_peered":"2024-09-13T14:46:04.086231+0000","last_clean":"2024-09-13T14:46:04.086231+0000","last_became_active":"2024-09-13T14:39:42.769266+0000","last_became_peered":"2024-09-13T14:39:42.769266+0000","last_unstale":"2024-09-13T14:46:04.086231+0000","last_undegraded":"2024-09-13T14:46:04.086231+0000","last_fullsized":"2024-09-13T14:46:04.086231+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:11.637732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":221,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.089317+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:46:04.089317+0000","last_peered":"2024-09-13T14:46:04.089317+0000","last_clean":"2024-09-13T14:46:04.089317+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:46:04.089317+0000","last_undegraded":"2024-09-13T14:46:04.089317+0000","last_fullsized":"2024-09-13T14:46:04.089317+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"170'8967","reported_seq":9028,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:12.361893+0000","last_change":"2024-09-13T14:43:58.301790+0000","last_active":"2024-09-13T14:46:12.361893+0000","last_peered":"2024-09-13T14:46:12.361893+0000","last_clean":"2024-09-13T14:46:12.361893+0000","last_became_active":"2024-09-13T14:43:58.301504+0000","last_became_peered":"2024-09-13T14:43:58.301504+0000","last_unstale":"2024-09-13T14:46:12.361893+0000","last_undegraded":"2024-09-13T14:46:12.361893+0000","last_fullsized":"2024-09-13T14:46:12.361893+0000","mapping_epoch":145,"log_start":"150'2300","ondisk_log_start":"150'2300","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":6667,"log_dups_size":0,"ondisk_log_size":6667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:05:44.303022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378863616,"num_objects":5781,"num_object_clones":0,"num_object_copies":17343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8967,"num_write_kb":471936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":70,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.079794+0000","last_change":"2024-09-13T14:40:26.896691+0000","last_active":"2024-09-13T14:46:03.079794+0000","last_peered":"2024-09-13T14:46:03.079794+0000","last_clean":"2024-09-13T14:46:03.079794+0000","last_became_active":"2024-09-13T14:40:22.949772+0000","last_became_peered":"2024-09-13T14:40:22.949772+0000","last_unstale":"2024-09-13T14:46:03.079794+0000","last_undegraded":"2024-09-13T14:46:03.079794+0000","last_fullsized":"2024-09-13T14:46:03.079794+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:26.896613+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:26.896613+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:01.615433+0000","scrub_duration":0.004120828,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":760,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.081187+0000","last_change":"2024-09-13T14:39:42.805055+0000","last_active":"2024-09-13T14:46:03.081187+0000","last_peered":"2024-09-13T14:46:03.081187+0000","last_clean":"2024-09-13T14:46:03.081187+0000","last_became_active":"2024-09-13T14:39:42.762456+0000","last_became_peered":"2024-09-13T14:39:42.762456+0000","last_unstale":"2024-09-13T14:46:03.081187+0000","last_undegraded":"2024-09-13T14:46:03.081187+0000","last_fullsized":"2024-09-13T14:46:03.081187+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:54:40.839991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":145,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.080895+0000","last_change":"2024-09-13T14:39:42.775159+0000","last_active":"2024-09-13T14:46:03.080895+0000","last_peered":"2024-09-13T14:46:03.080895+0000","last_clean":"2024-09-13T14:46:03.080895+0000","last_became_active":"2024-09-13T14:39:42.774841+0000","last_became_peered":"2024-09-13T14:39:42.774841+0000","last_unstale":"2024-09-13T14:46:03.080895+0000","last_undegraded":"2024-09-13T14:46:03.080895+0000","last_fullsized":"2024-09-13T14:46:03.080895+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:35:45.579397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":206,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.115218+0000","last_change":"2024-09-13T14:46:03.095898+0000","last_active":"2024-09-13T14:46:04.115218+0000","last_peered":"2024-09-13T14:46:04.115218+0000","last_clean":"2024-09-13T14:46:04.115218+0000","last_became_active":"2024-09-13T14:46:03.095403+0000","last_became_peered":"2024-09-13T14:46:03.095403+0000","last_unstale":"2024-09-13T14:46:04.115218+0000","last_undegraded":"2024-09-13T14:46:04.115218+0000","last_fullsized":"2024-09-13T14:46:04.115218+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:42:23.850698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"170'8961","reported_seq":11721,"reported_epoch":170,"state":"active+remapped+backfilling","last_fresh":"2024-09-13T14:46:13.586976+0000","last_change":"2024-09-13T14:46:04.231563+0000","last_active":"2024-09-13T14:46:13.586976+0000","last_peered":"2024-09-13T14:46:13.586976+0000","last_clean":"2024-09-13T14:46:02.062508+0000","last_became_active":"2024-09-13T14:46:04.209803+0000","last_became_peered":"2024-09-13T14:46:04.209803+0000","last_unstale":"2024-09-13T14:46:13.586976+0000","last_undegraded":"2024-09-13T14:46:13.586976+0000","last_fullsized":"2024-09-13T14:46:13.586976+0000","mapping_epoch":169,"log_start":"152'2982","ondisk_log_start":"152'2982","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":5979,"log_dups_size":0,"ondisk_log_size":5979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369819648,"num_objects":5643,"num_object_clones":0,"num_object_copies":16929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4346,"num_objects_unfound":0,"num_objects_dirty":5643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8961,"num_write_kb":467328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1346,"num_bytes_recovered":88211456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,2],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":5643}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":252,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.085443+0000","last_change":"2024-09-13T14:45:54.435571+0000","last_active":"2024-09-13T14:46:03.085443+0000","last_peered":"2024-09-13T14:46:03.085443+0000","last_clean":"2024-09-13T14:46:03.085443+0000","last_became_active":"2024-09-13T14:45:34.027765+0000","last_became_peered":"2024-09-13T14:45:34.027765+0000","last_unstale":"2024-09-13T14:46:03.085443+0000","last_undegraded":"2024-09-13T14:46:03.085443+0000","last_fullsized":"2024-09-13T14:46:03.085443+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:45:54.435491+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:45:54.435491+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:44.953646+0000","scrub_duration":0.009757333,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":151,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.081791+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:46:03.081791+0000","last_peered":"2024-09-13T14:46:03.081791+0000","last_clean":"2024-09-13T14:46:03.081791+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:46:03.081791+0000","last_undegraded":"2024-09-13T14:46:03.081791+0000","last_fullsized":"2024-09-13T14:46:03.081791+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":705,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.115319+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:46:04.115319+0000","last_peered":"2024-09-13T14:46:04.115319+0000","last_clean":"2024-09-13T14:46:04.115319+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:46:04.115319+0000","last_undegraded":"2024-09-13T14:46:04.115319+0000","last_fullsized":"2024-09-13T14:46:04.115319+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":177,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.093956+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:46:04.093956+0000","last_peered":"2024-09-13T14:46:04.093956+0000","last_clean":"2024-09-13T14:46:04.093956+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:46:04.093956+0000","last_undegraded":"2024-09-13T14:46:04.093956+0000","last_fullsized":"2024-09-13T14:46:04.093956+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"170'8865","reported_seq":10182,"reported_epoch":170,"state":"active+remapped+backfilling","last_fresh":"2024-09-13T14:46:12.320884+0000","last_change":"2024-09-13T14:46:04.259046+0000","last_active":"2024-09-13T14:46:12.320884+0000","last_peered":"2024-09-13T14:46:12.320884+0000","last_clean":"2024-09-13T14:46:02.061947+0000","last_became_active":"2024-09-13T14:46:04.201329+0000","last_became_peered":"2024-09-13T14:46:04.201329+0000","last_unstale":"2024-09-13T14:46:12.320884+0000","last_undegraded":"2024-09-13T14:46:12.320884+0000","last_fullsized":"2024-09-13T14:46:12.320884+0000","mapping_epoch":169,"log_start":"152'3240","ondisk_log_start":"152'3240","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":5625,"log_dups_size":0,"ondisk_log_size":5625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367591424,"num_objects":5609,"num_object_clones":0,"num_object_copies":16827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4998,"num_objects_unfound":0,"num_objects_dirty":5609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8865,"num_write_kb":463168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":625,"num_bytes_recovered":40960000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,2],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":5609}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":281,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.085740+0000","last_change":"2024-09-13T14:46:03.100057+0000","last_active":"2024-09-13T14:46:04.085740+0000","last_peered":"2024-09-13T14:46:04.085740+0000","last_clean":"2024-09-13T14:46:04.085740+0000","last_became_active":"2024-09-13T14:46:03.099686+0000","last_became_peered":"2024-09-13T14:46:03.099686+0000","last_unstale":"2024-09-13T14:46:04.085740+0000","last_undegraded":"2024-09-13T14:46:04.085740+0000","last_fullsized":"2024-09-13T14:46:04.085740+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.334379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.334379+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:25:20.311185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":195,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.115286+0000","last_change":"2024-09-13T14:46:03.098357+0000","last_active":"2024-09-13T14:46:04.115286+0000","last_peered":"2024-09-13T14:46:04.115286+0000","last_clean":"2024-09-13T14:46:04.115286+0000","last_became_active":"2024-09-13T14:46:03.097749+0000","last_became_peered":"2024-09-13T14:46:03.097749+0000","last_unstale":"2024-09-13T14:46:04.115286+0000","last_undegraded":"2024-09-13T14:46:04.115286+0000","last_fullsized":"2024-09-13T14:46:04.115286+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:30.939741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":383,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.085752+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:46:03.085752+0000","last_peered":"2024-09-13T14:46:03.085752+0000","last_clean":"2024-09-13T14:46:03.085752+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:46:03.085752+0000","last_undegraded":"2024-09-13T14:46:03.085752+0000","last_fullsized":"2024-09-13T14:46:03.085752+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":190,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.086304+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:46:04.086304+0000","last_peered":"2024-09-13T14:46:04.086304+0000","last_clean":"2024-09-13T14:46:04.086304+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:46:04.086304+0000","last_undegraded":"2024-09-13T14:46:04.086304+0000","last_fullsized":"2024-09-13T14:46:04.086304+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"170'8945","reported_seq":14779,"reported_epoch":170,"state":"active+remapped+backfilling","last_fresh":"2024-09-13T14:46:11.761709+0000","last_change":"2024-09-13T14:46:04.212196+0000","last_active":"2024-09-13T14:46:11.761709+0000","last_peered":"2024-09-13T14:46:11.761709+0000","last_clean":"2024-09-13T14:46:02.039938+0000","last_became_active":"2024-09-13T14:46:04.184293+0000","last_became_peered":"2024-09-13T14:46:04.184293+0000","last_unstale":"2024-09-13T14:46:11.761709+0000","last_undegraded":"2024-09-13T14:46:11.761709+0000","last_fullsized":"2024-09-13T14:46:11.761709+0000","mapping_epoch":169,"log_start":"150'2249","ondisk_log_start":"150'2249","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":6696,"log_dups_size":0,"ondisk_log_size":6696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:59:08.970091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383582208,"num_objects":5853,"num_object_clones":0,"num_object_copies":17559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3033,"num_objects_unfound":0,"num_objects_dirty":5853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8945,"num_write_kb":473536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2896,"num_bytes_recovered":189792256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":5853}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":81,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.103718+0000","last_change":"2024-09-13T14:46:03.103249+0000","last_active":"2024-09-13T14:46:03.103718+0000","last_peered":"2024-09-13T14:46:03.103718+0000","last_clean":"2024-09-13T14:46:03.103718+0000","last_became_active":"2024-09-13T14:46:03.102878+0000","last_became_peered":"2024-09-13T14:46:03.102878+0000","last_unstale":"2024-09-13T14:46:03.103718+0000","last_undegraded":"2024-09-13T14:46:03.103718+0000","last_fullsized":"2024-09-13T14:46:03.103718+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:40:30.500046+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:23:44.432499+0000","scrub_duration":0.00248832,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":203,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.103358+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:46:04.103358+0000","last_peered":"2024-09-13T14:46:04.103358+0000","last_clean":"2024-09-13T14:46:04.103358+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:46:04.103358+0000","last_undegraded":"2024-09-13T14:46:04.103358+0000","last_fullsized":"2024-09-13T14:46:04.103358+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":526,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.086896+0000","last_change":"2024-09-13T14:39:44.990216+0000","last_active":"2024-09-13T14:46:04.086896+0000","last_peered":"2024-09-13T14:46:04.086896+0000","last_clean":"2024-09-13T14:46:04.086896+0000","last_became_active":"2024-09-13T14:39:44.989433+0000","last_became_peered":"2024-09-13T14:39:44.989433+0000","last_unstale":"2024-09-13T14:46:04.086896+0000","last_undegraded":"2024-09-13T14:46:04.086896+0000","last_fullsized":"2024-09-13T14:46:04.086896+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:00.426042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":72,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.112057+0000","last_change":"2024-09-13T14:46:03.100714+0000","last_active":"2024-09-13T14:46:04.112057+0000","last_peered":"2024-09-13T14:46:04.112057+0000","last_clean":"2024-09-13T14:46:04.112057+0000","last_became_active":"2024-09-13T14:46:03.099689+0000","last_became_peered":"2024-09-13T14:46:03.099689+0000","last_unstale":"2024-09-13T14:46:04.112057+0000","last_undegraded":"2024-09-13T14:46:04.112057+0000","last_fullsized":"2024-09-13T14:46:04.112057+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:40:31.297028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.3","version":"170'8924","reported_seq":8957,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:11.745455+0000","last_change":"2024-09-13T14:43:58.299991+0000","last_active":"2024-09-13T14:46:11.745455+0000","last_peered":"2024-09-13T14:46:11.745455+0000","last_clean":"2024-09-13T14:46:11.745455+0000","last_became_active":"2024-09-13T14:43:58.299386+0000","last_became_peered":"2024-09-13T14:43:58.299386+0000","last_unstale":"2024-09-13T14:46:11.745455+0000","last_undegraded":"2024-09-13T14:46:11.745455+0000","last_fullsized":"2024-09-13T14:46:11.745455+0000","mapping_epoch":145,"log_start":"149'600","ondisk_log_start":"149'600","created":145,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:43:57.271267+0000","last_clean_scrub_stamp":"2024-09-13T14:43:57.271267+0000","objects_scrubbed":0,"log_size":8324,"log_dups_size":0,"ondisk_log_size":8324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:24:28.041082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374865920,"num_objects":5720,"num_object_clones":0,"num_object_copies":17160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8924,"num_write_kb":468608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":133,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.084493+0000","last_change":"2024-09-13T14:40:24.124750+0000","last_active":"2024-09-13T14:46:04.084493+0000","last_peered":"2024-09-13T14:46:04.084493+0000","last_clean":"2024-09-13T14:46:04.084493+0000","last_became_active":"2024-09-13T14:40:22.958820+0000","last_became_peered":"2024-09-13T14:40:22.958820+0000","last_unstale":"2024-09-13T14:46:04.084493+0000","last_undegraded":"2024-09-13T14:46:04.084493+0000","last_fullsized":"2024-09-13T14:46:04.084493+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:24.124531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:24.124531+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:47:30.267886+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":211,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.086498+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:46:04.086498+0000","last_peered":"2024-09-13T14:46:04.086498+0000","last_clean":"2024-09-13T14:46:04.086498+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:46:04.086498+0000","last_undegraded":"2024-09-13T14:46:04.086498+0000","last_fullsized":"2024-09-13T14:46:04.086498+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1042,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.096220+0000","last_change":"2024-09-13T14:45:28.982978+0000","last_active":"2024-09-13T14:46:04.096220+0000","last_peered":"2024-09-13T14:46:04.096220+0000","last_clean":"2024-09-13T14:46:04.096220+0000","last_became_active":"2024-09-13T14:45:28.982443+0000","last_became_peered":"2024-09-13T14:45:28.982443+0000","last_unstale":"2024-09-13T14:46:04.096220+0000","last_undegraded":"2024-09-13T14:46:04.096220+0000","last_fullsized":"2024-09-13T14:46:04.096220+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:01:18.450505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":127,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.080114+0000","last_change":"2024-09-13T14:39:42.769954+0000","last_active":"2024-09-13T14:46:03.080114+0000","last_peered":"2024-09-13T14:46:03.080114+0000","last_clean":"2024-09-13T14:46:03.080114+0000","last_became_active":"2024-09-13T14:39:42.769635+0000","last_became_peered":"2024-09-13T14:39:42.769635+0000","last_unstale":"2024-09-13T14:46:03.080114+0000","last_undegraded":"2024-09-13T14:46:03.080114+0000","last_fullsized":"2024-09-13T14:46:03.080114+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:51.588071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":144,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.080728+0000","last_change":"2024-09-13T14:39:42.775357+0000","last_active":"2024-09-13T14:46:03.080728+0000","last_peered":"2024-09-13T14:46:03.080728+0000","last_clean":"2024-09-13T14:46:03.080728+0000","last_became_active":"2024-09-13T14:39:42.775041+0000","last_became_peered":"2024-09-13T14:39:42.775041+0000","last_unstale":"2024-09-13T14:46:03.080728+0000","last_undegraded":"2024-09-13T14:46:03.080728+0000","last_fullsized":"2024-09-13T14:46:03.080728+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:44.768910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":202,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.085870+0000","last_change":"2024-09-13T14:39:42.787151+0000","last_active":"2024-09-13T14:46:04.085870+0000","last_peered":"2024-09-13T14:46:04.085870+0000","last_clean":"2024-09-13T14:46:04.085870+0000","last_became_active":"2024-09-13T14:39:42.771712+0000","last_became_peered":"2024-09-13T14:39:42.771712+0000","last_unstale":"2024-09-13T14:46:04.085870+0000","last_undegraded":"2024-09-13T14:46:04.085870+0000","last_fullsized":"2024-09-13T14:46:04.085870+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:54.216663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":194,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.093803+0000","last_change":"2024-09-13T14:39:42.772974+0000","last_active":"2024-09-13T14:46:04.093803+0000","last_peered":"2024-09-13T14:46:04.093803+0000","last_clean":"2024-09-13T14:46:04.093803+0000","last_became_active":"2024-09-13T14:39:42.771449+0000","last_became_peered":"2024-09-13T14:39:42.771449+0000","last_unstale":"2024-09-13T14:46:04.093803+0000","last_undegraded":"2024-09-13T14:46:04.093803+0000","last_fullsized":"2024-09-13T14:46:04.093803+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:46.441079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":150,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.085634+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:46:03.085634+0000","last_peered":"2024-09-13T14:46:03.085634+0000","last_clean":"2024-09-13T14:46:03.085634+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:46:03.085634+0000","last_undegraded":"2024-09-13T14:46:03.085634+0000","last_fullsized":"2024-09-13T14:46:03.085634+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":134,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.081514+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:46:03.081514+0000","last_peered":"2024-09-13T14:46:03.081514+0000","last_clean":"2024-09-13T14:46:03.081514+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:46:03.081514+0000","last_undegraded":"2024-09-13T14:46:03.081514+0000","last_fullsized":"2024-09-13T14:46:03.081514+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":164,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.084311+0000","last_change":"2024-09-13T14:39:42.767764+0000","last_active":"2024-09-13T14:46:04.084311+0000","last_peered":"2024-09-13T14:46:04.084311+0000","last_clean":"2024-09-13T14:46:04.084311+0000","last_became_active":"2024-09-13T14:39:42.767450+0000","last_became_peered":"2024-09-13T14:39:42.767450+0000","last_unstale":"2024-09-13T14:46:04.084311+0000","last_undegraded":"2024-09-13T14:46:04.084311+0000","last_fullsized":"2024-09-13T14:46:04.084311+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:47.729382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":199,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.087049+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:46:04.087049+0000","last_peered":"2024-09-13T14:46:04.087049+0000","last_clean":"2024-09-13T14:46:04.087049+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:46:04.087049+0000","last_undegraded":"2024-09-13T14:46:04.087049+0000","last_fullsized":"2024-09-13T14:46:04.087049+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":213,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.084382+0000","last_change":"2024-09-13T14:39:42.772266+0000","last_active":"2024-09-13T14:46:04.084382+0000","last_peered":"2024-09-13T14:46:04.084382+0000","last_clean":"2024-09-13T14:46:04.084382+0000","last_became_active":"2024-09-13T14:39:42.771880+0000","last_became_peered":"2024-09-13T14:39:42.771880+0000","last_unstale":"2024-09-13T14:46:04.084382+0000","last_undegraded":"2024-09-13T14:46:04.084382+0000","last_fullsized":"2024-09-13T14:46:04.084382+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:41:01.160105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":124,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.081621+0000","last_change":"2024-09-13T14:39:42.773890+0000","last_active":"2024-09-13T14:46:03.081621+0000","last_peered":"2024-09-13T14:46:03.081621+0000","last_clean":"2024-09-13T14:46:03.081621+0000","last_became_active":"2024-09-13T14:39:42.773523+0000","last_became_peered":"2024-09-13T14:39:42.773523+0000","last_unstale":"2024-09-13T14:46:03.081621+0000","last_undegraded":"2024-09-13T14:46:03.081621+0000","last_fullsized":"2024-09-13T14:46:03.081621+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:13.156261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":224,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.084311+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:46:04.084311+0000","last_peered":"2024-09-13T14:46:04.084311+0000","last_clean":"2024-09-13T14:46:04.084311+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:46:04.084311+0000","last_undegraded":"2024-09-13T14:46:04.084311+0000","last_fullsized":"2024-09-13T14:46:04.084311+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":208,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.086677+0000","last_change":"2024-09-13T14:45:34.028233+0000","last_active":"2024-09-13T14:46:04.086677+0000","last_peered":"2024-09-13T14:46:04.086677+0000","last_clean":"2024-09-13T14:46:04.086677+0000","last_became_active":"2024-09-13T14:45:34.027877+0000","last_became_peered":"2024-09-13T14:45:34.027877+0000","last_unstale":"2024-09-13T14:46:04.086677+0000","last_undegraded":"2024-09-13T14:46:04.086677+0000","last_fullsized":"2024-09-13T14:46:04.086677+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:15:01.660511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":187,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.087451+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:46:04.087451+0000","last_peered":"2024-09-13T14:46:04.087451+0000","last_clean":"2024-09-13T14:46:04.087451+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:46:04.087451+0000","last_undegraded":"2024-09-13T14:46:04.087451+0000","last_fullsized":"2024-09-13T14:46:04.087451+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":220,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.085316+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:46:04.085316+0000","last_peered":"2024-09-13T14:46:04.085316+0000","last_clean":"2024-09-13T14:46:04.085316+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:46:04.085316+0000","last_undegraded":"2024-09-13T14:46:04.085316+0000","last_fullsized":"2024-09-13T14:46:04.085316+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":153,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.081707+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:46:03.081707+0000","last_peered":"2024-09-13T14:46:03.081707+0000","last_clean":"2024-09-13T14:46:03.081707+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:46:03.081707+0000","last_undegraded":"2024-09-13T14:46:03.081707+0000","last_fullsized":"2024-09-13T14:46:03.081707+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":120,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.115428+0000","last_change":"2024-09-13T14:45:25.927190+0000","last_active":"2024-09-13T14:46:04.115428+0000","last_peered":"2024-09-13T14:46:04.115428+0000","last_clean":"2024-09-13T14:46:04.115428+0000","last_became_active":"2024-09-13T14:45:25.926884+0000","last_became_peered":"2024-09-13T14:45:25.926884+0000","last_unstale":"2024-09-13T14:46:04.115428+0000","last_undegraded":"2024-09-13T14:46:04.115428+0000","last_fullsized":"2024-09-13T14:46:04.115428+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:47.904664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":192,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.086180+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:46:04.086180+0000","last_peered":"2024-09-13T14:46:04.086180+0000","last_clean":"2024-09-13T14:46:04.086180+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:46:04.086180+0000","last_undegraded":"2024-09-13T14:46:04.086180+0000","last_fullsized":"2024-09-13T14:46:04.086180+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":151,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.080656+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:46:03.080656+0000","last_peered":"2024-09-13T14:46:03.080656+0000","last_clean":"2024-09-13T14:46:03.080656+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:46:03.080656+0000","last_undegraded":"2024-09-13T14:46:03.080656+0000","last_fullsized":"2024-09-13T14:46:03.080656+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":136,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.081225+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:46:03.081225+0000","last_peered":"2024-09-13T14:46:03.081225+0000","last_clean":"2024-09-13T14:46:03.081225+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:46:03.081225+0000","last_undegraded":"2024-09-13T14:46:03.081225+0000","last_fullsized":"2024-09-13T14:46:03.081225+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":151,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.081620+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:46:03.081620+0000","last_peered":"2024-09-13T14:46:03.081620+0000","last_clean":"2024-09-13T14:46:03.081620+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:46:03.081620+0000","last_undegraded":"2024-09-13T14:46:03.081620+0000","last_fullsized":"2024-09-13T14:46:03.081620+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":161,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.081702+0000","last_change":"2024-09-13T14:39:42.778413+0000","last_active":"2024-09-13T14:46:03.081702+0000","last_peered":"2024-09-13T14:46:03.081702+0000","last_clean":"2024-09-13T14:46:03.081702+0000","last_became_active":"2024-09-13T14:39:42.766624+0000","last_became_peered":"2024-09-13T14:39:42.766624+0000","last_unstale":"2024-09-13T14:46:03.081702+0000","last_undegraded":"2024-09-13T14:46:03.081702+0000","last_fullsized":"2024-09-13T14:46:03.081702+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:13.435604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":125,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.079848+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:46:03.079848+0000","last_peered":"2024-09-13T14:46:03.079848+0000","last_clean":"2024-09-13T14:46:03.079848+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:46:03.079848+0000","last_undegraded":"2024-09-13T14:46:03.079848+0000","last_fullsized":"2024-09-13T14:46:03.079848+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":220,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.087665+0000","last_change":"2024-09-13T14:39:42.767451+0000","last_active":"2024-09-13T14:46:04.087665+0000","last_peered":"2024-09-13T14:46:04.087665+0000","last_clean":"2024-09-13T14:46:04.087665+0000","last_became_active":"2024-09-13T14:39:42.767086+0000","last_became_peered":"2024-09-13T14:39:42.767086+0000","last_unstale":"2024-09-13T14:46:04.087665+0000","last_undegraded":"2024-09-13T14:46:04.087665+0000","last_fullsized":"2024-09-13T14:46:04.087665+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:02.674069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":147,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.086439+0000","last_change":"2024-09-13T14:46:03.085583+0000","last_active":"2024-09-13T14:46:03.086439+0000","last_peered":"2024-09-13T14:46:03.086439+0000","last_clean":"2024-09-13T14:46:03.086439+0000","last_became_active":"2024-09-13T14:46:03.084180+0000","last_became_peered":"2024-09-13T14:46:03.084180+0000","last_unstale":"2024-09-13T14:46:03.086439+0000","last_undegraded":"2024-09-13T14:46:03.086439+0000","last_fullsized":"2024-09-13T14:46:03.086439+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:59:23.553923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":217,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-09-13T14:46:04.087003+0000","last_change":"2024-09-13T14:45:34.028913+0000","last_active":"2024-09-13T14:46:04.087003+0000","last_peered":"2024-09-13T14:46:04.087003+0000","last_clean":"2024-09-13T14:46:04.087003+0000","last_became_active":"2024-09-13T14:45:34.023671+0000","last_became_peered":"2024-09-13T14:45:34.023671+0000","last_unstale":"2024-09-13T14:46:04.087003+0000","last_undegraded":"2024-09-13T14:46:04.087003+0000","last_fullsized":"2024-09-13T14:46:04.087003+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:16:12.730935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6006898712,"num_objects":91659,"num_object_clones":0,"num_object_copies":274977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":41103,"num_objects_unfound":0,"num_objects_dirty":91659,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142793,"num_write_kb":7502401,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11252,"num_bytes_recovered":737411072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18635239424,"data_stored":18635227208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109670,"ondisk_log_size":109670,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":10218,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397747,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3429340,"kb_used_data":3127828,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":90315812,"statfs":{"total":95995035648,"available":92483391488,"internally_reserved":0,"allocated":3202895872,"data_stored":3201833177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":184683593976,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790205,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2225492,"kb_used_data":1915660,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91519660,"statfs":{"total":95995035648,"available":93716131840,"internally_reserved":0,"allocated":1961635840,"data_stored":1960577374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986434,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2449364,"kb_used_data":2152012,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":91295788,"statfs":{"total":95995035648,"available":93486886912,"internally_reserved":0,"allocated":2203660288,"data_stored":2202601555,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598882,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3955976,"kb_used_data":3681152,"kb_used_omap":6,"kb_used_meta":274617,"kb_avail":89789176,"statfs":{"total":95995035648,"available":91944116224,"internally_reserved":0,"allocated":3769499648,"data_stored":3768452885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":281208800},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762409,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3941336,"kb_used_data":3612560,"kb_used_omap":8,"kb_used_meta":328695,"kb_avail":89803816,"statfs":{"total":95995035648,"available":91959107584,"internally_reserved":0,"allocated":3699261440,"data_stored":3698203023,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":336584138},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":94,"seq":403726925935,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2513804,"kb_used_data":2257156,"kb_used_omap":7,"kb_used_meta":256568,"kb_avail":91231348,"statfs":{"total":95995035648,"available":93420900352,"internally_reserved":0,"allocated":2311327744,"data_stored":2310276558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":262726345},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,2,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122175,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1787556,"kb_used_data":1509020,"kb_used_omap":7,"kb_used_meta":278456,"kb_avail":91957596,"statfs":{"total":95995035648,"available":94164578304,"internally_reserved":0,"allocated":1545236480,"data_stored":1544166144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":285139535},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1538265088,"data_stored":1538261016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2296188928,"data_stored":2296184856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3686596608,"data_stored":3686596608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3761111040,"data_stored":3761111040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2200698880,"data_stored":2200698880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1952452608,"data_stored":1952448536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3199926272,"data_stored":3199926272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:46:18.928 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:46:18.929 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_1 pg_num 26 2024-09-13T14:46:18.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:18 smithi204 ceph-mon[87071]: pgmap v502: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 747 op/s; 36742/266196 objects misplaced (13.803%); 58 MiB/s, 929 objects/s recovering 2024-09-13T14:46:18.993 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:19.022 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:19.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:19.124 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:19.166 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:19.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:19.214 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:18 smithi096 ceph-mon[98901]: pgmap v502: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 747 op/s; 36742/266196 objects misplaced (13.803%); 58 MiB/s, 929 objects/s recovering 2024-09-13T14:46:19.215 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:18 smithi096 ceph-mon[102928]: pgmap v502: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 747 op/s; 36742/266196 objects misplaced (13.803%); 58 MiB/s, 929 objects/s recovering 2024-09-13T14:46:19.269 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:19.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.325 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:19.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:19.426 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:19.449 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:19.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:19.551 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:19.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.582 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:19.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:19.683 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:19.721 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:19.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:19.823 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:19.861 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:19.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:19.962 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:19.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.994 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:19.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:20.095 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:20.139 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:20.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:20.241 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:20.280 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:20.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:20.382 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:20.412 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:20.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:20.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:46:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:46:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:20.514 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:20.560 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:20.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:20.650 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:20.662 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:20.692 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:20.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:20.713 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:20.794 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:20.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.852 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:20.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:20.921 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:46:20.953 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:20.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.994 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:20.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:21.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:20 smithi096 ceph-mon[98901]: pgmap v503: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 35679/264861 objects misplaced (13.471%); 46 MiB/s, 728 objects/s recovering 2024-09-13T14:46:21.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:20 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:46:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:20 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:46:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:20 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:46:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:20 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:20 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3833278570' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:21.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:20 smithi096 ceph-mon[102928]: pgmap v503: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 35679/264861 objects misplaced (13.471%); 46 MiB/s, 728 objects/s recovering 2024-09-13T14:46:21.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:20 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:46:21.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:20 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:46:21.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:20 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:46:21.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:20 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:21.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:20 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3833278570' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:21.091 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:20 smithi204 ceph-mon[87071]: pgmap v503: 99 pgs: 4 active+remapped+backfill_wait, 5 active+remapped+backfilling, 90 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 35679/264861 objects misplaced (13.471%); 46 MiB/s, 728 objects/s recovering 2024-09-13T14:46:21.091 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:20 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:46:21.091 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:20 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:46:21.091 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:20 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:46:21.091 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:20 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:21.091 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:20 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3833278570' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:21.095 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:21.119 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:21.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:21.221 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:21.245 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:21.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:21.346 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:21.371 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:21.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:21.472 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:21.499 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:21.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:21.601 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:21.625 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:21.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:21.727 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:21.749 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:21.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:21.775 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:21.851 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:21.876 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:21.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:21.978 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:22.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.013 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:22.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:22.114 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:22.140 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:22.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:22.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:21 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:22.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:21 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3833278570' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:46:22.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:21 smithi204 ceph-mon[87071]: osdmap e171: 8 total, 8 up, 7 in 2024-09-13T14:46:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:21 smithi204 ceph-mon[87071]: osdmap e172: 8 total, 8 up, 7 in 2024-09-13T14:46:22.242 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:22.271 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:22.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:22.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:21 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:22.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:21 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3833278570' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:46:22.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:21 smithi096 ceph-mon[98901]: osdmap e171: 8 total, 8 up, 7 in 2024-09-13T14:46:22.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:21 smithi096 ceph-mon[98901]: osdmap e172: 8 total, 8 up, 7 in 2024-09-13T14:46:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:21 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:21 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3833278570' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:46:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:21 smithi096 ceph-mon[102928]: osdmap e171: 8 total, 8 up, 7 in 2024-09-13T14:46:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:21 smithi096 ceph-mon[102928]: osdmap e172: 8 total, 8 up, 7 in 2024-09-13T14:46:22.374 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:22.416 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:22.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:22.518 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:22.545 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:22.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:22.647 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:22.671 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:22.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:22.772 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:22.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.818 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:22.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:22.919 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:22.957 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:22.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:23.059 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:23.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:22 smithi096 ceph-mon[98901]: pgmap v505: 99 pgs: 1 peering, 4 active+remapped+backfill_wait, 4 active+remapped+backfilling, 90 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 34269/262143 objects misplaced (13.073%); 47 MiB/s, 749 objects/s recovering 2024-09-13T14:46:23.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:22 smithi096 ceph-mon[102928]: pgmap v505: 99 pgs: 1 peering, 4 active+remapped+backfill_wait, 4 active+remapped+backfilling, 90 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 34269/262143 objects misplaced (13.073%); 47 MiB/s, 749 objects/s recovering 2024-09-13T14:46:23.086 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:23.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:23.188 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:23.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:22 smithi204 ceph-mon[87071]: pgmap v505: 99 pgs: 1 peering, 4 active+remapped+backfill_wait, 4 active+remapped+backfilling, 90 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 34269/262143 objects misplaced (13.073%); 47 MiB/s, 749 objects/s recovering 2024-09-13T14:46:23.229 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:23.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:23.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:23.374 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:23.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:23.476 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:23.529 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:23.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:23.631 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:23.678 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:23.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:23.780 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:23.808 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:23.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:23.910 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:23.939 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:23.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:24.040 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:24.069 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:24.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:24.171 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:24.221 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:24.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:24.322 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:24.359 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:24.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:24.461 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:24.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.505 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:24.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:24.606 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:24.617 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:46:24.649 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:24.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:24.751 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:24.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.801 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:24.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:24.902 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:24.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.970 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:24.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:25.071 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:25.098 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:25.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:25.199 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:24 smithi204 ceph-mon[87071]: pgmap v507: 99 pgs: 1 peering, 3 active+remapped+backfill_wait, 4 active+remapped+backfilling, 91 active+clean; 5.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 791 op/s; 30217/256056 objects misplaced (11.801%); 65 MiB/s, 1.04k objects/s recovering 2024-09-13T14:46:25.223 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:25.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:25.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:24 smithi096 ceph-mon[98901]: pgmap v507: 99 pgs: 1 peering, 3 active+remapped+backfill_wait, 4 active+remapped+backfilling, 91 active+clean; 5.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 791 op/s; 30217/256056 objects misplaced (11.801%); 65 MiB/s, 1.04k objects/s recovering 2024-09-13T14:46:25.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:24 smithi096 ceph-mon[102928]: pgmap v507: 99 pgs: 1 peering, 3 active+remapped+backfill_wait, 4 active+remapped+backfilling, 91 active+clean; 5.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 791 op/s; 30217/256056 objects misplaced (11.801%); 65 MiB/s, 1.04k objects/s recovering 2024-09-13T14:46:25.325 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:25.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.386 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:25.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:25.487 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:25.527 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:25.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:25.629 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:25.668 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:25.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:25.689 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:46:25.689 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:46:25.689 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:46:25.689 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:46:25.689 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:46:25.690 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:46:25.690 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:46:25.690 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:46:25.690 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:46:25.690 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:46:25.690 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:46:25.690 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:46:25.690 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:46:25.690 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:46:25.690 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:46:25.690 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:46:25.770 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:25.812 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:25.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:25.917 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:25.970 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:25.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:26.071 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:26.098 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:26.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:26.200 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:26.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:25 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3310751128' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:26.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.237 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:26.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:26.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:25 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3310751128' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:26.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:25 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3310751128' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:26.338 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:26.361 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:26.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:26.463 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:26.510 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:26.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:26.612 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:26.654 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:26.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:26.756 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:26.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.797 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:26.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:26.898 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:26.924 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:26.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:27.026 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:27.050 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:27.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:27.152 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:27.178 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:27.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:27.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:26 smithi204 ceph-mon[87071]: pgmap v508: 99 pgs: 1 peering, 3 active+remapped+backfill_wait, 4 active+remapped+backfilling, 91 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 30217/256056 objects misplaced (11.801%); 39 MiB/s, 626 objects/s recovering 2024-09-13T14:46:27.281 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:27.292 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:26 smithi096 ceph-mon[98901]: pgmap v508: 99 pgs: 1 peering, 3 active+remapped+backfill_wait, 4 active+remapped+backfilling, 91 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 30217/256056 objects misplaced (11.801%); 39 MiB/s, 626 objects/s recovering 2024-09-13T14:46:27.294 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:26 smithi096 ceph-mon[102928]: pgmap v508: 99 pgs: 1 peering, 3 active+remapped+backfill_wait, 4 active+remapped+backfilling, 91 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 30217/256056 objects misplaced (11.801%); 39 MiB/s, 626 objects/s recovering 2024-09-13T14:46:27.305 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:27.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:27.407 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:27.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.452 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:27.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:27.553 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:27.580 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:27.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:27.682 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:27.706 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:27.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:27.808 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:27.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.854 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:27.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:27.955 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:28.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.007 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:28.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:28.108 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:28.133 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:28.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:28.234 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:28.261 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:46:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:46:27] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.51.0" 2024-09-13T14:46:28.261 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:28.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:28.363 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:28.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.404 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:28.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:28.505 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:28.532 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:28.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:28.634 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:28.674 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:28.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:28.709 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:46:28.709 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:46:28.709 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.709 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.45167.0:154087 6.a e172/169 6:5d87afc9:::benchmark_data_smithi096_167087_object36859:head v 172'9847, mlcod=172'9845)", 2024-09-13T14:46:28.709 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:28.695763+0000", 2024-09-13T14:46:28.709 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.011466290000000001, 2024-09-13T14:46:28.709 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.01149321, 2024-09-13T14:46:28.709 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:46:28.710 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:46:28.710 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:28.710 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:46:28.710 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.710 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:46:28.710 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.695763+0000", 2024-09-13T14:46:28.710 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:46:28.710 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.710 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.710 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:46:28.710 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.695763+0000", 2024-09-13T14:46:28.710 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:46:28.710 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.711 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.711 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:46:28.711 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.695764+0000", 2024-09-13T14:46:28.711 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.3099999999999996e-07 2024-09-13T14:46:28.711 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.711 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.711 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:46:28.711 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.695775+0000", 2024-09-13T14:46:28.711 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0535e-05 2024-09-13T14:46:28.711 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.711 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.711 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:46:28.711 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.695786+0000", 2024-09-13T14:46:28.711 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0706999999999999e-05 2024-09-13T14:46:28.712 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.712 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.712 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:46:28.712 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.695793+0000", 2024-09-13T14:46:28.712 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1749999999999999e-06 2024-09-13T14:46:28.712 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.712 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.712 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:46:28.712 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.695912+0000", 2024-09-13T14:46:28.712 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000118799 2024-09-13T14:46:28.712 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.712 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.712 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:46:28.712 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.695939+0000", 2024-09-13T14:46:28.712 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7606e-05 2024-09-13T14:46:28.713 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.713 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.713 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:46:28.713 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.707149+0000", 2024-09-13T14:46:28.713 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011209643 2024-09-13T14:46:28.713 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:46:28.713 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:46:28.713 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:46:28.713 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.713 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.713 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.45167.0:154089 6.8 e172/145 6:1382d940:::benchmark_data_smithi096_167087_object36861:head v 172'9722, mlcod=172'9720)", 2024-09-13T14:46:28.713 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:28.695821+0000", 2024-09-13T14:46:28.713 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.011409183, 2024-09-13T14:46:28.713 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011497082, 2024-09-13T14:46:28.714 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:46:28.714 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:46:28.714 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:46:28.714 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:46:28.714 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.714 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:46:28.714 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.695821+0000", 2024-09-13T14:46:28.714 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:46:28.714 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.714 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.714 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:46:28.714 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.695821+0000", 2024-09-13T14:46:28.714 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:46:28.715 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.715 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.715 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:46:28.715 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.695821+0000", 2024-09-13T14:46:28.715 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7399999999999998e-07 2024-09-13T14:46:28.715 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.715 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.715 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:46:28.715 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.695825+0000", 2024-09-13T14:46:28.715 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0500000000000002e-06 2024-09-13T14:46:28.715 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.715 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.715 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:46:28.716 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.695838+0000", 2024-09-13T14:46:28.716 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2955999999999999e-05 2024-09-13T14:46:28.716 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.716 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.716 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:46:28.716 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.695843+0000", 2024-09-13T14:46:28.716 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0200000000000002e-06 2024-09-13T14:46:28.716 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.716 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.716 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:46:28.716 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.697198+0000", 2024-09-13T14:46:28.716 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0013549390000000001 2024-09-13T14:46:28.716 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.716 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.716 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:46:28.717 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.697213+0000", 2024-09-13T14:46:28.717 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5294999999999998e-05 2024-09-13T14:46:28.717 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:46:28.717 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:46:28.717 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:46:28.717 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.717 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.717 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.45167.0:154091 6.a e172/169 6:50fff38f:::benchmark_data_smithi096_167087_object36863:head v 172'9848, mlcod=172'9845)", 2024-09-13T14:46:28.717 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:28.698685+0000", 2024-09-13T14:46:28.717 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0085448529999999998, 2024-09-13T14:46:28.717 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0086799719999999993, 2024-09-13T14:46:28.717 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:46:28.717 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:46:28.717 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:46:28.717 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:46:28.718 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.718 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:46:28.718 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.698685+0000", 2024-09-13T14:46:28.718 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:46:28.718 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.718 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.718 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:46:28.718 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.698685+0000", 2024-09-13T14:46:28.718 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:46:28.718 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.718 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.718 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:46:28.718 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.698686+0000", 2024-09-13T14:46:28.719 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.421e-06 2024-09-13T14:46:28.719 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.719 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.719 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:46:28.719 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.698696+0000", 2024-09-13T14:46:28.719 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.5009999999999993e-06 2024-09-13T14:46:28.719 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.719 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.719 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:46:28.719 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.698707+0000", 2024-09-13T14:46:28.719 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1147e-05 2024-09-13T14:46:28.719 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.719 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.720 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:46:28.720 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.698715+0000", 2024-09-13T14:46:28.720 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9990000000000008e-06 2024-09-13T14:46:28.720 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.720 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.720 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:46:28.720 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.699468+0000", 2024-09-13T14:46:28.720 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00075285200000000004 2024-09-13T14:46:28.720 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.720 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.720 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:46:28.720 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.699499+0000", 2024-09-13T14:46:28.720 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.15e-05 2024-09-13T14:46:28.720 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:46:28.721 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:46:28.721 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:46:28.721 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.721 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.721 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.45167.0:154093 6.a e172/169 6:53aed52c:::benchmark_data_smithi096_167087_object36865:head v 172'9849, mlcod=172'9845)", 2024-09-13T14:46:28.721 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:28.701266+0000", 2024-09-13T14:46:28.721 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.00596365, 2024-09-13T14:46:28.721 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0061450350000000001, 2024-09-13T14:46:28.721 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:46:28.721 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:46:28.721 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:46:28.721 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:46:28.721 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.701266+0000", 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.701266+0000", 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.701267+0000", 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9800000000000003e-07 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.722 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:46:28.723 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.701276+0000", 2024-09-13T14:46:28.723 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9749999999999996e-06 2024-09-13T14:46:28.723 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.723 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.723 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:46:28.723 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.701291+0000", 2024-09-13T14:46:28.723 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4620000000000001e-05 2024-09-13T14:46:28.723 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.723 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.723 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:46:28.723 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.701298+0000", 2024-09-13T14:46:28.723 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2049999999999996e-06 2024-09-13T14:46:28.723 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.723 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.723 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:46:28.724 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.703429+0000", 2024-09-13T14:46:28.724 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0021315539999999999 2024-09-13T14:46:28.724 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:46:28.724 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:46:28.724 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:46:28.724 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:28.703476+0000", 2024-09-13T14:46:28.724 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6289999999999999e-05 2024-09-13T14:46:28.724 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:46:28.724 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:46:28.724 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:46:28.724 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:46:28.724 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2024-09-13T14:46:28.724 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 4 2024-09-13T14:46:28.724 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:46:28.776 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:28.799 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:28.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:28.901 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:28.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.945 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:28.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:29.046 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:29.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:28 smithi096 ceph-mon[98901]: pgmap v509: 99 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 92 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 28443/245958 objects misplaced (11.564%); 40 MiB/s, 634 objects/s recovering 2024-09-13T14:46:29.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:28 smithi096 ceph-mon[102928]: pgmap v509: 99 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 92 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 28443/245958 objects misplaced (11.564%); 40 MiB/s, 634 objects/s recovering 2024-09-13T14:46:29.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.084 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:29.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:29.185 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:29.209 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:29.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:29.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:28 smithi204 ceph-mon[87071]: pgmap v509: 99 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 92 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 28443/245958 objects misplaced (11.564%); 40 MiB/s, 634 objects/s recovering 2024-09-13T14:46:29.312 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:29.312 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:29.333 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:46:29.337 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:29.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:29.438 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:29.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.480 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:29.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:29.581 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:29.604 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:29.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:29.706 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:29.735 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:29.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:29.836 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:29.861 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:29.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:29.903 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:46:29.962 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:29.987 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:29.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:29.991 INFO:teuthology.orchestra.run.smithi096.stderr:set pool 6 pg_num to 26 2024-09-13T14:46:30.090 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:30.121 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:29 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2380204817' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:46:30.122 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:29 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2380204817' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:46:30.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.146 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:30.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:30.246 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:30.290 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:30.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:30.392 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:30.419 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:30.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:30.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:29 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2380204817' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:46:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:46:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:46:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:30.521 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:30.548 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:30.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:30.650 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:30.687 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:30.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:30.789 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:30.830 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:30.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:30.932 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:30.957 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:30.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:31.059 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:31.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.111 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:31.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:31.211 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:31.236 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:31.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:31.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:30 smithi096 ceph-mon[98901]: pgmap v510: 99 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 92 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 765 op/s; 27350/244524 objects misplaced (11.185%); 41 MiB/s, 659 objects/s recovering 2024-09-13T14:46:31.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:30 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2380204817' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:46:31.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:30 smithi096 ceph-mon[98901]: osdmap e173: 8 total, 8 up, 7 in 2024-09-13T14:46:31.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:30 smithi096 ceph-mon[102928]: pgmap v510: 99 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 92 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 765 op/s; 27350/244524 objects misplaced (11.185%); 41 MiB/s, 659 objects/s recovering 2024-09-13T14:46:31.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:30 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2380204817' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:46:31.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:30 smithi096 ceph-mon[102928]: osdmap e173: 8 total, 8 up, 7 in 2024-09-13T14:46:31.338 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:31.371 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:31.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:31.408 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:30 smithi204 ceph-mon[87071]: pgmap v510: 99 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 92 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 765 op/s; 27350/244524 objects misplaced (11.185%); 41 MiB/s, 659 objects/s recovering 2024-09-13T14:46:31.408 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:30 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2380204817' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:46:31.408 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:30 smithi204 ceph-mon[87071]: osdmap e173: 8 total, 8 up, 7 in 2024-09-13T14:46:31.473 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:31.513 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:31.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:31.615 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:31.652 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:31.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:31.753 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:31.779 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:31.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:31.880 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:31.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.909 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:31.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:32.010 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:32.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.062 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:32.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:32.162 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:32.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.200 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:32.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:32.301 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:32.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:31 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:46:32.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:31 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:46:32.345 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:32.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:32.447 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:32.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:31 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:46:32.470 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:32.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:32.571 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:32.593 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:32.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:32.695 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:32.717 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:32.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:32.819 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:32.841 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:32.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:32.943 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:32.965 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:32.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:33.067 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:33.095 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:33.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:33.197 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:33.223 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:33.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:33.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:33 smithi096 ceph-mon[98901]: pgmap v512: 99 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 92 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 27129/241047 objects misplaced (11.255%); 23 MiB/s, 370 objects/s recovering 2024-09-13T14:46:33.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:46:33.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:33 smithi096 ceph-mon[98901]: osdmap e174: 8 total, 8 up, 7 in 2024-09-13T14:46:33.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:33 smithi096 ceph-mon[102928]: pgmap v512: 99 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 92 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 27129/241047 objects misplaced (11.255%); 23 MiB/s, 370 objects/s recovering 2024-09-13T14:46:33.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:46:33.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:33 smithi096 ceph-mon[102928]: osdmap e174: 8 total, 8 up, 7 in 2024-09-13T14:46:33.325 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:33.348 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:33.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:33.450 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:33 smithi204 ceph-mon[87071]: pgmap v512: 99 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 92 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 27129/241047 objects misplaced (11.255%); 23 MiB/s, 370 objects/s recovering 2024-09-13T14:46:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:46:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:33 smithi204 ceph-mon[87071]: osdmap e174: 8 total, 8 up, 7 in 2024-09-13T14:46:33.474 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:33.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:33.576 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:33.602 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:33.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:33.705 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:33.728 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:33.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:33.830 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:33.853 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:33.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:33.955 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:33.978 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:33.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:34.079 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:34.103 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:34.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:34.115 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:34.205 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:34.229 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:34.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:34.270 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:34 smithi096 ceph-mon[98901]: osdmap e175: 8 total, 8 up, 7 in 2024-09-13T14:46:34.270 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:34 smithi096 ceph-mon[98901]: 6.1 continuing backfill to osd.0 from (153'4140,173'9750] MIN to 173'9750 2024-09-13T14:46:34.270 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:34 smithi096 ceph-mon[98901]: 6.4 continuing backfill to osd.4 from (153'4312,173'9908] MIN to 173'9908 2024-09-13T14:46:34.270 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:34 smithi096 ceph-mon[98901]: 6.9 starting backfill to osd.3 from (153'4024,173'9962] MIN to 173'9964 2024-09-13T14:46:34.270 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:34 smithi096 ceph-mon[98901]: 6.0 continuing backfill to osd.5 from (153'3782,173'9754] MIN to 173'9754 2024-09-13T14:46:34.270 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:34 smithi096 ceph-mon[102928]: osdmap e175: 8 total, 8 up, 7 in 2024-09-13T14:46:34.270 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:34 smithi096 ceph-mon[102928]: 6.1 continuing backfill to osd.0 from (153'4140,173'9750] MIN to 173'9750 2024-09-13T14:46:34.271 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:34 smithi096 ceph-mon[102928]: 6.4 continuing backfill to osd.4 from (153'4312,173'9908] MIN to 173'9908 2024-09-13T14:46:34.271 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:34 smithi096 ceph-mon[102928]: 6.9 starting backfill to osd.3 from (153'4024,173'9962] MIN to 173'9964 2024-09-13T14:46:34.271 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:34 smithi096 ceph-mon[102928]: 6.0 continuing backfill to osd.5 from (153'3782,173'9754] MIN to 173'9754 2024-09-13T14:46:34.330 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:34.354 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:34.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:34.456 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:34.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:34 smithi204 ceph-mon[87071]: osdmap e175: 8 total, 8 up, 7 in 2024-09-13T14:46:34.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:34 smithi204 ceph-mon[87071]: 6.1 continuing backfill to osd.0 from (153'4140,173'9750] MIN to 173'9750 2024-09-13T14:46:34.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:34 smithi204 ceph-mon[87071]: 6.4 continuing backfill to osd.4 from (153'4312,173'9908] MIN to 173'9908 2024-09-13T14:46:34.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:34 smithi204 ceph-mon[87071]: 6.9 starting backfill to osd.3 from (153'4024,173'9962] MIN to 173'9964 2024-09-13T14:46:34.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:34 smithi204 ceph-mon[87071]: 6.0 continuing backfill to osd.5 from (153'3782,173'9754] MIN to 173'9754 2024-09-13T14:46:34.478 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:34.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:34.580 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:34.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.605 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:34.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:34.706 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:34.734 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:34.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:34.836 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:34.859 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:34.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:34.960 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:34.984 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:34.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:35.086 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:35.115 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:35.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:35.199 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:35 smithi096 ceph-mon[98901]: 6.6 scrub starts 2024-09-13T14:46:35.199 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:35 smithi096 ceph-mon[98901]: pgmap v515: 109 pgs: 1 peering, 2 remapped+peering, 10 unknown, 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 91 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 14772/233862 objects misplaced (6.317%); 20 MiB/s, 326 objects/s recovering 2024-09-13T14:46:35.199 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:35 smithi096 ceph-mon[98901]: osdmap e176: 8 total, 8 up, 7 in 2024-09-13T14:46:35.200 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:35 smithi096 ceph-mon[102928]: 6.6 scrub starts 2024-09-13T14:46:35.200 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:35 smithi096 ceph-mon[102928]: pgmap v515: 109 pgs: 1 peering, 2 remapped+peering, 10 unknown, 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 91 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 14772/233862 objects misplaced (6.317%); 20 MiB/s, 326 objects/s recovering 2024-09-13T14:46:35.200 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:35 smithi096 ceph-mon[102928]: osdmap e176: 8 total, 8 up, 7 in 2024-09-13T14:46:35.216 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:35.243 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:35.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:35.345 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:35.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.385 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:35.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:35.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:35 smithi204 ceph-mon[87071]: 6.6 scrub starts 2024-09-13T14:46:35.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:35 smithi204 ceph-mon[87071]: pgmap v515: 109 pgs: 1 peering, 2 remapped+peering, 10 unknown, 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 91 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 14772/233862 objects misplaced (6.317%); 20 MiB/s, 326 objects/s recovering 2024-09-13T14:46:35.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:35 smithi204 ceph-mon[87071]: osdmap e176: 8 total, 8 up, 7 in 2024-09-13T14:46:35.486 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:35.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.525 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:35.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:35.626 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:35.650 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:35.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:35.751 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:35.775 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:35.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:35.877 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:35.900 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:35.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:35.910 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:46:35.910 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:46:35.910 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-09-13T14:46:35.911 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.0 'trim stale osdmaps' 2024-09-13T14:46:36.002 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:36.031 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:36.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:36.133 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:36.157 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:36.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:36.260 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:36.283 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:36.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:36.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[98901]: 6.16 scrub starts 2024-09-13T14:46:36.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[98901]: 6.14 continuing backfill to osd.4 from (153'4312,173'9908] MIN to 173'9908 2024-09-13T14:46:36.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[98901]: 6.11 continuing backfill to osd.0 from (153'4140,173'9750] MIN to 173'9750 2024-09-13T14:46:36.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[98901]: osdmap e177: 8 total, 8 up, 7 in 2024-09-13T14:46:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[98901]: 6.10 continuing backfill to osd.5 from (153'3782,173'9754] MIN to 173'9754 2024-09-13T14:46:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[98901]: 6.19 starting backfill to osd.3 from (153'4024,173'9962] MIN to 173'9964 2024-09-13T14:46:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:46:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/650377509' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[102928]: 6.16 scrub starts 2024-09-13T14:46:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[102928]: 6.14 continuing backfill to osd.4 from (153'4312,173'9908] MIN to 173'9908 2024-09-13T14:46:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[102928]: 6.11 continuing backfill to osd.0 from (153'4140,173'9750] MIN to 173'9750 2024-09-13T14:46:36.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[102928]: osdmap e177: 8 total, 8 up, 7 in 2024-09-13T14:46:36.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[102928]: 6.10 continuing backfill to osd.5 from (153'3782,173'9754] MIN to 173'9754 2024-09-13T14:46:36.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[102928]: 6.19 starting backfill to osd.3 from (153'4024,173'9962] MIN to 173'9964 2024-09-13T14:46:36.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:46:36.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:36.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/650377509' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:36.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:36 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:36.385 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:36.409 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:36.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:36.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:36 smithi204 ceph-mon[87071]: 6.16 scrub starts 2024-09-13T14:46:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:36 smithi204 ceph-mon[87071]: 6.14 continuing backfill to osd.4 from (153'4312,173'9908] MIN to 173'9908 2024-09-13T14:46:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:36 smithi204 ceph-mon[87071]: 6.11 continuing backfill to osd.0 from (153'4140,173'9750] MIN to 173'9750 2024-09-13T14:46:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:36 smithi204 ceph-mon[87071]: osdmap e177: 8 total, 8 up, 7 in 2024-09-13T14:46:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:36 smithi204 ceph-mon[87071]: 6.10 continuing backfill to osd.5 from (153'3782,173'9754] MIN to 173'9754 2024-09-13T14:46:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:36 smithi204 ceph-mon[87071]: 6.19 starting backfill to osd.3 from (153'4024,173'9962] MIN to 173'9964 2024-09-13T14:46:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:46:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:36 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/650377509' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:36 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:36.511 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:36.538 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:36.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:36.640 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:36.681 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:36.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:36.783 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:36.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.819 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:36.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:36.844 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:46:36.921 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:36.954 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:36.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:37.056 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:37.094 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:37.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:37.118 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:37 smithi096 ceph-mon[98901]: pgmap v518: 109 pgs: 1 peering, 2 remapped+peering, 10 unknown, 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 91 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 14772/233862 objects misplaced (6.317%); 19 MiB/s, 298 objects/s recovering 2024-09-13T14:46:37.118 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:37 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:37.118 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:37 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:37.118 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:37 smithi096 ceph-mon[98901]: osdmap e178: 8 total, 8 up, 7 in 2024-09-13T14:46:37.118 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:37 smithi096 ceph-mon[102928]: pgmap v518: 109 pgs: 1 peering, 2 remapped+peering, 10 unknown, 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 91 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 14772/233862 objects misplaced (6.317%); 19 MiB/s, 298 objects/s recovering 2024-09-13T14:46:37.118 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:37 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:37.118 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:37 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:37.118 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:37 smithi096 ceph-mon[102928]: osdmap e178: 8 total, 8 up, 7 in 2024-09-13T14:46:37.196 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:37.232 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:37.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:37.334 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:37.361 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:37.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:37.464 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:37.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:37 smithi204 ceph-mon[87071]: pgmap v518: 109 pgs: 1 peering, 2 remapped+peering, 10 unknown, 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 91 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 14772/233862 objects misplaced (6.317%); 19 MiB/s, 298 objects/s recovering 2024-09-13T14:46:37.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:37 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:37.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:37 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:37.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:37 smithi204 ceph-mon[87071]: osdmap e178: 8 total, 8 up, 7 in 2024-09-13T14:46:37.489 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:37.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:37.591 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:37.619 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:37.620 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:37.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:37.722 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:37.748 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:37.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:37.850 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:37.874 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:37.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:37.976 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:38.000 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:38.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:38.102 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:38.124 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:38.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:38.202 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:46:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:46:37] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.51.0" 2024-09-13T14:46:38.225 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:38.248 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:38.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:38.335 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:46:38.336 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:46:38.350 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:38.366 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:46:38.366 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:46:38.373 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:38.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:38.391 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:46:38.392 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:46:38.412 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:46:38.412 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:46:38.433 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:46:38.434 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:46:38.455 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:46:38.455 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:46:38.475 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:38.478 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:46:38.478 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:46:38.498 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:38.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:38.501 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:46:38.501 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:46:38.600 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:38.629 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:38.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:38.730 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:38.760 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:38.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:38.862 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:38.891 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:38.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:38.993 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:39.032 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:39.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:39.134 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:39.163 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:39.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:38 smithi204 ceph-mon[87071]: pgmap v520: 109 pgs: 1 peering, 2 unknown, 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 15045/230337 objects misplaced (6.532%); 79 MiB/s, 1.27k objects/s recovering 2024-09-13T14:46:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:38 smithi204 ceph-mon[87071]: osdmap e179: 8 total, 8 up, 7 in 2024-09-13T14:46:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:38 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2374949209' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:39.219 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:46:38 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=sqlstore.transactions t=2024-09-13T14:46:38.909185892Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:46:39.265 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:39.304 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:39.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:39.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:38 smithi096 ceph-mon[98901]: pgmap v520: 109 pgs: 1 peering, 2 unknown, 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 15045/230337 objects misplaced (6.532%); 79 MiB/s, 1.27k objects/s recovering 2024-09-13T14:46:39.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:38 smithi096 ceph-mon[98901]: osdmap e179: 8 total, 8 up, 7 in 2024-09-13T14:46:39.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:38 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2374949209' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:39.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:38 smithi096 ceph-mon[102928]: pgmap v520: 109 pgs: 1 peering, 2 unknown, 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 15045/230337 objects misplaced (6.532%); 79 MiB/s, 1.27k objects/s recovering 2024-09-13T14:46:39.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:38 smithi096 ceph-mon[102928]: osdmap e179: 8 total, 8 up, 7 in 2024-09-13T14:46:39.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:38 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2374949209' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:39.406 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:39.437 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:39.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:39.538 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:39.562 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:39.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:39.615 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:46:39.664 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:39.693 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:39.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:39.795 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:39.857 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:39.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:39.959 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:39.986 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:39.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:40.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:39 smithi096 ceph-mon[98901]: 6.6 scrub starts 2024-09-13T14:46:40.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:39 smithi096 ceph-mon[102928]: 6.6 scrub starts 2024-09-13T14:46:40.088 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:40.112 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:40.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:40.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:39 smithi204 ceph-mon[87071]: 6.6 scrub starts 2024-09-13T14:46:40.214 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:40.243 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:40.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:40.345 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:40.376 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:40.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:40.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:46:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:46:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:40.477 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:40.498 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:40.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:40.600 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:40.629 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:40.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:40.731 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:40.765 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:40.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:40.866 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:40.903 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:40.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:41.005 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:41.037 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:41.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:41.138 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:41.167 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:41.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:41.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:40 smithi204 ceph-mon[87071]: 6.16 scrub starts 2024-09-13T14:46:41.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:40 smithi204 ceph-mon[87071]: 6.1 scrub starts 2024-09-13T14:46:41.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:40 smithi204 ceph-mon[87071]: pgmap v522: 109 pgs: 1 peering, 6 active+remapped+backfill_wait, 4 active+remapped+backfilling, 98 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 238 op/s; 16718/229743 objects misplaced (7.277%); 72 MiB/s, 1.16k objects/s recovering 2024-09-13T14:46:41.269 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:41.299 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:41.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:41.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:40 smithi096 ceph-mon[98901]: 6.16 scrub starts 2024-09-13T14:46:41.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:40 smithi096 ceph-mon[98901]: 6.1 scrub starts 2024-09-13T14:46:41.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:40 smithi096 ceph-mon[98901]: pgmap v522: 109 pgs: 1 peering, 6 active+remapped+backfill_wait, 4 active+remapped+backfilling, 98 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 238 op/s; 16718/229743 objects misplaced (7.277%); 72 MiB/s, 1.16k objects/s recovering 2024-09-13T14:46:41.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:40 smithi096 ceph-mon[102928]: 6.16 scrub starts 2024-09-13T14:46:41.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:40 smithi096 ceph-mon[102928]: 6.1 scrub starts 2024-09-13T14:46:41.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:40 smithi096 ceph-mon[102928]: pgmap v522: 109 pgs: 1 peering, 6 active+remapped+backfill_wait, 4 active+remapped+backfilling, 98 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 238 op/s; 16718/229743 objects misplaced (7.277%); 72 MiB/s, 1.16k objects/s recovering 2024-09-13T14:46:41.400 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:41.429 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:41.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:41.531 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:41.552 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:41.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:41.653 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:41.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.692 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:41.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:41.793 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:41.814 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:41.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:41.916 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:41.938 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:41.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:42.040 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:42.061 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:42.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:42.162 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:42.191 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:42.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:42.293 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:42.315 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:42.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:42.417 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:42.438 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:42.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:42.540 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:42.561 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:42.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:42.662 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:42.683 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:42.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:42.785 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:42.806 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:42.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:42.908 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:42.941 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:42.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:43.043 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:43.065 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:43.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:43.167 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:43.192 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:43.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:43.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:42 smithi204 ceph-mon[87071]: pgmap v523: 109 pgs: 1 active+clean+remapped, 1 peering, 6 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 378 op/s; 14529/226182 objects misplaced (6.424%); 82 MiB/s, 1.31k objects/s recovering 2024-09-13T14:46:43.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:42 smithi204 ceph-mon[87071]: osdmap e180: 8 total, 8 up, 7 in 2024-09-13T14:46:43.294 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:43.320 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:43.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:43.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:42 smithi096 ceph-mon[98901]: pgmap v523: 109 pgs: 1 active+clean+remapped, 1 peering, 6 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 378 op/s; 14529/226182 objects misplaced (6.424%); 82 MiB/s, 1.31k objects/s recovering 2024-09-13T14:46:43.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:42 smithi096 ceph-mon[98901]: osdmap e180: 8 total, 8 up, 7 in 2024-09-13T14:46:43.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:42 smithi096 ceph-mon[102928]: pgmap v523: 109 pgs: 1 active+clean+remapped, 1 peering, 6 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 378 op/s; 14529/226182 objects misplaced (6.424%); 82 MiB/s, 1.31k objects/s recovering 2024-09-13T14:46:43.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:42 smithi096 ceph-mon[102928]: osdmap e180: 8 total, 8 up, 7 in 2024-09-13T14:46:43.422 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:43.447 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:43.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:43.549 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:43.571 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:43.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:43.672 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:43.700 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:43.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:43.802 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:43.842 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:43.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:43.944 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:43.969 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:43.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:44.071 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:44.092 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:44.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:44.194 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:44.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:43 smithi204 ceph-mon[87071]: osdmap e181: 8 total, 8 up, 7 in 2024-09-13T14:46:44.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:43 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:46:44.229 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:44.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:44.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:43 smithi096 ceph-mon[98901]: osdmap e181: 8 total, 8 up, 7 in 2024-09-13T14:46:44.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:43 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:46:44.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:43 smithi096 ceph-mon[102928]: osdmap e181: 8 total, 8 up, 7 in 2024-09-13T14:46:44.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:43 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:46:44.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:44.359 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:44.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:44.460 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:44.483 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:44.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:44.585 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:44.611 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:44.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:44.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:44.740 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:44.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:44.841 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:44.914 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:44.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:45.016 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:45.051 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:45.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:45.153 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:45.173 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:45.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:45.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:44 smithi204 ceph-mon[87071]: 6.6 scrub starts 2024-09-13T14:46:45.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:44 smithi204 ceph-mon[87071]: 6.0 scrub starts 2024-09-13T14:46:45.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:44 smithi204 ceph-mon[87071]: pgmap v526: 109 pgs: 2 active+clean+remapped, 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 99 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 10433/219060 objects misplaced (4.763%); 64 MiB/s, 1.03k objects/s recovering 2024-09-13T14:46:45.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:44 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:46:45.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:44 smithi204 ceph-mon[87071]: osdmap e182: 8 total, 8 up, 7 in 2024-09-13T14:46:45.275 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:45.297 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:45.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:45.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:44 smithi096 ceph-mon[102928]: 6.6 scrub starts 2024-09-13T14:46:45.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:44 smithi096 ceph-mon[102928]: 6.0 scrub starts 2024-09-13T14:46:45.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:44 smithi096 ceph-mon[102928]: pgmap v526: 109 pgs: 2 active+clean+remapped, 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 99 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 10433/219060 objects misplaced (4.763%); 64 MiB/s, 1.03k objects/s recovering 2024-09-13T14:46:45.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:44 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:46:45.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:44 smithi096 ceph-mon[102928]: osdmap e182: 8 total, 8 up, 7 in 2024-09-13T14:46:45.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:44 smithi096 ceph-mon[98901]: 6.6 scrub starts 2024-09-13T14:46:45.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:44 smithi096 ceph-mon[98901]: 6.0 scrub starts 2024-09-13T14:46:45.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:44 smithi096 ceph-mon[98901]: pgmap v526: 109 pgs: 2 active+clean+remapped, 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 99 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 10433/219060 objects misplaced (4.763%); 64 MiB/s, 1.03k objects/s recovering 2024-09-13T14:46:45.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:44 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:46:45.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:44 smithi096 ceph-mon[98901]: osdmap e182: 8 total, 8 up, 7 in 2024-09-13T14:46:45.398 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:45.445 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:45.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:45.549 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:45.570 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:45.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:45.672 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:45.693 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:45.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:45.795 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:45.838 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:45.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:45.940 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:45.961 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:45.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:46.062 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:46.088 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:46.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:46.190 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:46.218 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:46.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:45 smithi204 ceph-mon[87071]: 6.16 scrub starts 2024-09-13T14:46:46.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:45 smithi204 ceph-mon[87071]: 6.1 scrub starts 2024-09-13T14:46:46.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:45 smithi204 ceph-mon[87071]: 6.10 starting backfill to osd.1 from (0'0,0'0] MAX to 181'9930 2024-09-13T14:46:46.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:45 smithi204 ceph-mon[87071]: osdmap e183: 8 total, 8 up, 7 in 2024-09-13T14:46:46.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:45 smithi204 ceph-mon[87071]: 6.10 continuing backfill to osd.5 from (153'3782,181'9930] MIN to 181'9930 2024-09-13T14:46:46.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:45 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:46:46.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:46.320 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:45 smithi096 ceph-mon[98901]: 6.16 scrub starts 2024-09-13T14:46:46.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:45 smithi096 ceph-mon[98901]: 6.1 scrub starts 2024-09-13T14:46:46.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:45 smithi096 ceph-mon[98901]: 6.10 starting backfill to osd.1 from (0'0,0'0] MAX to 181'9930 2024-09-13T14:46:46.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:45 smithi096 ceph-mon[98901]: osdmap e183: 8 total, 8 up, 7 in 2024-09-13T14:46:46.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:45 smithi096 ceph-mon[98901]: 6.10 continuing backfill to osd.5 from (153'3782,181'9930] MIN to 181'9930 2024-09-13T14:46:46.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:45 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:46:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:45 smithi096 ceph-mon[102928]: 6.16 scrub starts 2024-09-13T14:46:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:45 smithi096 ceph-mon[102928]: 6.1 scrub starts 2024-09-13T14:46:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:45 smithi096 ceph-mon[102928]: 6.10 starting backfill to osd.1 from (0'0,0'0] MAX to 181'9930 2024-09-13T14:46:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:45 smithi096 ceph-mon[102928]: osdmap e183: 8 total, 8 up, 7 in 2024-09-13T14:46:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:45 smithi096 ceph-mon[102928]: 6.10 continuing backfill to osd.5 from (153'3782,181'9930] MIN to 181'9930 2024-09-13T14:46:46.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:45 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:46:46.342 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:46.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:46.443 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:46.474 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:46.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:46.576 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:46.604 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:46.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:46.705 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:46.733 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:46.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:46.835 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:46.856 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:46.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:46.958 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:46.979 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:46.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:47.081 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:47.106 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:47.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:47.193 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:46 smithi096 ceph-mon[98901]: 6.0 scrub starts 2024-09-13T14:46:47.193 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:47 smithi096 ceph-mon[98901]: 6.6 scrub starts 2024-09-13T14:46:47.194 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:47 smithi096 ceph-mon[98901]: 6.4 scrub starts 2024-09-13T14:46:47.194 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:47 smithi096 ceph-mon[98901]: pgmap v529: 109 pgs: 2 active+clean+remapped, 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 99 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 10433/219060 objects misplaced (4.763%); 46 MiB/s, 730 objects/s recovering 2024-09-13T14:46:47.194 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:47 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:46:47.194 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:47 smithi096 ceph-mon[98901]: osdmap e184: 8 total, 8 up, 7 in 2024-09-13T14:46:47.194 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:46 smithi096 ceph-mon[102928]: 6.0 scrub starts 2024-09-13T14:46:47.194 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:47 smithi096 ceph-mon[102928]: 6.6 scrub starts 2024-09-13T14:46:47.194 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:47 smithi096 ceph-mon[102928]: 6.4 scrub starts 2024-09-13T14:46:47.194 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:47 smithi096 ceph-mon[102928]: pgmap v529: 109 pgs: 2 active+clean+remapped, 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 99 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 10433/219060 objects misplaced (4.763%); 46 MiB/s, 730 objects/s recovering 2024-09-13T14:46:47.194 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:47 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:46:47.194 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:47 smithi096 ceph-mon[102928]: osdmap e184: 8 total, 8 up, 7 in 2024-09-13T14:46:47.208 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:47.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:46 smithi204 ceph-mon[87071]: 6.0 scrub starts 2024-09-13T14:46:47.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:46 smithi204 ceph-mon[87071]: 6.6 scrub starts 2024-09-13T14:46:47.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:46 smithi204 ceph-mon[87071]: 6.4 scrub starts 2024-09-13T14:46:47.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:46 smithi204 ceph-mon[87071]: pgmap v529: 109 pgs: 2 active+clean+remapped, 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 99 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 10433/219060 objects misplaced (4.763%); 46 MiB/s, 730 objects/s recovering 2024-09-13T14:46:47.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:46 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:46:47.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:46 smithi204 ceph-mon[87071]: osdmap e184: 8 total, 8 up, 7 in 2024-09-13T14:46:47.231 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:47.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:47.332 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:47.353 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:47.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:47.455 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:47.483 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:47.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:47.586 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:47.612 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:47.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:47.714 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:47.741 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:47.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:47.843 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:47.864 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:47.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:47.966 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:48.003 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:48.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:48.105 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:48.126 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:48.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:48.228 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:48.253 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:48.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:48.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:46:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:46:47] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.51.0" 2024-09-13T14:46:48.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:48 smithi096 ceph-mon[98901]: 6.0 scrub starts 2024-09-13T14:46:48.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:48 smithi096 ceph-mon[98901]: 6.16 scrub starts 2024-09-13T14:46:48.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:48 smithi096 ceph-mon[98901]: 6.1 scrub starts 2024-09-13T14:46:48.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:48 smithi096 ceph-mon[98901]: osdmap e185: 8 total, 8 up, 7 in 2024-09-13T14:46:48.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:48 smithi096 ceph-mon[102928]: 6.0 scrub starts 2024-09-13T14:46:48.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:48 smithi096 ceph-mon[102928]: 6.16 scrub starts 2024-09-13T14:46:48.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:48 smithi096 ceph-mon[102928]: 6.1 scrub starts 2024-09-13T14:46:48.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:48 smithi096 ceph-mon[102928]: osdmap e185: 8 total, 8 up, 7 in 2024-09-13T14:46:48.355 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:48.382 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:48.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:48.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:48 smithi204 ceph-mon[87071]: 6.0 scrub starts 2024-09-13T14:46:48.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:48 smithi204 ceph-mon[87071]: 6.16 scrub starts 2024-09-13T14:46:48.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:48 smithi204 ceph-mon[87071]: 6.1 scrub starts 2024-09-13T14:46:48.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:48 smithi204 ceph-mon[87071]: osdmap e185: 8 total, 8 up, 7 in 2024-09-13T14:46:48.483 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:48.509 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:48.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:48.611 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:48.648 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:48.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:48.750 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:48.780 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:48.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:48.882 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:48.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.913 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:48.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:49.014 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:49.053 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:49.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:49.154 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:49.186 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:49.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:49.287 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:49.316 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:49.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:49 smithi096 ceph-mon[98901]: 6.6 scrub starts 2024-09-13T14:46:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:49 smithi096 ceph-mon[98901]: pgmap v532: 109 pgs: 1 remapped+peering, 1 active+clean+remapped, 3 active+remapped+backfill_wait, 3 active+remapped+backfilling, 101 active+clean; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 7022/213687 objects misplaced (3.286%); 34 MiB/s, 546 objects/s recovering 2024-09-13T14:46:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:49 smithi096 ceph-mon[98901]: 6.4 scrub starts 2024-09-13T14:46:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:49 smithi096 ceph-mon[98901]: osdmap e186: 8 total, 8 up, 7 in 2024-09-13T14:46:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:49 smithi096 ceph-mon[98901]: 6.11 continuing backfill to osd.0 from (153'4140,183'9943] 6:881de9ac:::benchmark_data_smithi096_167087_object114036:head to 183'9943 2024-09-13T14:46:49.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:49 smithi096 ceph-mon[98901]: 6.11 starting backfill to osd.5 from (0'0,0'0] MAX to 183'9943 2024-09-13T14:46:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:49 smithi096 ceph-mon[102928]: 6.6 scrub starts 2024-09-13T14:46:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:49 smithi096 ceph-mon[102928]: pgmap v532: 109 pgs: 1 remapped+peering, 1 active+clean+remapped, 3 active+remapped+backfill_wait, 3 active+remapped+backfilling, 101 active+clean; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 7022/213687 objects misplaced (3.286%); 34 MiB/s, 546 objects/s recovering 2024-09-13T14:46:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:49 smithi096 ceph-mon[102928]: 6.4 scrub starts 2024-09-13T14:46:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:49 smithi096 ceph-mon[102928]: osdmap e186: 8 total, 8 up, 7 in 2024-09-13T14:46:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:49 smithi096 ceph-mon[102928]: 6.11 continuing backfill to osd.0 from (153'4140,183'9943] 6:881de9ac:::benchmark_data_smithi096_167087_object114036:head to 183'9943 2024-09-13T14:46:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:49 smithi096 ceph-mon[102928]: 6.11 starting backfill to osd.5 from (0'0,0'0] MAX to 183'9943 2024-09-13T14:46:49.418 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:49.441 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:49.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:49 smithi204 ceph-mon[87071]: 6.6 scrub starts 2024-09-13T14:46:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:49 smithi204 ceph-mon[87071]: pgmap v532: 109 pgs: 1 remapped+peering, 1 active+clean+remapped, 3 active+remapped+backfill_wait, 3 active+remapped+backfilling, 101 active+clean; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 7022/213687 objects misplaced (3.286%); 34 MiB/s, 546 objects/s recovering 2024-09-13T14:46:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:49 smithi204 ceph-mon[87071]: 6.4 scrub starts 2024-09-13T14:46:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:49 smithi204 ceph-mon[87071]: osdmap e186: 8 total, 8 up, 7 in 2024-09-13T14:46:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:49 smithi204 ceph-mon[87071]: 6.11 continuing backfill to osd.0 from (153'4140,183'9943] 6:881de9ac:::benchmark_data_smithi096_167087_object114036:head to 183'9943 2024-09-13T14:46:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:49 smithi204 ceph-mon[87071]: 6.11 starting backfill to osd.5 from (0'0,0'0] MAX to 183'9943 2024-09-13T14:46:49.543 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:49.569 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:49.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:49.671 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:49.694 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:49.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:49.796 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:49.819 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:49.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:49.920 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:49.946 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:49.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:50.007 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:46:50.048 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:50.080 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:50.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:50.182 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:50.204 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:50.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:50.306 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:50.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:50 smithi096 ceph-mon[98901]: 6.9 scrub starts 2024-09-13T14:46:50.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:50 smithi096 ceph-mon[98901]: osdmap e187: 8 total, 8 up, 7 in 2024-09-13T14:46:50.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:50 smithi096 ceph-mon[102928]: 6.9 scrub starts 2024-09-13T14:46:50.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:50 smithi096 ceph-mon[102928]: osdmap e187: 8 total, 8 up, 7 in 2024-09-13T14:46:50.334 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:50.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:50.369 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:50 smithi204 ceph-mon[87071]: 6.9 scrub starts 2024-09-13T14:46:50.369 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:50 smithi204 ceph-mon[87071]: osdmap e187: 8 total, 8 up, 7 in 2024-09-13T14:46:50.369 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:46:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:46:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:50.436 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:50.458 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:50.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:50.559 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:50.593 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:50.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:50.695 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:50.731 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:50.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:50.833 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:50.864 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:50.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:50.966 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:51.000 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:51.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:51.102 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:51.134 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:51.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:51.235 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:51.277 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:51.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:51 smithi096 ceph-mon[98901]: pgmap v535: 109 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 103 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 7843/212934 objects misplaced (3.683%); 53 MiB/s, 854 objects/s recovering 2024-09-13T14:46:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:51 smithi096 ceph-mon[98901]: osdmap e188: 8 total, 8 up, 7 in 2024-09-13T14:46:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:51 smithi096 ceph-mon[102928]: pgmap v535: 109 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 103 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 7843/212934 objects misplaced (3.683%); 53 MiB/s, 854 objects/s recovering 2024-09-13T14:46:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:51 smithi096 ceph-mon[102928]: osdmap e188: 8 total, 8 up, 7 in 2024-09-13T14:46:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:51.380 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:51.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.416 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:51.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:51 smithi204 ceph-mon[87071]: pgmap v535: 109 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 103 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 7843/212934 objects misplaced (3.683%); 53 MiB/s, 854 objects/s recovering 2024-09-13T14:46:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:51 smithi204 ceph-mon[87071]: osdmap e188: 8 total, 8 up, 7 in 2024-09-13T14:46:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:51.517 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:51.554 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:51.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:51.656 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:51.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.689 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:51.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:51.790 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:51.815 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:51.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:51.917 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:51.951 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:51.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:52.053 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:52.081 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:52.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:52.183 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:52.212 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:52.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:52.314 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:52.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.342 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:52.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:52.443 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:52.468 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:52.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:52.570 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:52.595 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:52.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:52.697 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:52.720 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:52.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:52.822 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:52.848 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:52.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:52.950 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:52.976 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:52.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:53.078 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:53.105 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:53.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:53.207 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:53.245 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:53.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:53.347 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:53.372 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:53.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.421 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:53.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:53.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:53 smithi204 ceph-mon[87071]: 6.0 scrub starts 2024-09-13T14:46:53.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:53 smithi204 ceph-mon[87071]: pgmap v537: 109 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 103 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 390 op/s; 7843/210414 objects misplaced (3.727%); 45 MiB/s, 718 objects/s recovering 2024-09-13T14:46:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:53 smithi204 ceph-mon[87071]: 6.1 scrub starts 2024-09-13T14:46:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:53 smithi204 ceph-mon[87071]: osdmap e189: 8 total, 8 up, 7 in 2024-09-13T14:46:53.522 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:53.538 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:53 smithi096 ceph-mon[98901]: 6.0 scrub starts 2024-09-13T14:46:53.539 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:53 smithi096 ceph-mon[98901]: pgmap v537: 109 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 103 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 390 op/s; 7843/210414 objects misplaced (3.727%); 45 MiB/s, 718 objects/s recovering 2024-09-13T14:46:53.539 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:53 smithi096 ceph-mon[98901]: 6.1 scrub starts 2024-09-13T14:46:53.539 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:53 smithi096 ceph-mon[98901]: osdmap e189: 8 total, 8 up, 7 in 2024-09-13T14:46:53.541 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:53 smithi096 ceph-mon[102928]: 6.0 scrub starts 2024-09-13T14:46:53.541 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:53 smithi096 ceph-mon[102928]: pgmap v537: 109 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 103 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 390 op/s; 7843/210414 objects misplaced (3.727%); 45 MiB/s, 718 objects/s recovering 2024-09-13T14:46:53.542 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:53 smithi096 ceph-mon[102928]: 6.1 scrub starts 2024-09-13T14:46:53.542 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:53 smithi096 ceph-mon[102928]: osdmap e189: 8 total, 8 up, 7 in 2024-09-13T14:46:53.552 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:53.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:53.654 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:53.690 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:53.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:53.792 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:53.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.838 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:53.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:53.939 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:53.978 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:53.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:54.080 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:54.109 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:54.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:54.210 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:54.263 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:54.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:54.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:54 smithi096 ceph-mon[98901]: 6.16 scrub starts 2024-09-13T14:46:54.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:54 smithi096 ceph-mon[98901]: 6.4 scrub starts 2024-09-13T14:46:54.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:54 smithi096 ceph-mon[98901]: osdmap e190: 8 total, 8 up, 7 in 2024-09-13T14:46:54.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:54 smithi096 ceph-mon[102928]: 6.16 scrub starts 2024-09-13T14:46:54.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:54 smithi096 ceph-mon[102928]: 6.4 scrub starts 2024-09-13T14:46:54.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:54 smithi096 ceph-mon[102928]: osdmap e190: 8 total, 8 up, 7 in 2024-09-13T14:46:54.366 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:54.395 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:54.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:54.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:54 smithi204 ceph-mon[87071]: 6.16 scrub starts 2024-09-13T14:46:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:54 smithi204 ceph-mon[87071]: 6.4 scrub starts 2024-09-13T14:46:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:54 smithi204 ceph-mon[87071]: osdmap e190: 8 total, 8 up, 7 in 2024-09-13T14:46:54.497 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:54.526 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:54.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:54.628 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:54.668 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:54.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:54.769 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:54.805 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:54.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:54.907 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:54.934 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:54.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:55.035 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:55.042 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:55.061 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:55.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:55.088 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:55.162 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:55.191 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:55.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:55.293 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:55.323 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:55.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:55.354 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:55 smithi096 ceph-mon[98901]: 6.9 scrub starts 2024-09-13T14:46:55.354 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:55 smithi096 ceph-mon[98901]: 6.6 scrub starts 2024-09-13T14:46:55.354 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:55 smithi096 ceph-mon[98901]: pgmap v540: 109 pgs: 1 active+remapped+backfill_wait, 1 peering, 3 active+remapped+backfilling, 104 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 7254/203562 objects misplaced (3.564%); 41 MiB/s, 659 objects/s recovering 2024-09-13T14:46:55.354 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:55 smithi096 ceph-mon[98901]: 6.14 scrub starts 2024-09-13T14:46:55.354 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:55 smithi096 ceph-mon[102928]: 6.9 scrub starts 2024-09-13T14:46:55.354 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:55 smithi096 ceph-mon[102928]: 6.6 scrub starts 2024-09-13T14:46:55.355 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:55 smithi096 ceph-mon[102928]: pgmap v540: 109 pgs: 1 active+remapped+backfill_wait, 1 peering, 3 active+remapped+backfilling, 104 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 7254/203562 objects misplaced (3.564%); 41 MiB/s, 659 objects/s recovering 2024-09-13T14:46:55.355 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:55 smithi096 ceph-mon[102928]: 6.14 scrub starts 2024-09-13T14:46:55.425 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:55.466 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:55.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:55 smithi204 ceph-mon[87071]: 6.9 scrub starts 2024-09-13T14:46:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:55 smithi204 ceph-mon[87071]: 6.6 scrub starts 2024-09-13T14:46:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:55 smithi204 ceph-mon[87071]: pgmap v540: 109 pgs: 1 active+remapped+backfill_wait, 1 peering, 3 active+remapped+backfilling, 104 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 7254/203562 objects misplaced (3.564%); 41 MiB/s, 659 objects/s recovering 2024-09-13T14:46:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:55 smithi204 ceph-mon[87071]: 6.14 scrub starts 2024-09-13T14:46:55.568 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:55.597 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:55.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:55.699 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:55.734 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:55.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:55.836 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:55.887 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:55.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:55.989 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:56.039 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:56.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:56.141 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:56.167 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:56.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:56.269 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:56.299 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:56.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:56.401 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:56.442 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:56.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:56.544 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:56.574 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:56.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:56.676 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:56.708 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:56.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:56.810 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:56.848 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:56.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:56.950 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:56.983 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:56.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:57.084 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:57.122 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:57.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:57.224 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:57.249 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:57.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:57.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:57 smithi096 ceph-mon[98901]: pgmap v541: 109 pgs: 1 active+remapped+backfill_wait, 1 peering, 3 active+remapped+backfilling, 104 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 7254/203562 objects misplaced (3.564%); 32 MiB/s, 519 objects/s recovering 2024-09-13T14:46:57.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:57 smithi096 ceph-mon[102928]: pgmap v541: 109 pgs: 1 active+remapped+backfill_wait, 1 peering, 3 active+remapped+backfilling, 104 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 7254/203562 objects misplaced (3.564%); 32 MiB/s, 519 objects/s recovering 2024-09-13T14:46:57.351 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:57.377 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:57.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:57 smithi204 ceph-mon[87071]: pgmap v541: 109 pgs: 1 active+remapped+backfill_wait, 1 peering, 3 active+remapped+backfilling, 104 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 7254/203562 objects misplaced (3.564%); 32 MiB/s, 519 objects/s recovering 2024-09-13T14:46:57.480 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:57.505 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:57.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:57.607 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:57.630 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:57.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:57.731 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:57.752 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:57.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:57.854 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:57.877 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:57.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:57.979 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:58.002 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:58.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:58.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:46:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:46:57] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-09-13T14:46:58.105 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:58.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.130 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:58.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:58.231 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:58.270 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:58.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:58.372 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:58.379 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:58 smithi096 ceph-mon[98901]: osdmap e191: 8 total, 8 up, 7 in 2024-09-13T14:46:58.379 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:58 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:46:58.379 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:58 smithi096 ceph-mon[102928]: osdmap e191: 8 total, 8 up, 7 in 2024-09-13T14:46:58.379 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:58 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:46:58.397 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:58.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:58.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:58 smithi204 ceph-mon[87071]: osdmap e191: 8 total, 8 up, 7 in 2024-09-13T14:46:58.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:58 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:46:58.499 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:58.527 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:58.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:58.629 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:58.651 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:58.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:58.753 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:58.775 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:58.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:58.877 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:58.899 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:58.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:59.000 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:59.021 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:59.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:59.123 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:59.150 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:46:59.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:59.253 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:59.277 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:59.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:59.379 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:59.386 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T14:46:59.386 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T14:46:59.386 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T14:46:59.386 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T14:46:59.386 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T14:46:59.386 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T14:46:59.386 INFO:teuthology.orchestra.run.smithi096.stderr:bench [] [] [] [] 2024-09-13T14:46:59.386 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:46:59.386 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T14:46:59.386 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T14:46:59.386 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore allocator dump block 2024-09-13T14:46:59.386 INFO:teuthology.orchestra.run.smithi096.stderr:Error EINVAL: invalid command 2024-09-13T14:46:59.387 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:59 smithi096 ceph-mon[98901]: 6.0 scrub starts 2024-09-13T14:46:59.387 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:59 smithi096 ceph-mon[98901]: 6.16 scrub starts 2024-09-13T14:46:59.387 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:59 smithi096 ceph-mon[98901]: pgmap v543: 109 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 105 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 4739/190059 objects misplaced (2.493%); 48 MiB/s, 773 objects/s recovering 2024-09-13T14:46:59.387 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:59 smithi096 ceph-mon[98901]: 6.1 scrub starts 2024-09-13T14:46:59.387 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:59 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:46:59.387 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:59 smithi096 ceph-mon[98901]: osdmap e192: 8 total, 8 up, 7 in 2024-09-13T14:46:59.387 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:46:59 smithi096 ceph-mon[98901]: osdmap e193: 8 total, 8 up, 7 in 2024-09-13T14:46:59.387 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:59 smithi096 ceph-mon[102928]: 6.0 scrub starts 2024-09-13T14:46:59.388 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:59 smithi096 ceph-mon[102928]: 6.16 scrub starts 2024-09-13T14:46:59.388 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:59 smithi096 ceph-mon[102928]: pgmap v543: 109 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 105 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 4739/190059 objects misplaced (2.493%); 48 MiB/s, 773 objects/s recovering 2024-09-13T14:46:59.388 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:59 smithi096 ceph-mon[102928]: 6.1 scrub starts 2024-09-13T14:46:59.388 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:59 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:46:59.388 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:59 smithi096 ceph-mon[102928]: osdmap e192: 8 total, 8 up, 7 in 2024-09-13T14:46:59.388 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:46:59 smithi096 ceph-mon[102928]: osdmap e193: 8 total, 8 up, 7 in 2024-09-13T14:46:59.401 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:59.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:59 smithi204 ceph-mon[87071]: 6.0 scrub starts 2024-09-13T14:46:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:59 smithi204 ceph-mon[87071]: 6.16 scrub starts 2024-09-13T14:46:59.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:59 smithi204 ceph-mon[87071]: pgmap v543: 109 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 105 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 4739/190059 objects misplaced (2.493%); 48 MiB/s, 773 objects/s recovering 2024-09-13T14:46:59.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:59 smithi204 ceph-mon[87071]: 6.1 scrub starts 2024-09-13T14:46:59.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:59 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:46:59.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:59 smithi204 ceph-mon[87071]: osdmap e192: 8 total, 8 up, 7 in 2024-09-13T14:46:59.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:46:59 smithi204 ceph-mon[87071]: osdmap e193: 8 total, 8 up, 7 in 2024-09-13T14:46:59.503 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:59.525 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:59.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:59.627 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:59.648 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:59.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:59.750 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:59.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.772 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:59.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:59.873 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:59.894 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:46:59.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:59.996 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:00.017 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:00.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:00.119 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:00.148 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:00.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:00.250 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:00.292 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:00.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:00.394 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:00.427 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:00.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:00 smithi204 ceph-mon[87071]: 6.0 scrub starts 2024-09-13T14:47:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:00 smithi204 ceph-mon[87071]: 6.16 scrub starts 2024-09-13T14:47:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:00 smithi204 ceph-mon[87071]: 6.1 scrub starts 2024-09-13T14:47:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:00 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T14:47:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:00 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T14:47:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:00 smithi204 ceph-mon[87071]: osdmap e194: 8 total, 8 up, 7 in 2024-09-13T14:47:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:47:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:47:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:00.529 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:00.556 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:00.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:00.559 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:00 smithi096 ceph-mon[98901]: 6.0 scrub starts 2024-09-13T14:47:00.559 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:00 smithi096 ceph-mon[98901]: 6.16 scrub starts 2024-09-13T14:47:00.559 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:00 smithi096 ceph-mon[98901]: 6.1 scrub starts 2024-09-13T14:47:00.559 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:00 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T14:47:00.559 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:00 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T14:47:00.559 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:00 smithi096 ceph-mon[98901]: osdmap e194: 8 total, 8 up, 7 in 2024-09-13T14:47:00.559 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:00 smithi096 ceph-mon[102928]: 6.0 scrub starts 2024-09-13T14:47:00.559 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:00 smithi096 ceph-mon[102928]: 6.16 scrub starts 2024-09-13T14:47:00.559 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:00 smithi096 ceph-mon[102928]: 6.1 scrub starts 2024-09-13T14:47:00.559 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:00 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T14:47:00.560 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:00 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T14:47:00.560 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:00 smithi096 ceph-mon[102928]: osdmap e194: 8 total, 8 up, 7 in 2024-09-13T14:47:00.658 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:00.681 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:00.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:00.782 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:00.804 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:00.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:00.905 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:00.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.929 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:00.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:01.030 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:01.053 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:01.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:01.154 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:01.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.214 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:01.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:01.315 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:01.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:01 smithi096 ceph-mon[98901]: 6.9 scrub starts 2024-09-13T14:47:01.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:01 smithi096 ceph-mon[98901]: 6.6 scrub starts 2024-09-13T14:47:01.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:01 smithi096 ceph-mon[98901]: pgmap v546: 109 pgs: 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 3.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 4066/188268 objects misplaced (2.160%); 20 MiB/s, 318 objects/s recovering 2024-09-13T14:47:01.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:01 smithi096 ceph-mon[98901]: 6.4 scrub starts 2024-09-13T14:47:01.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:01 smithi096 ceph-mon[98901]: osdmap e195: 8 total, 8 up, 7 in 2024-09-13T14:47:01.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:01 smithi096 ceph-mon[102928]: 6.9 scrub starts 2024-09-13T14:47:01.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:01 smithi096 ceph-mon[102928]: 6.6 scrub starts 2024-09-13T14:47:01.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:01 smithi096 ceph-mon[102928]: pgmap v546: 109 pgs: 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 3.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 4066/188268 objects misplaced (2.160%); 20 MiB/s, 318 objects/s recovering 2024-09-13T14:47:01.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:01 smithi096 ceph-mon[102928]: 6.4 scrub starts 2024-09-13T14:47:01.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:01 smithi096 ceph-mon[102928]: osdmap e195: 8 total, 8 up, 7 in 2024-09-13T14:47:01.336 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:01.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:01.437 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:01.455 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:01 smithi204 ceph-mon[87071]: 6.9 scrub starts 2024-09-13T14:47:01.455 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:01 smithi204 ceph-mon[87071]: 6.6 scrub starts 2024-09-13T14:47:01.455 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:01 smithi204 ceph-mon[87071]: pgmap v546: 109 pgs: 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 3.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 4066/188268 objects misplaced (2.160%); 20 MiB/s, 318 objects/s recovering 2024-09-13T14:47:01.455 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:01 smithi204 ceph-mon[87071]: 6.4 scrub starts 2024-09-13T14:47:01.455 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:01 smithi204 ceph-mon[87071]: osdmap e195: 8 total, 8 up, 7 in 2024-09-13T14:47:01.459 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:01.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:01.561 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:01.582 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:01.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:01.684 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:01.685 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:47:01.685 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:47:01.686 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.1 'trim stale osdmaps' 2024-09-13T14:47:01.749 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:01.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:01.851 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:01.875 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:01.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:01.977 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:01.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.000 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:02.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:02.101 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:02.122 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:02.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:02.224 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:02.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.264 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:02.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:02.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:02 smithi096 ceph-mon[98901]: 6.0 scrub starts 2024-09-13T14:47:02.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:02 smithi096 ceph-mon[98901]: 6.16 scrub starts 2024-09-13T14:47:02.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:02 smithi096 ceph-mon[98901]: 6.1 scrub starts 2024-09-13T14:47:02.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:02 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:47:02.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:02 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:47:02.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:02 smithi096 ceph-mon[98901]: osdmap e196: 8 total, 8 up, 7 in 2024-09-13T14:47:02.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:02 smithi096 ceph-mon[102928]: 6.0 scrub starts 2024-09-13T14:47:02.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:02 smithi096 ceph-mon[102928]: 6.16 scrub starts 2024-09-13T14:47:02.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:02 smithi096 ceph-mon[102928]: 6.1 scrub starts 2024-09-13T14:47:02.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:02 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:47:02.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:02 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:47:02.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:02 smithi096 ceph-mon[102928]: osdmap e196: 8 total, 8 up, 7 in 2024-09-13T14:47:02.365 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:02.410 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:02.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:02.448 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:47:02.449 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:47:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:02 smithi204 ceph-mon[87071]: 6.0 scrub starts 2024-09-13T14:47:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:02 smithi204 ceph-mon[87071]: 6.16 scrub starts 2024-09-13T14:47:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:02 smithi204 ceph-mon[87071]: 6.1 scrub starts 2024-09-13T14:47:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:02 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:47:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:02 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:47:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:02 smithi204 ceph-mon[87071]: osdmap e196: 8 total, 8 up, 7 in 2024-09-13T14:47:02.512 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:02.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.540 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:02.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:02.641 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:02.677 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:02.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:02.779 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:02.802 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:02.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:02.904 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:02.931 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:02.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:03.033 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:03.054 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:03.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:03.097 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:47:03.101 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:47:03.156 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:03.178 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:03.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:03.279 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:03.301 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:03.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:03.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[98901]: 6.19 scrub starts 2024-09-13T14:47:03.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[98901]: 6.6 scrub starts 2024-09-13T14:47:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[98901]: pgmap v549: 109 pgs: 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 4066/186570 objects misplaced (2.179%); 6.0 MiB/s, 95 objects/s recovering 2024-09-13T14:47:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[98901]: 6.4 scrub starts 2024-09-13T14:47:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[98901]: 6.13 starting backfill to osd.0 from (0'0,0'0] MAX to 193'10317 2024-09-13T14:47:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[98901]: 6.13 starting backfill to osd.3 from (0'0,0'0] MAX to 193'10317 2024-09-13T14:47:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4153360447' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:47:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[98901]: osdmap e197: 8 total, 8 up, 7 in 2024-09-13T14:47:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[102928]: 6.19 scrub starts 2024-09-13T14:47:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[102928]: 6.6 scrub starts 2024-09-13T14:47:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[102928]: pgmap v549: 109 pgs: 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 4066/186570 objects misplaced (2.179%); 6.0 MiB/s, 95 objects/s recovering 2024-09-13T14:47:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[102928]: 6.4 scrub starts 2024-09-13T14:47:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[102928]: 6.13 starting backfill to osd.0 from (0'0,0'0] MAX to 193'10317 2024-09-13T14:47:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[102928]: 6.13 starting backfill to osd.3 from (0'0,0'0] MAX to 193'10317 2024-09-13T14:47:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4153360447' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:47:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:03 smithi096 ceph-mon[102928]: osdmap e197: 8 total, 8 up, 7 in 2024-09-13T14:47:03.403 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:03.424 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:03.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:03.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:03 smithi204 ceph-mon[87071]: 6.19 scrub starts 2024-09-13T14:47:03.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:03 smithi204 ceph-mon[87071]: 6.6 scrub starts 2024-09-13T14:47:03.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:03 smithi204 ceph-mon[87071]: pgmap v549: 109 pgs: 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 4066/186570 objects misplaced (2.179%); 6.0 MiB/s, 95 objects/s recovering 2024-09-13T14:47:03.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:03 smithi204 ceph-mon[87071]: 6.4 scrub starts 2024-09-13T14:47:03.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:03 smithi204 ceph-mon[87071]: 6.13 starting backfill to osd.0 from (0'0,0'0] MAX to 193'10317 2024-09-13T14:47:03.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:03 smithi204 ceph-mon[87071]: 6.13 starting backfill to osd.3 from (0'0,0'0] MAX to 193'10317 2024-09-13T14:47:03.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:03 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4153360447' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:03.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:03 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:03.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:03 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:03.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:03 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:47:03.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:03 smithi204 ceph-mon[87071]: osdmap e197: 8 total, 8 up, 7 in 2024-09-13T14:47:03.527 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:03.547 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:03.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:03.649 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:03.680 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:03.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:03.782 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:03.817 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:03.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:03.920 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:03.943 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:03.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:04.045 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:04.072 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:04.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:04.174 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:04.200 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:04.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:04.302 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:04.342 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:04.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:04.440 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:04 smithi096 ceph-mon[98901]: 6.19 scrub starts 2024-09-13T14:47:04.440 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:04 smithi096 ceph-mon[98901]: 6.16 scrub starts 2024-09-13T14:47:04.441 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:04 smithi096 ceph-mon[98901]: 6.1 scrub starts 2024-09-13T14:47:04.441 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:04 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-13T14:47:04.441 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:04 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-13T14:47:04.441 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:04 smithi096 ceph-mon[98901]: osdmap e198: 8 total, 8 up, 7 in 2024-09-13T14:47:04.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:04 smithi096 ceph-mon[102928]: 6.19 scrub starts 2024-09-13T14:47:04.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:04 smithi096 ceph-mon[102928]: 6.16 scrub starts 2024-09-13T14:47:04.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:04 smithi096 ceph-mon[102928]: 6.1 scrub starts 2024-09-13T14:47:04.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:04 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-13T14:47:04.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:04 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-13T14:47:04.442 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:04 smithi096 ceph-mon[102928]: osdmap e198: 8 total, 8 up, 7 in 2024-09-13T14:47:04.444 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:04.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:04 smithi204 ceph-mon[87071]: 6.19 scrub starts 2024-09-13T14:47:04.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:04 smithi204 ceph-mon[87071]: 6.16 scrub starts 2024-09-13T14:47:04.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:04 smithi204 ceph-mon[87071]: 6.1 scrub starts 2024-09-13T14:47:04.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:04 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-13T14:47:04.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:04 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-13T14:47:04.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:04 smithi204 ceph-mon[87071]: osdmap e198: 8 total, 8 up, 7 in 2024-09-13T14:47:04.470 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:04.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:04.572 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:04.599 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:04.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:04.701 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:04.743 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:04.746 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:04.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:04.847 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:04.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.877 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:04.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:04.978 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:05.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.009 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:05.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:05.110 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:05.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.141 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:05.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:05.242 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:05.254 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[98901]: 6.9 scrub starts 2024-09-13T14:47:05.254 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[98901]: 6.3 scrub starts 2024-09-13T14:47:05.254 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[98901]: 6.6 scrub starts 2024-09-13T14:47:05.254 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[98901]: pgmap v552: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 106 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 408 op/s; 4281/183360 objects misplaced (2.335%); 53 MiB/s, 854 objects/s recovering 2024-09-13T14:47:05.254 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[98901]: 6.9 scrub ok 2024-09-13T14:47:05.255 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[98901]: 6.3 scrub ok 2024-09-13T14:47:05.255 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[98901]: 6.2 scrub starts 2024-09-13T14:47:05.255 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[98901]: 6.4 scrub starts 2024-09-13T14:47:05.255 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[98901]: 6.6 scrub ok 2024-09-13T14:47:05.255 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[98901]: 6.14 starting backfill to osd.2 from (0'0,0'0] MAX to 194'10440 2024-09-13T14:47:05.255 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[98901]: 6.14 starting backfill to osd.7 from (0'0,0'0] MAX to 194'10440 2024-09-13T14:47:05.255 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[98901]: 6.2 scrub ok 2024-09-13T14:47:05.255 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[98901]: 6.4 scrub ok 2024-09-13T14:47:05.255 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[98901]: osdmap e199: 8 total, 8 up, 7 in 2024-09-13T14:47:05.255 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[102928]: 6.9 scrub starts 2024-09-13T14:47:05.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[102928]: 6.3 scrub starts 2024-09-13T14:47:05.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[102928]: 6.6 scrub starts 2024-09-13T14:47:05.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[102928]: pgmap v552: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 106 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 408 op/s; 4281/183360 objects misplaced (2.335%); 53 MiB/s, 854 objects/s recovering 2024-09-13T14:47:05.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[102928]: 6.9 scrub ok 2024-09-13T14:47:05.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[102928]: 6.3 scrub ok 2024-09-13T14:47:05.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[102928]: 6.2 scrub starts 2024-09-13T14:47:05.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[102928]: 6.4 scrub starts 2024-09-13T14:47:05.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[102928]: 6.6 scrub ok 2024-09-13T14:47:05.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[102928]: 6.14 starting backfill to osd.2 from (0'0,0'0] MAX to 194'10440 2024-09-13T14:47:05.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[102928]: 6.14 starting backfill to osd.7 from (0'0,0'0] MAX to 194'10440 2024-09-13T14:47:05.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[102928]: 6.2 scrub ok 2024-09-13T14:47:05.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[102928]: 6.4 scrub ok 2024-09-13T14:47:05.256 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:05 smithi096 ceph-mon[102928]: osdmap e199: 8 total, 8 up, 7 in 2024-09-13T14:47:05.263 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:05.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:05.365 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:05.390 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:05.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:05 smithi204 ceph-mon[87071]: 6.9 scrub starts 2024-09-13T14:47:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:05 smithi204 ceph-mon[87071]: 6.3 scrub starts 2024-09-13T14:47:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:05 smithi204 ceph-mon[87071]: 6.6 scrub starts 2024-09-13T14:47:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:05 smithi204 ceph-mon[87071]: pgmap v552: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 106 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 408 op/s; 4281/183360 objects misplaced (2.335%); 53 MiB/s, 854 objects/s recovering 2024-09-13T14:47:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:05 smithi204 ceph-mon[87071]: 6.9 scrub ok 2024-09-13T14:47:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:05 smithi204 ceph-mon[87071]: 6.3 scrub ok 2024-09-13T14:47:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:05 smithi204 ceph-mon[87071]: 6.2 scrub starts 2024-09-13T14:47:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:05 smithi204 ceph-mon[87071]: 6.4 scrub starts 2024-09-13T14:47:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:05 smithi204 ceph-mon[87071]: 6.6 scrub ok 2024-09-13T14:47:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:05 smithi204 ceph-mon[87071]: 6.14 starting backfill to osd.2 from (0'0,0'0] MAX to 194'10440 2024-09-13T14:47:05.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:05 smithi204 ceph-mon[87071]: 6.14 starting backfill to osd.7 from (0'0,0'0] MAX to 194'10440 2024-09-13T14:47:05.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:05 smithi204 ceph-mon[87071]: 6.2 scrub ok 2024-09-13T14:47:05.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:05 smithi204 ceph-mon[87071]: 6.4 scrub ok 2024-09-13T14:47:05.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:05 smithi204 ceph-mon[87071]: osdmap e199: 8 total, 8 up, 7 in 2024-09-13T14:47:05.491 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:05.513 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:05.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:05.614 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:05.643 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:05.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:05.740 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:47:05.740 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:05.746 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:05.767 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:47:05.767 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:05.776 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:05.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:05.795 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:47:05.795 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:05.817 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:47:05.817 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:05.845 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:47:05.845 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:05.875 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:47:05.875 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:05.878 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:05.894 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:47:05.894 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:05.901 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:05.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:05.914 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:47:05.914 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:06.003 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:06.029 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:06.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:06.131 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:06.157 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:06.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:06.259 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:47:06.260 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:06.289 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:06.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:06.391 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:06.428 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:06.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:06.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:06 smithi204 ceph-mon[87071]: 6.19 scrub starts 2024-09-13T14:47:06.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:06 smithi204 ceph-mon[87071]: 6.16 scrub starts 2024-09-13T14:47:06.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:06 smithi204 ceph-mon[87071]: 6.1 scrub starts 2024-09-13T14:47:06.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:06 smithi204 ceph-mon[87071]: 6.19 scrub ok 2024-09-13T14:47:06.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:06 smithi204 ceph-mon[87071]: 6.16 scrub ok 2024-09-13T14:47:06.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:06 smithi204 ceph-mon[87071]: 6.1 scrub ok 2024-09-13T14:47:06.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:06 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/872070877' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:06.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-13T14:47:06.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:06.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-13T14:47:06.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:06 smithi204 ceph-mon[87071]: osdmap e200: 8 total, 8 up, 7 in 2024-09-13T14:47:06.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[98901]: 6.19 scrub starts 2024-09-13T14:47:06.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[98901]: 6.16 scrub starts 2024-09-13T14:47:06.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[98901]: 6.1 scrub starts 2024-09-13T14:47:06.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[98901]: 6.19 scrub ok 2024-09-13T14:47:06.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[98901]: 6.16 scrub ok 2024-09-13T14:47:06.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[98901]: 6.1 scrub ok 2024-09-13T14:47:06.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/872070877' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:06.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-13T14:47:06.522 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:06.522 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-13T14:47:06.522 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[98901]: osdmap e200: 8 total, 8 up, 7 in 2024-09-13T14:47:06.522 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[102928]: 6.19 scrub starts 2024-09-13T14:47:06.522 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[102928]: 6.16 scrub starts 2024-09-13T14:47:06.522 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[102928]: 6.1 scrub starts 2024-09-13T14:47:06.522 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[102928]: 6.19 scrub ok 2024-09-13T14:47:06.522 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[102928]: 6.16 scrub ok 2024-09-13T14:47:06.522 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[102928]: 6.1 scrub ok 2024-09-13T14:47:06.522 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/872070877' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:06.522 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-13T14:47:06.523 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:06.523 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-13T14:47:06.523 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:06 smithi096 ceph-mon[102928]: osdmap e200: 8 total, 8 up, 7 in 2024-09-13T14:47:06.531 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:06.554 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:06.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:06.656 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:06.680 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:06.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:06.782 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:06.807 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:06.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:06.908 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:06.933 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:06.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:07.035 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:07.059 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:07.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:07.161 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:07.197 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:07.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:07.299 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:07.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:07 smithi096 ceph-mon[98901]: 6.11 scrub starts 2024-09-13T14:47:07.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:07 smithi096 ceph-mon[98901]: pgmap v555: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 106 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 267 op/s; 4281/183360 objects misplaced (2.335%); 53 MiB/s, 854 objects/s recovering 2024-09-13T14:47:07.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:07 smithi096 ceph-mon[98901]: 6.11 scrub ok 2024-09-13T14:47:07.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:07 smithi096 ceph-mon[98901]: 6.15 starting backfill to osd.4 from (0'0,0'0] MAX to 196'10297 2024-09-13T14:47:07.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:07 smithi096 ceph-mon[98901]: 6.15 starting backfill to osd.5 from (0'0,0'0] MAX to 196'10297 2024-09-13T14:47:07.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:07 smithi096 ceph-mon[98901]: osdmap e201: 8 total, 8 up, 7 in 2024-09-13T14:47:07.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:07 smithi096 ceph-mon[102928]: 6.11 scrub starts 2024-09-13T14:47:07.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:07 smithi096 ceph-mon[102928]: pgmap v555: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 106 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 267 op/s; 4281/183360 objects misplaced (2.335%); 53 MiB/s, 854 objects/s recovering 2024-09-13T14:47:07.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:07 smithi096 ceph-mon[102928]: 6.11 scrub ok 2024-09-13T14:47:07.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:07 smithi096 ceph-mon[102928]: 6.15 starting backfill to osd.4 from (0'0,0'0] MAX to 196'10297 2024-09-13T14:47:07.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:07 smithi096 ceph-mon[102928]: 6.15 starting backfill to osd.5 from (0'0,0'0] MAX to 196'10297 2024-09-13T14:47:07.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:07 smithi096 ceph-mon[102928]: osdmap e201: 8 total, 8 up, 7 in 2024-09-13T14:47:07.325 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:07.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:07.427 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:07.453 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:07.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:07.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:07 smithi204 ceph-mon[87071]: 6.11 scrub starts 2024-09-13T14:47:07.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:07 smithi204 ceph-mon[87071]: pgmap v555: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 106 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 267 op/s; 4281/183360 objects misplaced (2.335%); 53 MiB/s, 854 objects/s recovering 2024-09-13T14:47:07.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:07 smithi204 ceph-mon[87071]: 6.11 scrub ok 2024-09-13T14:47:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:07 smithi204 ceph-mon[87071]: 6.15 starting backfill to osd.4 from (0'0,0'0] MAX to 196'10297 2024-09-13T14:47:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:07 smithi204 ceph-mon[87071]: 6.15 starting backfill to osd.5 from (0'0,0'0] MAX to 196'10297 2024-09-13T14:47:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:07 smithi204 ceph-mon[87071]: osdmap e201: 8 total, 8 up, 7 in 2024-09-13T14:47:07.554 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:07.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.579 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:07.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:07.680 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:07.707 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:07.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:07.780 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:07.809 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:07.832 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:07.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:07.934 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:07.957 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:07.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:08.027 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:47:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:47:07] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.51.0" 2024-09-13T14:47:08.060 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:08.084 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:08.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:08.185 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:08.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.217 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:08.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:08.294 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:08 smithi096 ceph-mon[98901]: 6.0 scrub starts 2024-09-13T14:47:08.294 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:08 smithi096 ceph-mon[98901]: 6.0 scrub ok 2024-09-13T14:47:08.294 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:08 smithi096 ceph-mon[98901]: osdmap e202: 8 total, 8 up, 7 in 2024-09-13T14:47:08.295 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:08 smithi096 ceph-mon[102928]: 6.0 scrub starts 2024-09-13T14:47:08.295 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:08 smithi096 ceph-mon[102928]: 6.0 scrub ok 2024-09-13T14:47:08.295 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:08 smithi096 ceph-mon[102928]: osdmap e202: 8 total, 8 up, 7 in 2024-09-13T14:47:08.318 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:08.359 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:08.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:08.461 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:08.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:08 smithi204 ceph-mon[87071]: 6.0 scrub starts 2024-09-13T14:47:08.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:08 smithi204 ceph-mon[87071]: 6.0 scrub ok 2024-09-13T14:47:08.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:08 smithi204 ceph-mon[87071]: osdmap e202: 8 total, 8 up, 7 in 2024-09-13T14:47:08.492 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:08.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:08.595 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:08.624 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:08.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:08.726 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:08.750 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:08.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:08.852 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:08.877 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:08.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:08.979 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:09.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.004 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:09.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:09.105 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:09.131 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:09.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:09.233 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:09.266 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:09.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:09.367 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:09.405 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:09.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:09.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:09 smithi204 ceph-mon[87071]: 6.10 scrub starts 2024-09-13T14:47:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:09 smithi204 ceph-mon[87071]: pgmap v558: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 3 active+remapped+backfilling, 103 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 7687/180078 objects misplaced (4.269%) 2024-09-13T14:47:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:09 smithi204 ceph-mon[87071]: 6.10 scrub ok 2024-09-13T14:47:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:09 smithi204 ceph-mon[87071]: 6.13 scrub starts 2024-09-13T14:47:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:09 smithi204 ceph-mon[87071]: 6.13 scrub ok 2024-09-13T14:47:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:09 smithi204 ceph-mon[87071]: 6.16 starting backfill to osd.0 from (0'0,0'0] MAX to 198'10342 2024-09-13T14:47:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:09 smithi204 ceph-mon[87071]: 6.16 starting backfill to osd.5 from (0'0,0'0] MAX to 198'10342 2024-09-13T14:47:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:09 smithi204 ceph-mon[87071]: osdmap e203: 8 total, 8 up, 7 in 2024-09-13T14:47:09.507 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:09.553 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:09.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:09.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[102928]: 6.10 scrub starts 2024-09-13T14:47:09.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[102928]: pgmap v558: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 3 active+remapped+backfilling, 103 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 7687/180078 objects misplaced (4.269%) 2024-09-13T14:47:09.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[102928]: 6.10 scrub ok 2024-09-13T14:47:09.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[102928]: 6.13 scrub starts 2024-09-13T14:47:09.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[102928]: 6.13 scrub ok 2024-09-13T14:47:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[102928]: 6.16 starting backfill to osd.0 from (0'0,0'0] MAX to 198'10342 2024-09-13T14:47:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[102928]: 6.16 starting backfill to osd.5 from (0'0,0'0] MAX to 198'10342 2024-09-13T14:47:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[102928]: osdmap e203: 8 total, 8 up, 7 in 2024-09-13T14:47:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[98901]: 6.10 scrub starts 2024-09-13T14:47:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[98901]: pgmap v558: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 3 active+remapped+backfilling, 103 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 7687/180078 objects misplaced (4.269%) 2024-09-13T14:47:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[98901]: 6.10 scrub ok 2024-09-13T14:47:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[98901]: 6.13 scrub starts 2024-09-13T14:47:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[98901]: 6.13 scrub ok 2024-09-13T14:47:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[98901]: 6.16 starting backfill to osd.0 from (0'0,0'0] MAX to 198'10342 2024-09-13T14:47:09.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[98901]: 6.16 starting backfill to osd.5 from (0'0,0'0] MAX to 198'10342 2024-09-13T14:47:09.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:09 smithi096 ceph-mon[98901]: osdmap e203: 8 total, 8 up, 7 in 2024-09-13T14:47:09.655 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:09.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.699 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:09.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:09.793 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:47:09.793 INFO:teuthology.orchestra.run.smithi096.stdout: "size": 20, 2024-09-13T14:47:09.793 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 600, 2024-09-13T14:47:09.793 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:47:09.794 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.794 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:67767 5.b 5:de6f400a:::benchmark_data_smithi096_138841_object67766:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:47:09.794 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.201338+0000", 2024-09-13T14:47:09.794 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 404.586656355, 2024-09-13T14:47:09.794 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.12556388199999999, 2024-09-13T14:47:09.794 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.794 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.794 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.794 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:47:09.794 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:47:09.794 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:47:09.794 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 67767 2024-09-13T14:47:09.794 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.795 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.795 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.795 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.795 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201338+0000", 2024-09-13T14:47:09.795 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.795 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.795 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.795 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.795 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201338+0000", 2024-09-13T14:47:09.795 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.795 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.796 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.796 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.796 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201339+0000", 2024-09-13T14:47:09.796 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.606e-06 2024-09-13T14:47:09.796 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.796 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.796 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.796 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201373+0000", 2024-09-13T14:47:09.796 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.3809000000000001e-05 2024-09-13T14:47:09.796 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.796 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.796 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.797 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201385+0000", 2024-09-13T14:47:09.797 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1912e-05 2024-09-13T14:47:09.797 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.797 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.797 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.797 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201393+0000", 2024-09-13T14:47:09.797 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.8800000000000008e-06 2024-09-13T14:47:09.797 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.797 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.797 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.797 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201458+0000", 2024-09-13T14:47:09.798 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5153000000000005e-05 2024-09-13T14:47:09.798 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.798 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.798 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.798 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201602+0000", 2024-09-13T14:47:09.798 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00014437099999999999 2024-09-13T14:47:09.798 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.798 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.798 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:09.798 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201729+0000", 2024-09-13T14:47:09.798 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012680699999999999 2024-09-13T14:47:09.798 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.798 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.799 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:47:09.799 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.270741+0000", 2024-09-13T14:47:09.799 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.069011822 2024-09-13T14:47:09.799 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.799 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.799 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.799 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.270796+0000", 2024-09-13T14:47:09.799 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4780000000000001e-05 2024-09-13T14:47:09.799 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.799 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.799 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.800 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.326561+0000", 2024-09-13T14:47:09.800 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.055765796999999999 2024-09-13T14:47:09.800 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.800 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.800 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.800 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.326860+0000", 2024-09-13T14:47:09.800 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00029859599999999999 2024-09-13T14:47:09.800 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.800 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.801 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.801 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.326901+0000", 2024-09-13T14:47:09.801 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1349e-05 2024-09-13T14:47:09.801 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.801 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.801 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.801 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.801 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.801 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:67768 5.b 5:d396125c:::benchmark_data_smithi096_138841_object67767:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:47:09.802 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.201416+0000", 2024-09-13T14:47:09.802 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 404.58657753900002, 2024-09-13T14:47:09.802 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.125687204, 2024-09-13T14:47:09.802 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.802 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.802 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.802 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:47:09.802 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:47:09.803 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:47:09.803 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 67768 2024-09-13T14:47:09.803 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.803 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.803 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.803 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.803 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201416+0000", 2024-09-13T14:47:09.803 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.803 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.803 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.803 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.803 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201416+0000", 2024-09-13T14:47:09.803 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.803 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.803 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.804 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.804 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201417+0000", 2024-09-13T14:47:09.804 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3200000000000005e-07 2024-09-13T14:47:09.804 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.804 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.804 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.804 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201436+0000", 2024-09-13T14:47:09.804 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8709e-05 2024-09-13T14:47:09.804 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.804 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.804 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.804 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201443+0000", 2024-09-13T14:47:09.804 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.6790000000000005e-06 2024-09-13T14:47:09.804 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.804 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.805 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.805 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201449+0000", 2024-09-13T14:47:09.805 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0379999999999996e-06 2024-09-13T14:47:09.805 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.805 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.805 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.805 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.204867+0000", 2024-09-13T14:47:09.805 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.003418019 2024-09-13T14:47:09.805 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.805 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.805 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.805 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.204977+0000", 2024-09-13T14:47:09.805 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010982899999999999 2024-09-13T14:47:09.805 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.806 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.806 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:09.806 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.205098+0000", 2024-09-13T14:47:09.806 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012060799999999999 2024-09-13T14:47:09.806 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.806 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.806 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:47:09.806 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.270753+0000", 2024-09-13T14:47:09.806 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.065654668999999999 2024-09-13T14:47:09.806 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.806 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.806 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.806 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.270919+0000", 2024-09-13T14:47:09.806 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000166317 2024-09-13T14:47:09.807 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.807 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.807 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.807 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.326942+0000", 2024-09-13T14:47:09.807 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.056023363999999999 2024-09-13T14:47:09.807 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.807 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.807 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.807 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.327066+0000", 2024-09-13T14:47:09.807 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012347899999999999 2024-09-13T14:47:09.807 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.807 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.807 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.807 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.327104+0000", 2024-09-13T14:47:09.807 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7861000000000002e-05 2024-09-13T14:47:09.808 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.808 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.808 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.808 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.808 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.808 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:67769 5.b 5:db42c8a7:::benchmark_data_smithi096_138841_object67768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:47:09.808 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.201459+0000", 2024-09-13T14:47:09.808 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 404.58653506000002, 2024-09-13T14:47:09.808 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.12606088500000001, 2024-09-13T14:47:09.808 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.808 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.808 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.809 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:47:09.809 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:47:09.809 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:47:09.809 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 67769 2024-09-13T14:47:09.809 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.809 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.809 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.809 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.809 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201459+0000", 2024-09-13T14:47:09.809 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.809 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.809 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.809 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.810 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201459+0000", 2024-09-13T14:47:09.810 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.810 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.810 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.810 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.810 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201459+0000", 2024-09-13T14:47:09.810 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9299999999999998e-07 2024-09-13T14:47:09.810 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.810 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.810 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.810 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201505+0000", 2024-09-13T14:47:09.810 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5206000000000001e-05 2024-09-13T14:47:09.810 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.811 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.811 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.811 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201511+0000", 2024-09-13T14:47:09.811 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5209999999999999e-06 2024-09-13T14:47:09.811 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.811 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.811 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.811 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.201516+0000", 2024-09-13T14:47:09.811 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2870000000000004e-06 2024-09-13T14:47:09.811 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.811 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.811 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.811 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.207154+0000", 2024-09-13T14:47:09.811 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0056371290000000003 2024-09-13T14:47:09.812 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.812 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.812 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.812 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.207227+0000", 2024-09-13T14:47:09.812 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2915000000000002e-05 2024-09-13T14:47:09.812 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.812 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.812 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:09.812 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.207300+0000", 2024-09-13T14:47:09.812 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3622999999999996e-05 2024-09-13T14:47:09.812 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.812 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.812 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:47:09.813 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.270758+0000", 2024-09-13T14:47:09.813 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.063458328999999994 2024-09-13T14:47:09.813 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.813 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.813 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.813 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.270959+0000", 2024-09-13T14:47:09.813 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000200822 2024-09-13T14:47:09.813 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.813 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.813 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.813 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.327247+0000", 2024-09-13T14:47:09.813 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.056287225000000003 2024-09-13T14:47:09.813 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.813 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.813 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.814 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.327479+0000", 2024-09-13T14:47:09.814 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00023268599999999999 2024-09-13T14:47:09.814 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.814 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.814 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.814 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.327520+0000", 2024-09-13T14:47:09.814 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0549000000000001e-05 2024-09-13T14:47:09.814 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.814 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.814 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.814 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.814 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.814 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:68071 5.b 5:dff79d32:::benchmark_data_smithi096_138841_object68070:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:47:09.814 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.511726+0000", 2024-09-13T14:47:09.814 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 404.27626775599998, 2024-09-13T14:47:09.815 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.17123097500000001, 2024-09-13T14:47:09.815 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.815 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.815 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.815 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:47:09.815 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:47:09.815 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:47:09.815 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 68071 2024-09-13T14:47:09.815 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.815 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.815 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.815 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.815 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.511726+0000", 2024-09-13T14:47:09.815 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.815 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.816 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.816 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.816 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.511726+0000", 2024-09-13T14:47:09.816 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.816 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.816 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.816 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.816 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.511728+0000", 2024-09-13T14:47:09.816 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.345e-06 2024-09-13T14:47:09.816 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.816 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.816 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.816 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.511748+0000", 2024-09-13T14:47:09.816 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0415e-05 2024-09-13T14:47:09.817 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.817 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.817 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.817 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.511760+0000", 2024-09-13T14:47:09.817 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1895999999999999e-05 2024-09-13T14:47:09.817 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.817 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.817 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.817 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.511767+0000", 2024-09-13T14:47:09.817 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8249999999999999e-06 2024-09-13T14:47:09.817 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.817 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.817 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.817 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.511821+0000", 2024-09-13T14:47:09.818 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3983000000000001e-05 2024-09-13T14:47:09.818 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.818 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.818 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.818 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.511933+0000", 2024-09-13T14:47:09.818 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011260999999999999 2024-09-13T14:47:09.818 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.818 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.818 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:09.818 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.512054+0000", 2024-09-13T14:47:09.818 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000120714 2024-09-13T14:47:09.818 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.818 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.818 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:47:09.818 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.514766+0000", 2024-09-13T14:47:09.819 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027118480000000002 2024-09-13T14:47:09.819 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.819 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.819 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.819 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519138+0000", 2024-09-13T14:47:09.819 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0043726920000000001 2024-09-13T14:47:09.819 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.819 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.819 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.819 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.682860+0000", 2024-09-13T14:47:09.819 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16372144 2024-09-13T14:47:09.819 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.819 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.819 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.820 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.682914+0000", 2024-09-13T14:47:09.820 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4413000000000002e-05 2024-09-13T14:47:09.820 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.820 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.820 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.820 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.682957+0000", 2024-09-13T14:47:09.820 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2793999999999997e-05 2024-09-13T14:47:09.820 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.820 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.820 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.820 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.820 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.820 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:68069 5.4 e119/107 5:20652590:::benchmark_data_smithi096_138841_object68068:head v 119'4263, mlcod=119'4260)", 2024-09-13T14:47:09.821 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.514616+0000", 2024-09-13T14:47:09.821 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 404.273378369, 2024-09-13T14:47:09.821 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16298720899999999, 2024-09-13T14:47:09.821 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.821 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.821 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.821 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.821 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.821 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.821 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.514616+0000", 2024-09-13T14:47:09.821 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.821 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.821 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.821 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.822 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.514616+0000", 2024-09-13T14:47:09.822 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.822 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.822 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.822 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.822 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.514616+0000", 2024-09-13T14:47:09.822 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9999999999999997e-07 2024-09-13T14:47:09.822 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.822 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.822 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.822 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:09.823 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:47:09.824 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.514649+0000", 2024-09-13T14:47:09.824 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.2880999999999999e-05 2024-09-13T14:47:09.824 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.824 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.824 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.824 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.514658+0000", 2024-09-13T14:47:09.825 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3559999999999999e-06 2024-09-13T14:47:09.825 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.825 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.825 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.825 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.514664+0000", 2024-09-13T14:47:09.825 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7830000000000004e-06 2024-09-13T14:47:09.825 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.825 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.825 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.825 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.516281+0000", 2024-09-13T14:47:09.825 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001617077 2024-09-13T14:47:09.825 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.825 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.825 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.826 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.516310+0000", 2024-09-13T14:47:09.826 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.9014999999999999e-05 2024-09-13T14:47:09.826 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.826 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.826 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.826 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677559+0000", 2024-09-13T14:47:09.826 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16124888300000001 2024-09-13T14:47:09.826 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.826 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.826 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.826 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677603+0000", 2024-09-13T14:47:09.826 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3614e-05 2024-09-13T14:47:09.826 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.827 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.827 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.827 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.827 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.827 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:68078 5.8 5:10ccc917:::benchmark_data_smithi096_138841_object68077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:47:09.827 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.515531+0000", 2024-09-13T14:47:09.827 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 404.27246312099999, 2024-09-13T14:47:09.827 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.163309593, 2024-09-13T14:47:09.827 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.827 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.827 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.827 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:47:09.827 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:47:09.827 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:47:09.828 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 68078 2024-09-13T14:47:09.828 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.828 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.828 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.828 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.828 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.515531+0000", 2024-09-13T14:47:09.828 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.828 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.828 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.828 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.828 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.515531+0000", 2024-09-13T14:47:09.828 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.829 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.829 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.829 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.829 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.515533+0000", 2024-09-13T14:47:09.829 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7600000000000001e-06 2024-09-13T14:47:09.829 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.829 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.829 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.829 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.515564+0000", 2024-09-13T14:47:09.829 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.1671999999999998e-05 2024-09-13T14:47:09.829 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.829 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.829 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.830 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.515576+0000", 2024-09-13T14:47:09.830 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1832e-05 2024-09-13T14:47:09.830 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.830 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.830 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.830 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.515583+0000", 2024-09-13T14:47:09.830 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1040000000000001e-06 2024-09-13T14:47:09.830 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.830 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.830 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.830 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.516893+0000", 2024-09-13T14:47:09.830 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001309424 2024-09-13T14:47:09.830 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.830 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.831 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.831 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517075+0000", 2024-09-13T14:47:09.831 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00018258900000000001 2024-09-13T14:47:09.831 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.831 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.831 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:47:09.831 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.517262+0000", 2024-09-13T14:47:09.831 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00018652400000000001 2024-09-13T14:47:09.831 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.831 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.831 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:47:09.831 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519557+0000", 2024-09-13T14:47:09.832 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.002295268 2024-09-13T14:47:09.832 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.832 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.832 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.832 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677726+0000", 2024-09-13T14:47:09.832 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.158168592 2024-09-13T14:47:09.832 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.832 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.832 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.832 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.678409+0000", 2024-09-13T14:47:09.832 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00068320499999999999 2024-09-13T14:47:09.832 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.832 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.833 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.833 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.678795+0000", 2024-09-13T14:47:09.833 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00038649300000000001 2024-09-13T14:47:09.833 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.833 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.833 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.833 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.678840+0000", 2024-09-13T14:47:09.833 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5130000000000003e-05 2024-09-13T14:47:09.833 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.833 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.833 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.833 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.833 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.834 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:68077 5.4 e119/107 5:2c136659:::benchmark_data_smithi096_138841_object68076:head v 119'4264, mlcod=119'4260)", 2024-09-13T14:47:09.834 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.516828+0000", 2024-09-13T14:47:09.834 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 404.271166139, 2024-09-13T14:47:09.834 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16082680999999999, 2024-09-13T14:47:09.834 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.834 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.834 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.834 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.834 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.834 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.834 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.516828+0000", 2024-09-13T14:47:09.834 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.834 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.834 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.835 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.835 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.516828+0000", 2024-09-13T14:47:09.835 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.835 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.835 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.835 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.835 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.516828+0000", 2024-09-13T14:47:09.835 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2399999999999998e-07 2024-09-13T14:47:09.835 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.835 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.835 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.835 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.516855+0000", 2024-09-13T14:47:09.835 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7008999999999998e-05 2024-09-13T14:47:09.835 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.835 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.836 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.836 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.516864+0000", 2024-09-13T14:47:09.836 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6479999999999992e-06 2024-09-13T14:47:09.836 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.836 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.836 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.836 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.516870+0000", 2024-09-13T14:47:09.836 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0739999999999998e-06 2024-09-13T14:47:09.836 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.836 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.836 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.836 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.518385+0000", 2024-09-13T14:47:09.837 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001514907 2024-09-13T14:47:09.837 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.837 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.837 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.837 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.518403+0000", 2024-09-13T14:47:09.837 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7876e-05 2024-09-13T14:47:09.837 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.837 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.837 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.837 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677609+0000", 2024-09-13T14:47:09.837 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15920569000000001 2024-09-13T14:47:09.837 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.838 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.838 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.838 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677655+0000", 2024-09-13T14:47:09.838 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5982000000000001e-05 2024-09-13T14:47:09.838 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.838 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.838 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.838 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.838 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.838 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:68080 5.f e119/107 5:f42639ae:::benchmark_data_smithi096_138841_object68079:head v 119'4189, mlcod=119'4187)", 2024-09-13T14:47:09.838 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.518747+0000", 2024-09-13T14:47:09.838 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 404.26924702899998, 2024-09-13T14:47:09.838 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15896859299999999, 2024-09-13T14:47:09.839 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.839 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.839 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.839 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.839 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.839 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.839 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.518747+0000", 2024-09-13T14:47:09.839 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.839 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.839 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.839 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.839 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.518747+0000", 2024-09-13T14:47:09.839 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.840 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.840 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.840 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.840 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.518747+0000", 2024-09-13T14:47:09.840 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.9400000000000001e-07 2024-09-13T14:47:09.840 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.840 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.840 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.840 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.518778+0000", 2024-09-13T14:47:09.840 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.0676999999999998e-05 2024-09-13T14:47:09.840 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.840 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.840 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.840 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.518784+0000", 2024-09-13T14:47:09.840 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.55e-06 2024-09-13T14:47:09.841 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.841 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.841 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.841 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.518789+0000", 2024-09-13T14:47:09.841 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5900000000000001e-06 2024-09-13T14:47:09.841 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.841 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.841 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.841 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.518944+0000", 2024-09-13T14:47:09.841 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00015532600000000001 2024-09-13T14:47:09.841 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.841 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.841 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.841 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.518962+0000", 2024-09-13T14:47:09.842 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7710000000000002e-05 2024-09-13T14:47:09.842 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.842 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.842 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.842 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677683+0000", 2024-09-13T14:47:09.842 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.158720786 2024-09-13T14:47:09.842 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.842 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.842 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.842 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677715+0000", 2024-09-13T14:47:09.842 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.2660000000000002e-05 2024-09-13T14:47:09.842 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.842 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.843 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.843 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.843 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.843 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:68085 5.a 5:53edf856:::benchmark_data_smithi096_138841_object68084:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:47:09.843 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.519112+0000", 2024-09-13T14:47:09.843 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 404.268882093, 2024-09-13T14:47:09.843 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.17612778000000001, 2024-09-13T14:47:09.843 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.843 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.843 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.843 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:47:09.843 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:47:09.843 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:47:09.844 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 68085 2024-09-13T14:47:09.844 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.844 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.844 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.844 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.844 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519112+0000", 2024-09-13T14:47:09.844 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.844 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.844 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.844 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.844 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519112+0000", 2024-09-13T14:47:09.844 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.844 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.844 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.845 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.845 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519113+0000", 2024-09-13T14:47:09.845 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5999999999999999e-06 2024-09-13T14:47:09.845 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.845 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.845 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.845 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519135+0000", 2024-09-13T14:47:09.845 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1617999999999999e-05 2024-09-13T14:47:09.845 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.845 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.845 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.845 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519146+0000", 2024-09-13T14:47:09.845 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.066e-05 2024-09-13T14:47:09.845 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.846 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.846 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.846 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519152+0000", 2024-09-13T14:47:09.846 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4459999999999998e-06 2024-09-13T14:47:09.846 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.846 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.846 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.846 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519573+0000", 2024-09-13T14:47:09.846 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00042057100000000001 2024-09-13T14:47:09.846 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.846 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.846 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.846 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519734+0000", 2024-09-13T14:47:09.846 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000161767 2024-09-13T14:47:09.847 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.847 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.847 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:47:09.847 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677598+0000", 2024-09-13T14:47:09.847 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.157863366 2024-09-13T14:47:09.847 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.847 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.847 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.847 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.687392+0000", 2024-09-13T14:47:09.847 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0097941620000000004 2024-09-13T14:47:09.847 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.847 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.847 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:47:09.848 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.687404+0000", 2024-09-13T14:47:09.848 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2133000000000001e-05 2024-09-13T14:47:09.848 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.848 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.848 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.848 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.687439+0000", 2024-09-13T14:47:09.848 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4999999999999997e-05 2024-09-13T14:47:09.848 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.848 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.848 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.848 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.687595+0000", 2024-09-13T14:47:09.848 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000155413 2024-09-13T14:47:09.848 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.848 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.849 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.849 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.695240+0000", 2024-09-13T14:47:09.849 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0076450440000000001 2024-09-13T14:47:09.849 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.849 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.849 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.849 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.849 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.849 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:68088 5.b 5:d6511789:::benchmark_data_smithi096_138841_object68087:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:47:09.849 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.519736+0000", 2024-09-13T14:47:09.849 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 404.26825811600003, 2024-09-13T14:47:09.849 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.164806018, 2024-09-13T14:47:09.850 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.850 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.850 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.850 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:47:09.850 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:47:09.850 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:47:09.850 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 68088 2024-09-13T14:47:09.850 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.850 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.850 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.850 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.850 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519736+0000", 2024-09-13T14:47:09.850 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.850 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.851 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.851 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.851 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519736+0000", 2024-09-13T14:47:09.851 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.851 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.851 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.851 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.851 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519737+0000", 2024-09-13T14:47:09.851 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9999999999999996e-07 2024-09-13T14:47:09.851 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.851 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.851 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.852 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519757+0000", 2024-09-13T14:47:09.852 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9879000000000001e-05 2024-09-13T14:47:09.852 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.852 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.852 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.852 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519768+0000", 2024-09-13T14:47:09.852 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1596e-05 2024-09-13T14:47:09.852 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.852 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.852 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.852 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.519775+0000", 2024-09-13T14:47:09.852 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9090000000000003e-06 2024-09-13T14:47:09.852 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.852 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.853 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.853 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677574+0000", 2024-09-13T14:47:09.853 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15779865800000001 2024-09-13T14:47:09.853 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.853 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.853 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.853 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677733+0000", 2024-09-13T14:47:09.853 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000159431 2024-09-13T14:47:09.853 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.853 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.853 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:09.853 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677872+0000", 2024-09-13T14:47:09.854 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013923200000000001 2024-09-13T14:47:09.854 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.854 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.854 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.854 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.682998+0000", 2024-09-13T14:47:09.854 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0051260359999999996 2024-09-13T14:47:09.854 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.854 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.854 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.854 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.683664+0000", 2024-09-13T14:47:09.854 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00066554400000000001 2024-09-13T14:47:09.854 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.854 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.854 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:47:09.855 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684452+0000", 2024-09-13T14:47:09.855 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00078841000000000002 2024-09-13T14:47:09.855 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.855 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.855 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.855 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684497+0000", 2024-09-13T14:47:09.855 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4249999999999998e-05 2024-09-13T14:47:09.855 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.855 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.855 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.855 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684542+0000", 2024-09-13T14:47:09.855 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5173000000000003e-05 2024-09-13T14:47:09.856 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.856 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.856 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.856 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.856 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.856 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.35049.0:68078 5.8 e119/107 ondisk, result = 0)", 2024-09-13T14:47:09.856 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.520878+0000", 2024-09-13T14:47:09.856 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 404.26711620600003, 2024-09-13T14:47:09.856 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15685665200000001, 2024-09-13T14:47:09.856 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.856 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.856 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:47:09.856 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.856 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.857 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.857 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520878+0000", 2024-09-13T14:47:09.857 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.857 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.857 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.857 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.857 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520878+0000", 2024-09-13T14:47:09.857 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.857 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.857 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.857 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.857 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520878+0000", 2024-09-13T14:47:09.857 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.4499999999999996e-07 2024-09-13T14:47:09.857 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.858 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.858 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.858 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.520883+0000", 2024-09-13T14:47:09.858 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.335e-06 2024-09-13T14:47:09.858 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.858 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.858 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.858 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677490+0000", 2024-09-13T14:47:09.858 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15660695499999999 2024-09-13T14:47:09.858 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.858 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.858 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.858 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677503+0000", 2024-09-13T14:47:09.859 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3674e-05 2024-09-13T14:47:09.859 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.859 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.859 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.859 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677654+0000", 2024-09-13T14:47:09.859 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000150648 2024-09-13T14:47:09.859 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.859 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.859 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.859 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677717+0000", 2024-09-13T14:47:09.859 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2476000000000003e-05 2024-09-13T14:47:09.859 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.859 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.860 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.860 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677734+0000", 2024-09-13T14:47:09.860 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7819e-05 2024-09-13T14:47:09.860 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.860 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.860 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.860 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.860 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.860 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:68090 5.b 5:d3a7cfec:::benchmark_data_smithi096_138841_object68089:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:47:09.860 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.521040+0000", 2024-09-13T14:47:09.860 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 404.26695344900003, 2024-09-13T14:47:09.860 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16556853699999999, 2024-09-13T14:47:09.860 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.861 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.861 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.861 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:47:09.861 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:47:09.861 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:47:09.861 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 68090 2024-09-13T14:47:09.861 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.861 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.861 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.861 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.861 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.521040+0000", 2024-09-13T14:47:09.861 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.861 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.861 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.862 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.862 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.521040+0000", 2024-09-13T14:47:09.862 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.862 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.862 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.862 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.862 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.521042+0000", 2024-09-13T14:47:09.862 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3120000000000001e-06 2024-09-13T14:47:09.862 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.862 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.862 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.862 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.521064+0000", 2024-09-13T14:47:09.862 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1946e-05 2024-09-13T14:47:09.862 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.862 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.863 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.863 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677499+0000", 2024-09-13T14:47:09.863 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15643504 2024-09-13T14:47:09.863 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.863 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.863 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.863 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677512+0000", 2024-09-13T14:47:09.863 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3271999999999999e-05 2024-09-13T14:47:09.863 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.863 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.863 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.863 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.680314+0000", 2024-09-13T14:47:09.863 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0028019579999999998 2024-09-13T14:47:09.863 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.863 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.864 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.864 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.680427+0000", 2024-09-13T14:47:09.864 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000113318 2024-09-13T14:47:09.864 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.864 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.864 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:09.864 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.680558+0000", 2024-09-13T14:47:09.864 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000130646 2024-09-13T14:47:09.864 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.864 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.864 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.864 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.683603+0000", 2024-09-13T14:47:09.864 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0030445839999999999 2024-09-13T14:47:09.865 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.865 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.865 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:47:09.865 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.684555+0000", 2024-09-13T14:47:09.865 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00095244199999999996 2024-09-13T14:47:09.865 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.865 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.865 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.865 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.686514+0000", 2024-09-13T14:47:09.865 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0019590190000000002 2024-09-13T14:47:09.865 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.865 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.865 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.866 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.686571+0000", 2024-09-13T14:47:09.866 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7058000000000002e-05 2024-09-13T14:47:09.866 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.866 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.866 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.866 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.686609+0000", 2024-09-13T14:47:09.866 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7941999999999997e-05 2024-09-13T14:47:09.866 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.866 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.866 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.866 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.866 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.866 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:68083 5.f e119/107 5:f7da7ef6:::benchmark_data_smithi096_138841_object68082:head v 119'4190, mlcod=119'4187)", 2024-09-13T14:47:09.867 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:40:25.521090+0000", 2024-09-13T14:47:09.867 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 404.26690393600001, 2024-09-13T14:47:09.867 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.16448037300000001, 2024-09-13T14:47:09.867 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.867 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.867 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.868 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.868 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.868 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.868 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.521090+0000", 2024-09-13T14:47:09.868 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.868 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.868 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.869 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.869 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.521090+0000", 2024-09-13T14:47:09.869 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.869 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.869 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.869 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.869 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.521090+0000", 2024-09-13T14:47:09.869 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6400000000000003e-07 2024-09-13T14:47:09.869 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.869 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.869 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.869 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.521123+0000", 2024-09-13T14:47:09.869 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.2444999999999999e-05 2024-09-13T14:47:09.870 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.870 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.870 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.870 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677913+0000", 2024-09-13T14:47:09.870 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15678971 2024-09-13T14:47:09.870 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.870 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.870 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.870 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677918+0000", 2024-09-13T14:47:09.870 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7860000000000002e-06 2024-09-13T14:47:09.870 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.870 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.870 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.871 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677962+0000", 2024-09-13T14:47:09.871 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3720000000000002e-05 2024-09-13T14:47:09.871 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.871 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.871 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.871 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.677982+0000", 2024-09-13T14:47:09.871 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0231999999999999e-05 2024-09-13T14:47:09.871 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.871 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.871 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.871 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.685505+0000", 2024-09-13T14:47:09.871 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0075223540000000002 2024-09-13T14:47:09.871 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.872 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.872 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.872 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:40:25.685570+0000", 2024-09-13T14:47:09.872 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5661999999999996e-05 2024-09-13T14:47:09.872 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.872 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.872 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.872 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.872 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.872 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:155343 5.8 5:1b6701d1:::benchmark_data_smithi096_138841_object25147:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:47:09.872 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.989548+0000", 2024-09-13T14:47:09.872 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 321.79844636000001, 2024-09-13T14:47:09.872 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.140049653, 2024-09-13T14:47:09.873 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.873 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.873 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.873 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:47:09.873 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:47:09.873 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:47:09.873 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 155343 2024-09-13T14:47:09.873 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.873 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.873 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.873 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.873 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989548+0000", 2024-09-13T14:47:09.873 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989548+0000", 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989554+0000", 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9460000000000003e-06 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989558+0000", 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5560000000000001e-06 2024-09-13T14:47:09.874 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.875 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.875 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.875 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989568+0000", 2024-09-13T14:47:09.875 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.6500000000000008e-06 2024-09-13T14:47:09.875 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.875 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.875 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.875 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989574+0000", 2024-09-13T14:47:09.875 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2380000000000002e-06 2024-09-13T14:47:09.875 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.875 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.875 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.875 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989619+0000", 2024-09-13T14:47:09.875 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5169000000000001e-05 2024-09-13T14:47:09.875 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.876 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.876 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.876 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989725+0000", 2024-09-13T14:47:09.876 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010623 2024-09-13T14:47:09.876 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.876 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.876 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:47:09.876 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.989829+0000", 2024-09-13T14:47:09.876 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000103185 2024-09-13T14:47:09.876 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.876 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.876 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.876 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.995687+0000", 2024-09-13T14:47:09.876 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0058581939999999997 2024-09-13T14:47:09.876 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.877 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.877 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:47:09.877 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.995764+0000", 2024-09-13T14:47:09.877 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.7142000000000001e-05 2024-09-13T14:47:09.877 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.877 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.877 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.877 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129398+0000", 2024-09-13T14:47:09.877 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.13363418899999999 2024-09-13T14:47:09.877 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.877 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.877 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.877 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129554+0000", 2024-09-13T14:47:09.877 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000155864 2024-09-13T14:47:09.878 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.878 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.878 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.878 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129597+0000", 2024-09-13T14:47:09.878 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3290000000000001e-05 2024-09-13T14:47:09.878 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.878 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.878 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.878 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.878 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.878 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:155341 5.f e130/107 5:fa928cd7:::benchmark_data_smithi096_138841_object25145:head v 130'9712, mlcod=130'9709)", 2024-09-13T14:47:09.878 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.990428+0000", 2024-09-13T14:47:09.878 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 321.79756635500001, 2024-09-13T14:47:09.878 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.13086341300000001, 2024-09-13T14:47:09.878 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.879 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.879 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.879 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.879 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.879 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.879 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.990428+0000", 2024-09-13T14:47:09.879 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.879 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.879 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.879 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.879 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.990428+0000", 2024-09-13T14:47:09.879 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.879 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.879 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.879 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.880 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.990428+0000", 2024-09-13T14:47:09.880 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9999999999999997e-07 2024-09-13T14:47:09.880 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.880 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.880 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.880 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.990433+0000", 2024-09-13T14:47:09.880 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6179999999999997e-06 2024-09-13T14:47:09.880 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.880 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.880 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.880 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.990442+0000", 2024-09-13T14:47:09.880 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8089999999999999e-06 2024-09-13T14:47:09.880 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.880 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.881 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.881 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.990453+0000", 2024-09-13T14:47:09.881 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0893999999999999e-05 2024-09-13T14:47:09.881 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.881 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.881 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.881 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.990997+0000", 2024-09-13T14:47:09.881 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00054462499999999997 2024-09-13T14:47:09.881 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.881 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.881 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.881 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.991014+0000", 2024-09-13T14:47:09.881 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7082999999999999e-05 2024-09-13T14:47:09.882 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.882 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.882 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.882 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.121242+0000", 2024-09-13T14:47:09.882 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.13022734799999999 2024-09-13T14:47:09.882 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.882 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.882 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.882 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.121291+0000", 2024-09-13T14:47:09.882 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9336000000000003e-05 2024-09-13T14:47:09.882 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.882 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.882 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.882 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.883 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.883 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:155345 5.d 5:b3daeffa:::benchmark_data_smithi096_138841_object25149:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:47:09.883 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.991594+0000", 2024-09-13T14:47:09.883 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 321.79639945399998, 2024-09-13T14:47:09.883 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.13238444599999999, 2024-09-13T14:47:09.883 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.883 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.883 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.883 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:47:09.883 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:47:09.883 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:47:09.883 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 155345 2024-09-13T14:47:09.883 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.883 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.884 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.884 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.884 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.991594+0000", 2024-09-13T14:47:09.884 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.884 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.884 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.884 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.884 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.991594+0000", 2024-09-13T14:47:09.884 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.884 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.884 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.884 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.884 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.991596+0000", 2024-09-13T14:47:09.885 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.291e-06 2024-09-13T14:47:09.885 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.885 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.885 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.885 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.991600+0000", 2024-09-13T14:47:09.885 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1180000000000002e-06 2024-09-13T14:47:09.885 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.885 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.885 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.885 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.991611+0000", 2024-09-13T14:47:09.885 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0992000000000001e-05 2024-09-13T14:47:09.885 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.885 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.886 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.886 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.991617+0000", 2024-09-13T14:47:09.886 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5039999999999999e-06 2024-09-13T14:47:09.886 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.886 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.886 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.886 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.991679+0000", 2024-09-13T14:47:09.886 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1624000000000005e-05 2024-09-13T14:47:09.886 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.886 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.886 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.886 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.995088+0000", 2024-09-13T14:47:09.886 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0034087509999999998 2024-09-13T14:47:09.887 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.887 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.887 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:47:09.887 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.995200+0000", 2024-09-13T14:47:09.887 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000112123 2024-09-13T14:47:09.887 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.887 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.887 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.887 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120038+0000", 2024-09-13T14:47:09.887 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.124838485 2024-09-13T14:47:09.887 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.887 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.888 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.888 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120071+0000", 2024-09-13T14:47:09.888 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.2869e-05 2024-09-13T14:47:09.888 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.888 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.888 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:47:09.888 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.123873+0000", 2024-09-13T14:47:09.888 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0038015810000000001 2024-09-13T14:47:09.888 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.888 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.888 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.888 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.123932+0000", 2024-09-13T14:47:09.888 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9286999999999997e-05 2024-09-13T14:47:09.888 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.889 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.889 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.889 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.123979+0000", 2024-09-13T14:47:09.889 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6820999999999998e-05 2024-09-13T14:47:09.889 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.889 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.889 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.889 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.889 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.889 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35049.0:155347 5.f e130/107 5:f61a39fc:::benchmark_data_smithi096_138841_object25151:head v 130'9713, mlcod=130'9709)", 2024-09-13T14:47:09.889 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.995683+0000", 2024-09-13T14:47:09.889 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 321.79231066800003, 2024-09-13T14:47:09.889 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.12563966600000001, 2024-09-13T14:47:09.890 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.890 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.890 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.890 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.890 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.890 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.890 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.995683+0000", 2024-09-13T14:47:09.890 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.890 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.890 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.890 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.890 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.995683+0000", 2024-09-13T14:47:09.890 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.890 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.891 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.891 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.891 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.995684+0000", 2024-09-13T14:47:09.891 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1800000000000005e-07 2024-09-13T14:47:09.891 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.891 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.891 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.891 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.995689+0000", 2024-09-13T14:47:09.891 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.203e-06 2024-09-13T14:47:09.891 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.891 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.891 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.891 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.995700+0000", 2024-09-13T14:47:09.891 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0998e-05 2024-09-13T14:47:09.892 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.892 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.892 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.892 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.995706+0000", 2024-09-13T14:47:09.892 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1480000000000003e-06 2024-09-13T14:47:09.892 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.892 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.892 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.892 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.996376+0000", 2024-09-13T14:47:09.892 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00066941400000000001 2024-09-13T14:47:09.892 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.892 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.892 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.892 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.996393+0000", 2024-09-13T14:47:09.893 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6889e-05 2024-09-13T14:47:09.893 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.893 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.893 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.893 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.121297+0000", 2024-09-13T14:47:09.893 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.124904127 2024-09-13T14:47:09.893 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.893 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.893 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.893 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.121323+0000", 2024-09-13T14:47:09.893 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6169e-05 2024-09-13T14:47:09.893 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.894 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.894 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.894 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.894 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.894 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35049.0:155349 5.8 5:19cdefb6:::benchmark_data_smithi096_138841_object25153:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-13T14:47:09.894 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.996442+0000", 2024-09-13T14:47:09.894 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 321.79155210499999, 2024-09-13T14:47:09.894 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.14170163099999999, 2024-09-13T14:47:09.894 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.894 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.894 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:09.894 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:47:09.894 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35049", 2024-09-13T14:47:09.894 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:47:09.895 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 155349 2024-09-13T14:47:09.895 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.895 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.895 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.895 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.895 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.996442+0000", 2024-09-13T14:47:09.895 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.895 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.895 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.896 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.896 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.996442+0000", 2024-09-13T14:47:09.896 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.896 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.896 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.896 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.896 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.996443+0000", 2024-09-13T14:47:09.896 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0410000000000001e-06 2024-09-13T14:47:09.896 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.896 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.896 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.896 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.996447+0000", 2024-09-13T14:47:09.896 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1180000000000002e-06 2024-09-13T14:47:09.897 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.897 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.897 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.897 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.996454+0000", 2024-09-13T14:47:09.897 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8870000000000003e-06 2024-09-13T14:47:09.897 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.897 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.897 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.897 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.996459+0000", 2024-09-13T14:47:09.897 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.1460000000000004e-06 2024-09-13T14:47:09.897 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.897 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.898 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.898 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.119738+0000", 2024-09-13T14:47:09.898 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.123279177 2024-09-13T14:47:09.898 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.898 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.898 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.898 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.119879+0000", 2024-09-13T14:47:09.898 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00014079899999999999 2024-09-13T14:47:09.898 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.898 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.898 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:47:09.898 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120013+0000", 2024-09-13T14:47:09.898 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013419199999999999 2024-09-13T14:47:09.899 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.899 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.899 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.899 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129638+0000", 2024-09-13T14:47:09.899 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0096251080000000003 2024-09-13T14:47:09.899 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.899 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.899 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:47:09.899 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.129878+0000", 2024-09-13T14:47:09.899 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000239392 2024-09-13T14:47:09.899 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.899 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.899 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:09.900 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.138043+0000", 2024-09-13T14:47:09.900 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0081652470000000005 2024-09-13T14:47:09.900 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.900 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.900 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:47:09.900 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.138099+0000", 2024-09-13T14:47:09.900 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6267000000000002e-05 2024-09-13T14:47:09.900 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.900 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.900 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.900 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.138143+0000", 2024-09-13T14:47:09.900 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4257e-05 2024-09-13T14:47:09.900 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.900 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.901 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.901 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.901 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.901 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.35049.0:155345 5.d e130/107 ondisk, result = 0)", 2024-09-13T14:47:09.901 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:41:47.999462+0000", 2024-09-13T14:47:09.901 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 321.78853191299999, 2024-09-13T14:47:09.901 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.120583053, 2024-09-13T14:47:09.901 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.901 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.901 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:47:09.901 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.901 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.901 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.901 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.999462+0000", 2024-09-13T14:47:09.902 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.902 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.902 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.902 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.902 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.999462+0000", 2024-09-13T14:47:09.902 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.902 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.902 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.902 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.902 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.999463+0000", 2024-09-13T14:47:09.902 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4699999999999999e-06 2024-09-13T14:47:09.902 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.902 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.902 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.903 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:47.999479+0000", 2024-09-13T14:47:09.903 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5226e-05 2024-09-13T14:47:09.903 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.903 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.903 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.903 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.119579+0000", 2024-09-13T14:47:09.903 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.12010038100000001 2024-09-13T14:47:09.903 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.903 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.903 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.903 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.119596+0000", 2024-09-13T14:47:09.903 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7065e-05 2024-09-13T14:47:09.903 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.903 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.904 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.904 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120025+0000", 2024-09-13T14:47:09.904 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00042851400000000001 2024-09-13T14:47:09.904 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.904 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.904 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:47:09.904 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120033+0000", 2024-09-13T14:47:09.904 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9540000000000004e-06 2024-09-13T14:47:09.904 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.904 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.904 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.904 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:41:48.120045+0000", 2024-09-13T14:47:09.904 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2442999999999999e-05 2024-09-13T14:47:09.904 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.905 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.905 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.905 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.905 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.905 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.45167.0:156614 6.1 e173/169)", 2024-09-13T14:47:09.905 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:32.023488+0000", 2024-09-13T14:47:09.905 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 37.764506134000001, 2024-09-13T14:47:09.905 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.262835561, 2024-09-13T14:47:09.905 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:47:09.905 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:47:09.905 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "reached pg", 2024-09-13T14:47:09.905 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:47:09.905 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.906 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:47:09.906 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.023488+0000", 2024-09-13T14:47:09.906 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.906 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.906 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.906 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:47:09.906 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.023488+0000", 2024-09-13T14:47:09.906 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:47:09.906 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.906 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.906 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:47:09.906 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.023489+0000", 2024-09-13T14:47:09.906 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.124e-06 2024-09-13T14:47:09.907 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.907 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.907 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:47:09.907 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.023502+0000", 2024-09-13T14:47:09.907 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.345e-05 2024-09-13T14:47:09.907 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.907 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.907 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:47:09.907 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.023514+0000", 2024-09-13T14:47:09.907 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1834999999999999e-05 2024-09-13T14:47:09.907 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.907 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.907 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:47:09.907 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.023522+0000", 2024-09-13T14:47:09.907 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.4510000000000003e-06 2024-09-13T14:47:09.908 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.908 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.908 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:47:09.908 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.286308+0000", 2024-09-13T14:47:09.908 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.26278601800000001 2024-09-13T14:47:09.908 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:47:09.908 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:47:09.908 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:47:09.908 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.286323+0000", 2024-09-13T14:47:09.908 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5682999999999999e-05 2024-09-13T14:47:09.908 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.908 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.908 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.909 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:47:09.909 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:47:09.909 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:47:09.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.931 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:09.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:10.032 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:10.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.092 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:10.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:10.193 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:10.243 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:10.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:10.345 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:10.387 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:10.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:10.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:47:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:47:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:10.488 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:10.530 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:10.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:10.631 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:10.654 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:10.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:10.756 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:10.796 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:10.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:10.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:10 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:47:10.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:10 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:47:10.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:10 smithi096 ceph-mon[98901]: 6.13 scrub starts 2024-09-13T14:47:10.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:10 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:47:10.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:10 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:47:10.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:10 smithi096 ceph-mon[102928]: 6.13 scrub starts 2024-09-13T14:47:10.898 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:10.926 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:10.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:10.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:10 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:47:10.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:10 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:47:10.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:10 smithi204 ceph-mon[87071]: 6.13 scrub starts 2024-09-13T14:47:11.027 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:11.055 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:11.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:11.157 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:11.207 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:11.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:11.309 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:11.336 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:11.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:11.438 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:11.469 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:11.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:11.571 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:11.608 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:11.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:11.710 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:11.734 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:11.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:11.741 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:47:11.779 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:11.796 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:11 smithi096 ceph-mon[98901]: pgmap v561: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 105 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 7168/179406 objects misplaced (3.995%) 2024-09-13T14:47:11.796 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:11 smithi096 ceph-mon[98901]: 6.13 scrub ok 2024-09-13T14:47:11.796 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:11 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:47:11.796 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:11 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:47:11.796 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:11 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:47:11.797 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:11 smithi096 ceph-mon[102928]: pgmap v561: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 105 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 7168/179406 objects misplaced (3.995%) 2024-09-13T14:47:11.797 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:11 smithi096 ceph-mon[102928]: 6.13 scrub ok 2024-09-13T14:47:11.797 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:11 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:47:11.797 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:11 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:47:11.797 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:11 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:47:11.836 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:11.874 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:11.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:11.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:11 smithi204 ceph-mon[87071]: pgmap v561: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 105 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 7168/179406 objects misplaced (3.995%) 2024-09-13T14:47:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:11 smithi204 ceph-mon[87071]: 6.13 scrub ok 2024-09-13T14:47:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:11 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:47:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:11 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:47:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:11 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:47:11.975 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:12.004 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:12.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:12.106 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:12.133 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:12.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:12.235 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:12.283 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:12.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:12.385 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:12.397 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:12.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.413 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:12.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:12.488 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T14:47:12.488 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T14:47:12.488 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T14:47:12.488 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T14:47:12.488 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T14:47:12.488 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T14:47:12.488 INFO:teuthology.orchestra.run.smithi096.stderr:bench [] [] [] [] 2024-09-13T14:47:12.488 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:47:12.489 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T14:47:12.489 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T14:47:12.489 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore allocator dump block 2024-09-13T14:47:12.489 INFO:teuthology.orchestra.run.smithi096.stderr:Error EINVAL: invalid command 2024-09-13T14:47:12.514 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:12.559 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:12.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:12.662 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:12.709 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:12.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:12.811 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:12.847 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:12.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:12.948 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:12.991 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:12.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:13.093 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:13.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.126 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:13.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:13.227 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:13.266 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:13.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:13.368 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:13.402 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:13.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:13.504 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:13.538 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:13.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:13.640 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:13.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.673 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:13.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:13.774 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:13.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.807 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:13.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:13.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:13 smithi096 ceph-mon[98901]: pgmap v562: 109 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 105 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 4946/176160 objects misplaced (2.808%) 2024-09-13T14:47:13.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:13 smithi096 ceph-mon[102928]: pgmap v562: 109 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 105 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 4946/176160 objects misplaced (2.808%) 2024-09-13T14:47:13.909 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:13.941 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:13.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:13.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:13 smithi204 ceph-mon[87071]: pgmap v562: 109 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 105 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 4946/176160 objects misplaced (2.808%) 2024-09-13T14:47:14.042 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:14.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.075 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:14.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:14.176 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:14.212 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:14.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:14.313 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:14.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.336 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:14.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:14.437 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:14.470 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:14.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:14.572 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:14.610 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:14.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:14.712 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:14.732 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:14.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:14.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:14 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-13T14:47:14.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:14 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-13T14:47:14.834 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:14.857 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:14.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:14.958 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:14.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:14 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-13T14:47:14.992 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:14.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:15.094 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:15.127 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:15.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:15.229 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:15.257 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:15.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:15.359 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:15.385 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:15.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:15.487 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:15.510 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:15.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:15.611 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:15.660 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:15.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:15.762 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:15.785 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:15.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:15.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:15 smithi096 ceph-mon[98901]: pgmap v563: 109 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 106 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5504/167229 objects misplaced (3.291%) 2024-09-13T14:47:15.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:15 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-13T14:47:15.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:15 smithi096 ceph-mon[98901]: osdmap e204: 8 total, 8 up, 7 in 2024-09-13T14:47:15.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:15 smithi096 ceph-mon[102928]: pgmap v563: 109 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 106 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5504/167229 objects misplaced (3.291%) 2024-09-13T14:47:15.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:15 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-13T14:47:15.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:15 smithi096 ceph-mon[102928]: osdmap e204: 8 total, 8 up, 7 in 2024-09-13T14:47:15.886 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:15.909 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:15.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:15.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:15 smithi204 ceph-mon[87071]: pgmap v563: 109 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 106 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5504/167229 objects misplaced (3.291%) 2024-09-13T14:47:15.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:15 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-13T14:47:15.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:15 smithi204 ceph-mon[87071]: osdmap e204: 8 total, 8 up, 7 in 2024-09-13T14:47:16.011 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:16.037 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:16.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:16.139 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:16.163 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:16.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:16.264 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:16.291 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:16.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:16.393 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:16.415 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:16.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:16.516 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:16.540 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:16.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:16.641 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:16.670 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:16.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:16.772 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:16.809 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:16.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:16.911 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:16.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.937 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:16.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:16.962 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:16 smithi096 ceph-mon[98901]: osdmap e205: 8 total, 8 up, 7 in 2024-09-13T14:47:16.962 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:16 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-13T14:47:16.962 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:16 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-13T14:47:16.962 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:16 smithi096 ceph-mon[102928]: osdmap e205: 8 total, 8 up, 7 in 2024-09-13T14:47:16.962 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:16 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-13T14:47:16.962 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:16 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-13T14:47:16.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:16 smithi204 ceph-mon[87071]: osdmap e205: 8 total, 8 up, 7 in 2024-09-13T14:47:16.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:16 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-13T14:47:16.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:16 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-13T14:47:17.039 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:17.063 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:17.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:17.165 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:17.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.186 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:17.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:17.287 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:17.313 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:17.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:17.415 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:17.437 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:17.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:17.539 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:17.561 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:17.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:17.663 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:17.694 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:17.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:17.796 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:17.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.831 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:17.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:17.932 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:17.947 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:17 smithi096 ceph-mon[98901]: pgmap v566: 109 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 106 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5504/167229 objects misplaced (3.291%); 5.4 MiB/s, 86 objects/s recovering 2024-09-13T14:47:17.947 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:17 smithi096 ceph-mon[98901]: 6.17 starting backfill to osd.2 from (0'0,0'0] MAX to 203'10349 2024-09-13T14:47:17.947 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:17 smithi096 ceph-mon[98901]: osdmap e206: 8 total, 8 up, 7 in 2024-09-13T14:47:17.947 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:17 smithi096 ceph-mon[98901]: osdmap e207: 8 total, 8 up, 7 in 2024-09-13T14:47:17.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:17 smithi096 ceph-mon[102928]: pgmap v566: 109 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 106 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5504/167229 objects misplaced (3.291%); 5.4 MiB/s, 86 objects/s recovering 2024-09-13T14:47:17.948 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:17 smithi096 ceph-mon[102928]: 6.17 starting backfill to osd.2 from (0'0,0'0] MAX to 203'10349 2024-09-13T14:47:17.948 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:17 smithi096 ceph-mon[102928]: osdmap e206: 8 total, 8 up, 7 in 2024-09-13T14:47:17.948 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:17 smithi096 ceph-mon[102928]: osdmap e207: 8 total, 8 up, 7 in 2024-09-13T14:47:17.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.957 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:17.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:17.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:17 smithi204 ceph-mon[87071]: pgmap v566: 109 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 106 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5504/167229 objects misplaced (3.291%); 5.4 MiB/s, 86 objects/s recovering 2024-09-13T14:47:17.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:17 smithi204 ceph-mon[87071]: 6.17 starting backfill to osd.2 from (0'0,0'0] MAX to 203'10349 2024-09-13T14:47:17.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:17 smithi204 ceph-mon[87071]: osdmap e206: 8 total, 8 up, 7 in 2024-09-13T14:47:17.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:17 smithi204 ceph-mon[87071]: osdmap e207: 8 total, 8 up, 7 in 2024-09-13T14:47:18.058 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:18.084 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:18.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:18.186 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:18.212 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:18.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:18.314 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:18.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:47:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:47:17] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.51.0" 2024-09-13T14:47:18.338 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:18.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:18.440 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:18.453 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:18.464 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:18.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:18.566 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:18.599 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:18.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:18.700 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:18.731 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:18.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:18.833 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:18.861 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:18.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:18.963 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:18.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:18 smithi204 ceph-mon[87071]: 6.14 scrub starts 2024-09-13T14:47:18.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:18 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-13T14:47:18.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:18 smithi204 ceph-mon[87071]: 6.14 scrub ok 2024-09-13T14:47:18.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:18 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-13T14:47:18.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:18 smithi204 ceph-mon[87071]: osdmap e208: 8 total, 8 up, 7 in 2024-09-13T14:47:18.988 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:18.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:18.995 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:18 smithi096 ceph-mon[98901]: 6.14 scrub starts 2024-09-13T14:47:18.995 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:18 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-13T14:47:18.995 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:18 smithi096 ceph-mon[98901]: 6.14 scrub ok 2024-09-13T14:47:18.996 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:18 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-13T14:47:18.996 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:18 smithi096 ceph-mon[98901]: osdmap e208: 8 total, 8 up, 7 in 2024-09-13T14:47:18.996 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:18 smithi096 ceph-mon[102928]: 6.14 scrub starts 2024-09-13T14:47:18.996 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:18 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-13T14:47:18.996 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:18 smithi096 ceph-mon[102928]: 6.14 scrub ok 2024-09-13T14:47:18.996 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:18 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-13T14:47:18.996 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:18 smithi096 ceph-mon[102928]: osdmap e208: 8 total, 8 up, 7 in 2024-09-13T14:47:19.091 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:19.119 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:19.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:19.221 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:19.253 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:19.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:19.355 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:19.379 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:19.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:19.476 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:47:19.477 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:47:19.477 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.2 'trim stale osdmaps' 2024-09-13T14:47:19.481 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:19.545 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:19.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:19.647 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:19.674 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:19.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:19.766 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:19 smithi096 ceph-mon[102928]: pgmap v569: 109 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 105 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 4594/158553 objects misplaced (2.897%); 19 MiB/s, 297 objects/s recovering 2024-09-13T14:47:19.766 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:19 smithi096 ceph-mon[102928]: 6.18 starting backfill to osd.0 from (0'0,0'0] MAX to 205'10676 2024-09-13T14:47:19.766 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:19 smithi096 ceph-mon[102928]: 6.18 starting backfill to osd.1 from (0'0,0'0] MAX to 205'10676 2024-09-13T14:47:19.766 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:19 smithi096 ceph-mon[102928]: osdmap e209: 8 total, 8 up, 7 in 2024-09-13T14:47:19.767 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:19 smithi096 ceph-mon[98901]: pgmap v569: 109 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 105 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 4594/158553 objects misplaced (2.897%); 19 MiB/s, 297 objects/s recovering 2024-09-13T14:47:19.767 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:19 smithi096 ceph-mon[98901]: 6.18 starting backfill to osd.0 from (0'0,0'0] MAX to 205'10676 2024-09-13T14:47:19.767 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:19 smithi096 ceph-mon[98901]: 6.18 starting backfill to osd.1 from (0'0,0'0] MAX to 205'10676 2024-09-13T14:47:19.767 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:19 smithi096 ceph-mon[98901]: osdmap e209: 8 total, 8 up, 7 in 2024-09-13T14:47:19.776 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:19.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.804 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:19.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:19.905 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:19.933 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:19.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:19.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:19 smithi204 ceph-mon[87071]: pgmap v569: 109 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 105 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 4594/158553 objects misplaced (2.897%); 19 MiB/s, 297 objects/s recovering 2024-09-13T14:47:19.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:19 smithi204 ceph-mon[87071]: 6.18 starting backfill to osd.0 from (0'0,0'0] MAX to 205'10676 2024-09-13T14:47:19.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:19 smithi204 ceph-mon[87071]: 6.18 starting backfill to osd.1 from (0'0,0'0] MAX to 205'10676 2024-09-13T14:47:19.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:19 smithi204 ceph-mon[87071]: osdmap e209: 8 total, 8 up, 7 in 2024-09-13T14:47:20.035 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:20.060 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:20.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:20.161 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:20.182 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:20.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:20.284 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:20.315 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:20.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:20.417 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:20.446 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:20.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:20.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:47:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:47:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:20.548 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:20.572 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:20.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:20.674 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:20.696 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:20.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:20.798 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:20.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.820 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:20.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:20.921 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:20.942 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:20.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:20.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:20 smithi204 ceph-mon[87071]: 6.7 scrub starts 2024-09-13T14:47:20.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:20 smithi204 ceph-mon[87071]: osdmap e210: 8 total, 8 up, 7 in 2024-09-13T14:47:20.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:20 smithi204 ceph-mon[87071]: 6.19 starting backfill to osd.1 from (0'0,0'0] MAX to 207'10811 2024-09-13T14:47:20.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:20 smithi204 ceph-mon[87071]: 6.19 starting backfill to osd.2 from (0'0,0'0] MAX to 207'10811 2024-09-13T14:47:20.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:20 smithi204 ceph-mon[87071]: 6.19 starting backfill to osd.5 from (0'0,0'0] MAX to 207'10811 2024-09-13T14:47:21.044 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:21.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:20 smithi096 ceph-mon[102928]: 6.7 scrub starts 2024-09-13T14:47:21.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:20 smithi096 ceph-mon[102928]: osdmap e210: 8 total, 8 up, 7 in 2024-09-13T14:47:21.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:20 smithi096 ceph-mon[102928]: 6.19 starting backfill to osd.1 from (0'0,0'0] MAX to 207'10811 2024-09-13T14:47:21.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:20 smithi096 ceph-mon[102928]: 6.19 starting backfill to osd.2 from (0'0,0'0] MAX to 207'10811 2024-09-13T14:47:21.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:20 smithi096 ceph-mon[102928]: 6.19 starting backfill to osd.5 from (0'0,0'0] MAX to 207'10811 2024-09-13T14:47:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:20 smithi096 ceph-mon[98901]: 6.7 scrub starts 2024-09-13T14:47:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:20 smithi096 ceph-mon[98901]: osdmap e210: 8 total, 8 up, 7 in 2024-09-13T14:47:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:20 smithi096 ceph-mon[98901]: 6.19 starting backfill to osd.1 from (0'0,0'0] MAX to 207'10811 2024-09-13T14:47:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:20 smithi096 ceph-mon[98901]: 6.19 starting backfill to osd.2 from (0'0,0'0] MAX to 207'10811 2024-09-13T14:47:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:20 smithi096 ceph-mon[98901]: 6.19 starting backfill to osd.5 from (0'0,0'0] MAX to 207'10811 2024-09-13T14:47:21.074 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:21.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:21.176 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:21.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.209 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:21.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:21.310 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:21.331 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:21.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:21.432 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:21.464 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:21.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:21.566 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:21.597 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:21.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:21.699 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:21.726 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:21.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:21.828 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:21.866 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:21.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:21.968 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:21.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:21 smithi204 ceph-mon[87071]: pgmap v572: 109 pgs: 1 peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 105 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5004/157086 objects misplaced (3.186%) 2024-09-13T14:47:21.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:21 smithi204 ceph-mon[87071]: 6.8 scrub starts 2024-09-13T14:47:21.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:21 smithi204 ceph-mon[87071]: 6.8 scrub ok 2024-09-13T14:47:21.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:21.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:21 smithi204 ceph-mon[87071]: osdmap e211: 8 total, 8 up, 7 in 2024-09-13T14:47:21.996 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:21.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:22.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:21 smithi096 ceph-mon[98901]: pgmap v572: 109 pgs: 1 peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 105 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5004/157086 objects misplaced (3.186%) 2024-09-13T14:47:22.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:21 smithi096 ceph-mon[98901]: 6.8 scrub starts 2024-09-13T14:47:22.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:21 smithi096 ceph-mon[98901]: 6.8 scrub ok 2024-09-13T14:47:22.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:22.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:21 smithi096 ceph-mon[98901]: osdmap e211: 8 total, 8 up, 7 in 2024-09-13T14:47:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:21 smithi096 ceph-mon[102928]: pgmap v572: 109 pgs: 1 peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 105 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5004/157086 objects misplaced (3.186%) 2024-09-13T14:47:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:21 smithi096 ceph-mon[102928]: 6.8 scrub starts 2024-09-13T14:47:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:21 smithi096 ceph-mon[102928]: 6.8 scrub ok 2024-09-13T14:47:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:21 smithi096 ceph-mon[102928]: osdmap e211: 8 total, 8 up, 7 in 2024-09-13T14:47:22.098 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:22.132 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:22.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:22.233 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:22.267 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:22.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:22.369 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:22.389 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:22.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:22.491 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:22.518 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:22.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:22.620 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:22.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.657 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:22.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:22.758 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:22.795 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:22.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:22.897 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:22.925 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:22.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:23.027 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:23.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.066 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:23.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:23.167 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:23.204 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:23.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:23.306 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:23.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.346 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:23.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:23.447 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:23.490 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:23.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:23.592 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:23.613 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:23.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:23.715 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:23.746 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:23.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:23.848 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:23.889 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:23.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:23.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:23 smithi204 ceph-mon[87071]: pgmap v575: 109 pgs: 1 peering, 1 active+remapped+backfilling, 107 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 2416/155511 objects misplaced (1.554%) 2024-09-13T14:47:23.991 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:24.021 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:24.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:24.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:23 smithi096 ceph-mon[98901]: pgmap v575: 109 pgs: 1 peering, 1 active+remapped+backfilling, 107 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 2416/155511 objects misplaced (1.554%) 2024-09-13T14:47:24.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:23 smithi096 ceph-mon[102928]: pgmap v575: 109 pgs: 1 peering, 1 active+remapped+backfilling, 107 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 2416/155511 objects misplaced (1.554%) 2024-09-13T14:47:24.123 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:24.145 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:24.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:24.247 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:24.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.275 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:24.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:24.376 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:24.402 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:24.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:24.504 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:24.541 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:24.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:24.643 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:24.682 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:24.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:24.784 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:24.817 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:24.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:24.919 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:24.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.959 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:24.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:24.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:24 smithi204 ceph-mon[87071]: 6.17 scrub starts 2024-09-13T14:47:24.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:24 smithi204 ceph-mon[87071]: 6.17 scrub ok 2024-09-13T14:47:25.060 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:25.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:24 smithi096 ceph-mon[98901]: 6.17 scrub starts 2024-09-13T14:47:25.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:24 smithi096 ceph-mon[98901]: 6.17 scrub ok 2024-09-13T14:47:25.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:24 smithi096 ceph-mon[102928]: 6.17 scrub starts 2024-09-13T14:47:25.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:24 smithi096 ceph-mon[102928]: 6.17 scrub ok 2024-09-13T14:47:25.102 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:25.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:25.204 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:25.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.245 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:25.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:25.346 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:25.375 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:25.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:25.477 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:25.512 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:25.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:25.614 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:25.655 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:25.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:25.757 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:25.798 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:25.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:25.900 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:25.938 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:25.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:25.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:25 smithi204 ceph-mon[87071]: pgmap v576: 109 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 107 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 6970/150039 objects misplaced (4.645%) 2024-09-13T14:47:25.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:25 smithi204 ceph-mon[87071]: 6.5 scrub starts 2024-09-13T14:47:25.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:25 smithi204 ceph-mon[87071]: 6.7 scrub starts 2024-09-13T14:47:25.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:25 smithi204 ceph-mon[87071]: 6.5 scrub ok 2024-09-13T14:47:26.040 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:26.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:25 smithi096 ceph-mon[98901]: pgmap v576: 109 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 107 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 6970/150039 objects misplaced (4.645%) 2024-09-13T14:47:26.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:25 smithi096 ceph-mon[98901]: 6.5 scrub starts 2024-09-13T14:47:26.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:25 smithi096 ceph-mon[98901]: 6.7 scrub starts 2024-09-13T14:47:26.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:25 smithi096 ceph-mon[98901]: 6.5 scrub ok 2024-09-13T14:47:26.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:25 smithi096 ceph-mon[102928]: pgmap v576: 109 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 107 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 6970/150039 objects misplaced (4.645%) 2024-09-13T14:47:26.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:25 smithi096 ceph-mon[102928]: 6.5 scrub starts 2024-09-13T14:47:26.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:25 smithi096 ceph-mon[102928]: 6.7 scrub starts 2024-09-13T14:47:26.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:25 smithi096 ceph-mon[102928]: 6.5 scrub ok 2024-09-13T14:47:26.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.081 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:26.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:26.182 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:26.214 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:26.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:26.315 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:26.336 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:26.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:26.438 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:26.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.469 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:26.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:26.570 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:26.591 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:26.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:26.692 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:26.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.734 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:26.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:26.835 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:26.855 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:26.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:26.957 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:26.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:26 smithi204 ceph-mon[87071]: 6.7 scrub ok 2024-09-13T14:47:26.988 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:26.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:27.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:26 smithi096 ceph-mon[98901]: 6.7 scrub ok 2024-09-13T14:47:27.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:26 smithi096 ceph-mon[102928]: 6.7 scrub ok 2024-09-13T14:47:27.090 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:27.119 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:27.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:27.221 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:27.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.260 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:27.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:27.361 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:27.403 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:27.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:27.504 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:27.536 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:27.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:27.637 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:27.665 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:27.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:27.767 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:27.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.806 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:27.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:27.820 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:27.907 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:27.919 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:27 smithi096 ceph-mon[98901]: pgmap v577: 109 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 107 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 5448/149241 objects misplaced (3.650%) 2024-09-13T14:47:27.920 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:27 smithi096 ceph-mon[102928]: pgmap v577: 109 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 107 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 5448/149241 objects misplaced (3.650%) 2024-09-13T14:47:27.934 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:27.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:27.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:27 smithi204 ceph-mon[87071]: pgmap v577: 109 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 107 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 5448/149241 objects misplaced (3.650%) 2024-09-13T14:47:28.036 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:28.084 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:28.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:28.186 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:28.216 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:47:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:47:27] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.51.0" 2024-09-13T14:47:28.233 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:28.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:28.335 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:28.374 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:28.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:28.476 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:28.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.509 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:28.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:28.610 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:28.643 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:28.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:28.745 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:28.770 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:28.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:28.872 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:28.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.894 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:28.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:28.958 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:28 smithi096 ceph-mon[98901]: pgmap v578: 109 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 107 active+clean; 2.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 5448/137022 objects misplaced (3.976%); 9.2 MiB/s, 147 objects/s recovering 2024-09-13T14:47:28.958 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:28 smithi096 ceph-mon[102928]: pgmap v578: 109 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 107 active+clean; 2.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 5448/137022 objects misplaced (3.976%); 9.2 MiB/s, 147 objects/s recovering 2024-09-13T14:47:28.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:28 smithi204 ceph-mon[87071]: pgmap v578: 109 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 107 active+clean; 2.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 5448/137022 objects misplaced (3.976%); 9.2 MiB/s, 147 objects/s recovering 2024-09-13T14:47:28.995 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:29.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.039 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:29.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:29.140 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:29.167 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:29.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:29.269 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:29.296 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:47:29.297 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:47:29.297 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:47:29.297 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:47:29.297 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:47:29.297 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:47:29.297 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:47:29.297 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:47:29.297 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:29.297 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:29.298 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:29.298 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:29.298 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:29.298 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:29.298 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:29.298 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:29.309 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:29.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:29.412 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:29.435 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:29.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:29.536 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:29.567 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:29.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:29.669 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:29.690 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:29.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:29.792 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:29.823 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:29.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:29.925 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:29.960 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:29.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:29.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:29 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4219770753' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:30.062 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:30.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:29 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4219770753' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:30.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:29 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4219770753' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:30.092 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:30.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:30.194 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:30.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.223 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:30.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:30.324 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:30.364 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:30.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:30.466 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:47:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:47:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:30.496 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:30.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:30.598 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:30.632 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:30.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:30.738 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:30.765 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:30.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:30.867 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:30.896 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:30.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:30.926 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:30 smithi096 ceph-mon[98901]: pgmap v579: 109 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 107 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 4004/132672 objects misplaced (3.018%); 10 MiB/s, 162 objects/s recovering 2024-09-13T14:47:30.926 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:30 smithi096 ceph-mon[102928]: pgmap v579: 109 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 107 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 4004/132672 objects misplaced (3.018%); 10 MiB/s, 162 objects/s recovering 2024-09-13T14:47:30.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:30 smithi204 ceph-mon[87071]: pgmap v579: 109 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 107 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 4004/132672 objects misplaced (3.018%); 10 MiB/s, 162 objects/s recovering 2024-09-13T14:47:30.998 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:31.027 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:31.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:31.129 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:31.155 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:31.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:31.257 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:31.278 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:31.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:31.379 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:31.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.402 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:31.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:31.503 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:31.508 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:47:31.508 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:47:31.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.530 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:31.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:31.631 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:31.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.658 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:31.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:31.758 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:31.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.790 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:31.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:31.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:31 smithi096 ceph-mon[98901]: osdmap e212: 8 total, 8 up, 7 in 2024-09-13T14:47:31.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:31 smithi096 ceph-mon[102928]: osdmap e212: 8 total, 8 up, 7 in 2024-09-13T14:47:31.891 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:31.917 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:31.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:31.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:31 smithi204 ceph-mon[87071]: osdmap e212: 8 total, 8 up, 7 in 2024-09-13T14:47:32.019 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:32.051 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:32.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:32.152 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:32.188 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:32.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:32.290 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:32.322 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:32.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:32.405 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:47:32.424 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:32.456 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:32.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:32.557 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:32.585 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:32.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:32.687 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:32.713 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:32.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:32.815 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:32.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.845 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:32.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:32.905 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:32 smithi096 ceph-mon[98901]: osdmap e213: 8 total, 8 up, 7 in 2024-09-13T14:47:32.905 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:32 smithi096 ceph-mon[98901]: pgmap v582: 109 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 107 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 4004/128505 objects misplaced (3.116%) 2024-09-13T14:47:32.906 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:32 smithi096 ceph-mon[102928]: osdmap e213: 8 total, 8 up, 7 in 2024-09-13T14:47:32.906 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:32 smithi096 ceph-mon[102928]: pgmap v582: 109 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 107 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 4004/128505 objects misplaced (3.116%) 2024-09-13T14:47:32.928 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:47:32.946 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:32.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:32 smithi204 ceph-mon[87071]: osdmap e213: 8 total, 8 up, 7 in 2024-09-13T14:47:32.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:32 smithi204 ceph-mon[87071]: pgmap v582: 109 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 107 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 4004/128505 objects misplaced (3.116%) 2024-09-13T14:47:33.018 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:33.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:33.120 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:33.157 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:33.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:33.259 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:33.289 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:33.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:33.391 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:33.427 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:33.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:33.529 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:33.573 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:33.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:33.675 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:33.706 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:33.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:33.809 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:33.845 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:33.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:33.918 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:33 smithi096 ceph-mon[98901]: 6.18 scrub starts 2024-09-13T14:47:33.918 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:33 smithi096 ceph-mon[98901]: 6.18 scrub ok 2024-09-13T14:47:33.919 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:33 smithi096 ceph-mon[102928]: 6.18 scrub starts 2024-09-13T14:47:33.919 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:33 smithi096 ceph-mon[102928]: 6.18 scrub ok 2024-09-13T14:47:33.946 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:33.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:33 smithi204 ceph-mon[87071]: 6.18 scrub starts 2024-09-13T14:47:33.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:33 smithi204 ceph-mon[87071]: 6.18 scrub ok 2024-09-13T14:47:34.011 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:34.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:34.028 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:34.065 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:34.117 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:34.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.171 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:34.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:34.272 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:34.317 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:34.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:34.418 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:34.447 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:34.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:34.549 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:34.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.580 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:34.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:34.681 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:34.687 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:47:34.736 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:34.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:34.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:34 smithi096 ceph-mon[98901]: 6.12 scrub starts 2024-09-13T14:47:34.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:34 smithi096 ceph-mon[98901]: pgmap v583: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 2082/118479 objects misplaced (1.757%) 2024-09-13T14:47:34.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:34 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4135533504' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:34.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:34 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:34.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:34 smithi096 ceph-mon[98901]: 6.12 scrub ok 2024-09-13T14:47:34.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:34 smithi096 ceph-mon[102928]: 6.12 scrub starts 2024-09-13T14:47:34.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:34 smithi096 ceph-mon[102928]: pgmap v583: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 2082/118479 objects misplaced (1.757%) 2024-09-13T14:47:34.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:34 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4135533504' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:34.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:34 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:34.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:34 smithi096 ceph-mon[102928]: 6.12 scrub ok 2024-09-13T14:47:34.838 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:34.875 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:34.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:34.878 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T14:47:34.878 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T14:47:34.878 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T14:47:34.879 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T14:47:34.879 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T14:47:34.879 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T14:47:34.879 INFO:teuthology.orchestra.run.smithi096.stderr:bench [] [] [] [] 2024-09-13T14:47:34.879 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:47:34.879 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T14:47:34.879 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T14:47:34.879 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore allocator dump block 2024-09-13T14:47:34.879 INFO:teuthology.orchestra.run.smithi096.stderr:Error EINVAL: invalid command 2024-09-13T14:47:34.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:34 smithi204 ceph-mon[87071]: 6.12 scrub starts 2024-09-13T14:47:34.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:34 smithi204 ceph-mon[87071]: pgmap v583: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 2082/118479 objects misplaced (1.757%) 2024-09-13T14:47:34.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:34 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4135533504' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:34.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:34 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:34.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:34 smithi204 ceph-mon[87071]: 6.12 scrub ok 2024-09-13T14:47:34.977 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:35.014 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:35.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:35.116 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:35.161 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:35.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:35.263 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:35.291 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:35.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:35.393 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:35.439 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:35.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:35.541 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:35.568 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:35.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:35.670 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:35.698 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:35.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:35.800 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:35.829 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:35.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:35.931 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:35.963 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:35.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:35.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:35 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:47:35.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:35 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:47:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:35 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:35 smithi204 ceph-mon[87071]: osdmap e214: 8 total, 8 up, 7 in 2024-09-13T14:47:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:35 smithi204 ceph-mon[87071]: 6.15 scrub starts 2024-09-13T14:47:36.066 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:36.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:35 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:47:36.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:35 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:47:36.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:35 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:36.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:35 smithi096 ceph-mon[98901]: osdmap e214: 8 total, 8 up, 7 in 2024-09-13T14:47:36.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:35 smithi096 ceph-mon[98901]: 6.15 scrub starts 2024-09-13T14:47:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:35 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:47:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:35 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:47:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:35 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:35 smithi096 ceph-mon[102928]: osdmap e214: 8 total, 8 up, 7 in 2024-09-13T14:47:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:35 smithi096 ceph-mon[102928]: 6.15 scrub starts 2024-09-13T14:47:36.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.107 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:36.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:36.208 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:36.236 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:36.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:36.338 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:36.364 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:36.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:36.466 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:36.467 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:47:36.467 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:47:36.468 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.3 'trim stale osdmaps' 2024-09-13T14:47:36.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.509 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:36.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:36.609 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:36.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.661 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:36.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:36.762 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:36.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.802 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:36.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:36.903 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:36.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.948 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:36.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:37.049 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:37.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:36 smithi096 ceph-mon[98901]: pgmap v585: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 2082/117966 objects misplaced (1.765%) 2024-09-13T14:47:37.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:47:37.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:37.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 7]} : dispatch 2024-09-13T14:47:37.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:36 smithi096 ceph-mon[102928]: pgmap v585: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 2082/117966 objects misplaced (1.765%) 2024-09-13T14:47:37.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:47:37.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:37.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 7]} : dispatch 2024-09-13T14:47:37.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.082 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:37.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:37.183 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:37.212 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:37.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:37.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:36 smithi204 ceph-mon[87071]: pgmap v585: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 2082/117966 objects misplaced (1.765%) 2024-09-13T14:47:37.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:47:37.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 7]} : dispatch 2024-09-13T14:47:37.313 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:37.338 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:37.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:37.439 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:37.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.469 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:37.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:37.570 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:37.622 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:37.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:37.724 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:37.766 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:37.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:37.869 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:37.896 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:37.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:37.998 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:38.027 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:38.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:38.055 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:37 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 7]}]': finished 2024-09-13T14:47:38.055 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:37 smithi096 ceph-mon[98901]: osdmap e215: 8 total, 8 up, 7 in 2024-09-13T14:47:38.055 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:47:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:47:37] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.51.0" 2024-09-13T14:47:38.056 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:37 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 7]}]': finished 2024-09-13T14:47:38.056 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:37 smithi096 ceph-mon[102928]: osdmap e215: 8 total, 8 up, 7 in 2024-09-13T14:47:38.129 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:38.155 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:38.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:38.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:37 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 7]}]': finished 2024-09-13T14:47:38.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:37 smithi204 ceph-mon[87071]: osdmap e215: 8 total, 8 up, 7 in 2024-09-13T14:47:38.257 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:38.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.301 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:38.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:38.402 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:38.445 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:38.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:38.546 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:38.583 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:38.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:38.650 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:38.685 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:38.711 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:38.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:38.813 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:38.877 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:38.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:38.978 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:39.005 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:39.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:39.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:38 smithi096 ceph-mon[98901]: 6.15 scrub starts 2024-09-13T14:47:39.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:38 smithi096 ceph-mon[98901]: pgmap v587: 109 pgs: 1 peering, 1 active+remapped+backfilling, 107 active+clean; 2.1 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 2082/105237 objects misplaced (1.978%) 2024-09-13T14:47:39.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:38 smithi096 ceph-mon[98901]: osdmap e216: 8 total, 8 up, 7 in 2024-09-13T14:47:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:38 smithi096 ceph-mon[102928]: 6.15 scrub starts 2024-09-13T14:47:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:38 smithi096 ceph-mon[102928]: pgmap v587: 109 pgs: 1 peering, 1 active+remapped+backfilling, 107 active+clean; 2.1 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 2082/105237 objects misplaced (1.978%) 2024-09-13T14:47:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:38 smithi096 ceph-mon[102928]: osdmap e216: 8 total, 8 up, 7 in 2024-09-13T14:47:39.107 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:39.142 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:39.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:38 smithi204 ceph-mon[87071]: 6.15 scrub starts 2024-09-13T14:47:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:38 smithi204 ceph-mon[87071]: pgmap v587: 109 pgs: 1 peering, 1 active+remapped+backfilling, 107 active+clean; 2.1 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 2082/105237 objects misplaced (1.978%) 2024-09-13T14:47:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:38 smithi204 ceph-mon[87071]: osdmap e216: 8 total, 8 up, 7 in 2024-09-13T14:47:39.244 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:39.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.277 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:39.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:39.378 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:39.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.415 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:39.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:39.494 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:47:39.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:39.543 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:39.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:39.645 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:39.673 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:39.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:39.774 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:39.798 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:39.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:39.828 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:47:39.828 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:47:39.900 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:39.943 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:39.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:40.045 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:40.074 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:40.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:40.176 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:40.205 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:40.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:40.307 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:40.339 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:40.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:40.440 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:40.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:47:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:47:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:40.479 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:40.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:40.581 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:40.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.620 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:40.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:40.722 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:40.752 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:40.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:40.854 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:40.890 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:40.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:40.992 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:41.034 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:41.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:41.059 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:40 smithi096 ceph-mon[98901]: pgmap v589: 109 pgs: 1 active+remapped+backfilling, 1 peering, 107 active+clean; 2.1 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 870/100947 objects misplaced (0.862%) 2024-09-13T14:47:41.059 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:40 smithi096 ceph-mon[102928]: pgmap v589: 109 pgs: 1 active+remapped+backfilling, 1 peering, 107 active+clean; 2.1 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 870/100947 objects misplaced (0.862%) 2024-09-13T14:47:41.136 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:41.162 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:41.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:41.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:40 smithi204 ceph-mon[87071]: pgmap v589: 109 pgs: 1 active+remapped+backfilling, 1 peering, 107 active+clean; 2.1 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 870/100947 objects misplaced (0.862%) 2024-09-13T14:47:41.264 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:41.293 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:47:41.293 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:41.298 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:47:41.299 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:41.322 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:41.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:41.338 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:47:41.338 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:41.372 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:47:41.372 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:41.403 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:47:41.403 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:41.424 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:41.437 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:47:41.437 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:41.451 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:41.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:41.471 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:47:41.471 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:41.499 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:47:41.499 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:41.553 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:41.599 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:41.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:41.705 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:41.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.744 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:41.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:41.845 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:41.891 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:41.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:41.992 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:42.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.026 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:42.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:42.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:41 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/791199433' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:42.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:41 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/791199433' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:42.126 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:42.160 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:42.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:42.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:41 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/791199433' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:42.262 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:42.300 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:42.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:42.402 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:42.451 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:42.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:42.553 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:42.581 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:42.581 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T14:47:42.581 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T14:47:42.581 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T14:47:42.582 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T14:47:42.582 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T14:47:42.582 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T14:47:42.582 INFO:teuthology.orchestra.run.smithi096.stderr:bench [] [] [] [] 2024-09-13T14:47:42.582 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:47:42.582 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T14:47:42.582 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T14:47:42.582 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore allocator dump block 2024-09-13T14:47:42.582 INFO:teuthology.orchestra.run.smithi096.stderr:Error EINVAL: invalid command 2024-09-13T14:47:42.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:42.684 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:42.713 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:42.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:42.814 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:42.852 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:42.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:42.954 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:42.987 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:42.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:43.090 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:43.115 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:43.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:43.217 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:43.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:42 smithi204 ceph-mon[87071]: 6.15 scrub starts 2024-09-13T14:47:43.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:42 smithi204 ceph-mon[87071]: pgmap v590: 109 pgs: 1 active+remapped+backfilling, 1 peering, 107 active+clean; 2.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 163 op/s; 870/96411 objects misplaced (0.902%) 2024-09-13T14:47:43.219 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:42 smithi096 ceph-mon[98901]: 6.15 scrub starts 2024-09-13T14:47:43.220 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:42 smithi096 ceph-mon[98901]: pgmap v590: 109 pgs: 1 active+remapped+backfilling, 1 peering, 107 active+clean; 2.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 163 op/s; 870/96411 objects misplaced (0.902%) 2024-09-13T14:47:43.220 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:42 smithi096 ceph-mon[102928]: 6.15 scrub starts 2024-09-13T14:47:43.220 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:42 smithi096 ceph-mon[102928]: pgmap v590: 109 pgs: 1 active+remapped+backfilling, 1 peering, 107 active+clean; 2.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 163 op/s; 870/96411 objects misplaced (0.902%) 2024-09-13T14:47:43.238 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:43.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:43.340 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:43.361 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:43.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:43.462 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:43.484 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:43.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:43.585 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:43.593 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:47:43.593 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:47:43.593 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.4 'trim stale osdmaps' 2024-09-13T14:47:43.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.607 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:43.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:43.709 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:43.730 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:43.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:43.831 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:43.853 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:43.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:43.954 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:43.985 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:43.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:44.086 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:44.121 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:44.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:44.158 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:43 smithi096 ceph-mon[98901]: osdmap e217: 8 total, 8 up, 7 in 2024-09-13T14:47:44.159 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:43 smithi096 ceph-mon[102928]: osdmap e217: 8 total, 8 up, 7 in 2024-09-13T14:47:44.196 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:47:44.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:43 smithi204 ceph-mon[87071]: osdmap e217: 8 total, 8 up, 7 in 2024-09-13T14:47:44.223 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:44.231 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:44.300 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:44.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:44.401 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:44.435 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:47:44.450 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:44.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:44.551 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:44.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.621 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:44.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:44.722 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:44.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.754 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:44.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:44.855 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:44.891 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:44.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:44.993 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:45.030 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:45.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:45.131 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:45.168 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:45.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:45.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:44 smithi204 ceph-mon[87071]: 6.15 scrub starts 2024-09-13T14:47:45.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:44 smithi204 ceph-mon[87071]: 6.15 scrub ok 2024-09-13T14:47:45.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:44 smithi204 ceph-mon[87071]: pgmap v592: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.33k op/s; 870/85338 objects misplaced (1.019%); 3.1 MiB/s, 48 objects/s recovering 2024-09-13T14:47:45.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:44 smithi204 ceph-mon[87071]: osdmap e218: 8 total, 8 up, 7 in 2024-09-13T14:47:45.270 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:45.280 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:44 smithi096 ceph-mon[102928]: 6.15 scrub starts 2024-09-13T14:47:45.280 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:44 smithi096 ceph-mon[102928]: 6.15 scrub ok 2024-09-13T14:47:45.280 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:44 smithi096 ceph-mon[102928]: pgmap v592: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.33k op/s; 870/85338 objects misplaced (1.019%); 3.1 MiB/s, 48 objects/s recovering 2024-09-13T14:47:45.280 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:44 smithi096 ceph-mon[102928]: osdmap e218: 8 total, 8 up, 7 in 2024-09-13T14:47:45.282 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:44 smithi096 ceph-mon[98901]: 6.15 scrub starts 2024-09-13T14:47:45.282 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:44 smithi096 ceph-mon[98901]: 6.15 scrub ok 2024-09-13T14:47:45.282 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:44 smithi096 ceph-mon[98901]: pgmap v592: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.33k op/s; 870/85338 objects misplaced (1.019%); 3.1 MiB/s, 48 objects/s recovering 2024-09-13T14:47:45.282 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:44 smithi096 ceph-mon[98901]: osdmap e218: 8 total, 8 up, 7 in 2024-09-13T14:47:45.307 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:45.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:45.409 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:45.446 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:45.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:45.548 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:45.573 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:45.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:45.675 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:45.710 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:45.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:45.812 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:45.833 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:45.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:45.935 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:45.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.973 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:45.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:46.074 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:46.095 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:46.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:46.197 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:46.228 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:46.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:46.330 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:46.363 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:46.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:46.400 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:46.465 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:46.493 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:46.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:46.595 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:46.635 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:46.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:46.737 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:46.777 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:46.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:46.879 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:46.916 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:46.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:47.018 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:47.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.052 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:47.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:47.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:46 smithi096 ceph-mon[98901]: pgmap v594: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 857 op/s; 870/84801 objects misplaced (1.026%); 2.7 MiB/s, 42 objects/s recovering 2024-09-13T14:47:47.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:46 smithi096 ceph-mon[102928]: pgmap v594: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 857 op/s; 870/84801 objects misplaced (1.026%); 2.7 MiB/s, 42 objects/s recovering 2024-09-13T14:47:47.154 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:47.195 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:47.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:47.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:46 smithi204 ceph-mon[87071]: pgmap v594: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 857 op/s; 870/84801 objects misplaced (1.026%); 2.7 MiB/s, 42 objects/s recovering 2024-09-13T14:47:47.297 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:47.325 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:47.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:47.427 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:47.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.456 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:47.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:47.557 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:47.595 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:47.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:47.697 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:47.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.733 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:47.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:47.834 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:47.868 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:47.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:47.970 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:48.001 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:48.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:48.103 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:48.141 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:48.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:48.211 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:47:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:47:47] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.51.0" 2024-09-13T14:47:48.243 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:48.274 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:48.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:48.376 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:48.406 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:48.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:48.508 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:48.547 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:48.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:48.649 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:48.678 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:48.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:48.780 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:48.815 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:48.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:48.917 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:48.961 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:48.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:49.062 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:49.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.100 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:49.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:49.201 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:49.202 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:48 smithi096 ceph-mon[98901]: pgmap v595: 109 pgs: 109 active+clean; 1.5 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 366 op/s 2024-09-13T14:47:49.202 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:48 smithi096 ceph-mon[102928]: pgmap v595: 109 pgs: 109 active+clean; 1.5 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 366 op/s 2024-09-13T14:47:49.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:48 smithi204 ceph-mon[87071]: pgmap v595: 109 pgs: 109 active+clean; 1.5 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 366 op/s 2024-09-13T14:47:49.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.239 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:49.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:49.340 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:49.380 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:49.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:49.482 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:49.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.517 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:49.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:49.618 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:49.646 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:49.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:49.748 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:49.789 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:49.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:49.891 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:49.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.925 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:49.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:50.026 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:50.060 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:50.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:50.162 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:50.185 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:50.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:50.287 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:50.317 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:50.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:50.419 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:50.453 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:50.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:50.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:47:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:47:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:50.556 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:50.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.592 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:50.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:50.693 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:50.714 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:50.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:50.816 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:50.848 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:50.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:50.949 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:50.984 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:50.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:51.086 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:51.115 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:51.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:51.196 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:50 smithi096 ceph-mon[98901]: pgmap v596: 109 pgs: 109 active+clean; 1.4 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 338 op/s 2024-09-13T14:47:51.196 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:50 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:51.197 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:50 smithi096 ceph-mon[102928]: pgmap v596: 109 pgs: 109 active+clean; 1.4 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 338 op/s 2024-09-13T14:47:51.197 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:50 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:51.217 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:51.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:50 smithi204 ceph-mon[87071]: pgmap v596: 109 pgs: 109 active+clean; 1.4 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 338 op/s 2024-09-13T14:47:51.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:50 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:51.263 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:51.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:51.365 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:51.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.409 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:51.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:51.510 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:51.539 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:51.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:51.640 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:51.680 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:51.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:51.781 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:51.803 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:51.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:51.904 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:51.934 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:51.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:51.948 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:47:52.035 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:52.070 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:52.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:52.172 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:52.203 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:52.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:52.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:51 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/390818399' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:52.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:51 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:52.306 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:52.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:51 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/390818399' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:52.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:51 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:52.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:51 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/390818399' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:52.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:51 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:52.341 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:52.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:52.443 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:52.487 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:52.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:52.588 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:52.616 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:52.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:52.718 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:52.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.753 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:52.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:52.854 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:52.895 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:52.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:52.997 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:53.027 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:53.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:53.129 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:53.164 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:53.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:53.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:52 smithi204 ceph-mon[87071]: pgmap v597: 109 pgs: 109 active+clean; 1.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 108 op/s 2024-09-13T14:47:53.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:52 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:53.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:52 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:47:53.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:52 smithi204 ceph-mon[87071]: osdmap e219: 8 total, 8 up, 7 in 2024-09-13T14:47:53.266 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:53.295 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:53.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:53.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:52 smithi096 ceph-mon[98901]: pgmap v597: 109 pgs: 109 active+clean; 1.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 108 op/s 2024-09-13T14:47:53.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:52 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:53.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:52 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:47:53.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:52 smithi096 ceph-mon[98901]: osdmap e219: 8 total, 8 up, 7 in 2024-09-13T14:47:53.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:52 smithi096 ceph-mon[102928]: pgmap v597: 109 pgs: 109 active+clean; 1.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 108 op/s 2024-09-13T14:47:53.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:52 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:53.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:52 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:47:53.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:52 smithi096 ceph-mon[102928]: osdmap e219: 8 total, 8 up, 7 in 2024-09-13T14:47:53.397 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:53.421 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:53.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:53.523 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.559 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:53.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:53.660 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:53.693 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:53.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:53.795 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:53.831 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:53.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:53.933 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:53.969 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:53.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:54.071 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:54.100 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:54.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:54.202 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:54.250 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:54.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:54.352 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:54.380 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:54.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:54.482 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:54.507 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:54.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:54.609 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:54.646 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:54.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:54.748 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:54.781 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:54.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:54.883 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:54.913 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:54.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:55.015 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:55.040 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:55.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:55.050 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T14:47:55.050 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T14:47:55.050 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T14:47:55.050 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T14:47:55.050 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T14:47:55.050 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T14:47:55.050 INFO:teuthology.orchestra.run.smithi096.stderr:bench [] [] [] [] 2024-09-13T14:47:55.050 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:47:55.050 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T14:47:55.050 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T14:47:55.051 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore allocator dump block 2024-09-13T14:47:55.051 INFO:teuthology.orchestra.run.smithi096.stderr:Error EINVAL: invalid command 2024-09-13T14:47:55.142 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:55.171 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:55.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:55.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:54 smithi204 ceph-mon[87071]: pgmap v599: 109 pgs: 109 active+clean; 1.0 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 491 op/s 2024-09-13T14:47:55.273 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:55.310 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:55.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:55.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:54 smithi096 ceph-mon[98901]: pgmap v599: 109 pgs: 109 active+clean; 1.0 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 491 op/s 2024-09-13T14:47:55.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:54 smithi096 ceph-mon[102928]: pgmap v599: 109 pgs: 109 active+clean; 1.0 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 491 op/s 2024-09-13T14:47:55.412 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:55.446 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:55.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:55.493 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:55.548 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:55.554 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:55.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.608 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:55.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:55.709 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:55.743 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:55.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:55.845 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:55.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.876 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:55.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:55.977 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:56.009 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:56.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:56.110 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:56.144 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:56.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:56.245 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:56.273 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:56.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:56.365 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:47:56.365 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:47:56.366 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.5 'trim stale osdmaps' 2024-09-13T14:47:56.374 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:56.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.402 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:56.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:56.503 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:56.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.544 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:56.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:56.646 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:56.698 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:56.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:56.800 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:56.829 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:56.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:56.931 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:56.965 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:56.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:57.068 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:57.109 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:57.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:57.211 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:57.247 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:57.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:57.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:56 smithi096 ceph-mon[98901]: pgmap v600: 109 pgs: 109 active+clean; 1.0 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 486 op/s 2024-09-13T14:47:57.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:56 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:47:57.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:56 smithi096 ceph-mon[102928]: pgmap v600: 109 pgs: 109 active+clean; 1.0 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 486 op/s 2024-09-13T14:47:57.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:56 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:47:57.349 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:57.379 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:57.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:56 smithi204 ceph-mon[87071]: pgmap v600: 109 pgs: 109 active+clean; 1.0 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 486 op/s 2024-09-13T14:47:57.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:56 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:47:57.481 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:57.516 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:57.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:57.532 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:47:57.617 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:57.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.649 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:57.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:57.750 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:57.777 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:57.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:57.879 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:57.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.907 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:57.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:58.008 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:58.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.050 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:58.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:58.151 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:58.183 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:58.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:58.266 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:47:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:47:57] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.51.0" 2024-09-13T14:47:58.284 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:58.311 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:58.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:58.413 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:58.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.455 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:58.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:58.556 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:58.609 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:58.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:58.712 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:58.740 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:58.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:58.842 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:58.865 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:47:58.865 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:58.885 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:58.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:58.900 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:47:58.900 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:58.932 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:47:58.932 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:58.966 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:47:58.967 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:58.986 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:59.004 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:47:59.004 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:59.025 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:59.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:59.033 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:47:59.034 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:59.072 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:47:59.072 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:59.101 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:47:59.101 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:59.116 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:59 smithi096 ceph-mon[98901]: pgmap v601: 109 pgs: 109 active+clean; 779 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-13T14:47:59.116 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:47:59 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1768750904' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:59.120 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:59 smithi096 ceph-mon[102928]: pgmap v601: 109 pgs: 109 active+clean; 779 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-13T14:47:59.120 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:47:59 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1768750904' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:59.128 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:59.156 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:59.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:59.258 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:59.294 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:59.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:59.396 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:59.431 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:59.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:59 smithi204 ceph-mon[87071]: pgmap v601: 109 pgs: 109 active+clean; 779 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-13T14:47:59.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:47:59 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1768750904' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:59.534 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:59.580 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:47:59.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:59.682 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:59.712 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:59.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:59.814 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:59.845 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:59.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:59.947 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:59.982 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:47:59.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:00.085 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:00.112 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:00.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:00.187 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:00.214 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:00.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.270 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:00.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:00.372 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:00.406 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:00.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:48:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:48:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:00.508 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:00.548 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:00.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:00.618 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:00.650 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:00.681 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:00.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:00.784 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:00.829 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:00.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:00.931 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:00.961 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:00.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:01.063 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:01.102 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:01.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:01.204 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:01.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.238 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:01.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:01.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:01 smithi096 ceph-mon[98901]: pgmap v602: 109 pgs: 109 active+clean; 699 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T14:48:01.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:01 smithi096 ceph-mon[102928]: pgmap v602: 109 pgs: 109 active+clean; 699 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T14:48:01.339 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:01.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.378 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:01.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:01.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:01 smithi204 ceph-mon[87071]: pgmap v602: 109 pgs: 109 active+clean; 699 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T14:48:01.479 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:01.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.521 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:01.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:01.622 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:01.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.657 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:01.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:01.758 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:01.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.780 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:01.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:01.882 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:01.923 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:01.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:02.025 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:02.064 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:02.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:02.165 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:02.215 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:02.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:02.317 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:02.325 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:02.325 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:02.346 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:02.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:02.447 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:02.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.483 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:02.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:02.584 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:02.622 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:02.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:02.650 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:48:02.724 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:02.755 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:02.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:02.857 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:02.898 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:02.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:03.000 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:03.036 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:03.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:03.138 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:03.175 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:03.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:03.277 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:03.309 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:03.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:03.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:03 smithi096 ceph-mon[98901]: pgmap v603: 109 pgs: 109 active+clean; 594 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s 2024-09-13T14:48:03.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:03 smithi096 ceph-mon[102928]: pgmap v603: 109 pgs: 109 active+clean; 594 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s 2024-09-13T14:48:03.411 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:03.445 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:03.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:03.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:03 smithi204 ceph-mon[87071]: pgmap v603: 109 pgs: 109 active+clean; 594 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s 2024-09-13T14:48:03.547 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:03.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.600 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:03.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:03.701 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:03.735 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:03.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:03.837 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:03.872 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:03.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:03.974 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:04.014 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:04.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:04.116 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:04.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.157 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:04.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:04.258 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:04.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.289 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:04.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:04.371 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:48:04.389 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:04.425 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:04.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:04.527 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:04.559 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:04.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:04.660 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:04.696 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:04.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:04.768 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:04.799 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:04.841 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:04.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:04.943 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:04.984 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:04.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:05.086 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:05.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.135 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:05.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:05.209 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:05 smithi204 ceph-mon[87071]: pgmap v604: 109 pgs: 109 active+clean; 341 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.24k op/s 2024-09-13T14:48:05.236 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:05.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.280 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:05.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:05.381 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:05.402 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:05 smithi096 ceph-mon[98901]: pgmap v604: 109 pgs: 109 active+clean; 341 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.24k op/s 2024-09-13T14:48:05.402 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:05 smithi096 ceph-mon[102928]: pgmap v604: 109 pgs: 109 active+clean; 341 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.24k op/s 2024-09-13T14:48:05.429 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:05.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:05.531 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:05.566 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:05.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:05.668 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:05.714 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:05.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:05.816 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:05.862 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:05.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:05.964 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:05.993 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:05.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:06.092 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T14:48:06.092 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T14:48:06.092 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T14:48:06.093 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T14:48:06.093 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T14:48:06.093 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T14:48:06.093 INFO:teuthology.orchestra.run.smithi096.stderr:bench [] [] [] [] 2024-09-13T14:48:06.093 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:48:06.094 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T14:48:06.094 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T14:48:06.094 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore allocator dump block 2024-09-13T14:48:06.094 INFO:teuthology.orchestra.run.smithi096.stderr:Error EINVAL: invalid command 2024-09-13T14:48:06.094 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:06.119 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:06.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:06.221 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:06.260 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:06.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:06.362 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:06.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.390 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:06.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:06.492 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:06.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.529 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:06.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:06.630 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:06.665 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:06.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:06.766 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:06.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.800 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:06.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:06.901 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:06.919 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:06.922 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:48:06.923 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:48:06.923 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.7 'trim stale osdmaps' 2024-09-13T14:48:06.935 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:06.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:06.978 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:06 smithi096 ceph-mon[98901]: pgmap v605: 109 pgs: 109 active+clean; 331 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 990 op/s 2024-09-13T14:48:06.978 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:06.978 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:06.978 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:06 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2788561104' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:06.978 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:06 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:06.978 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:06 smithi096 ceph-mon[102928]: pgmap v605: 109 pgs: 109 active+clean; 331 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 990 op/s 2024-09-13T14:48:06.978 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:06.978 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:06.978 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:06 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2788561104' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:06.979 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:06 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:07.035 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:07.035 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:07.035 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:06 smithi204 ceph-mon[87071]: pgmap v605: 109 pgs: 109 active+clean; 331 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 990 op/s 2024-09-13T14:48:07.036 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:07.036 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:07.036 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:06 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2788561104' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:07.036 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:06 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:07.036 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:07.078 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:07.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:07.133 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:48:07.180 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:07.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.221 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:07.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:07.322 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:07.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.356 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:07.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:07.444 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Removed 117225 objects 2024-09-13T14:48:07.444 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Clean up completed and total clean up time :148.493 2024-09-13T14:48:07.456 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:07.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.478 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:07.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:07.579 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:07.600 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:07.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:07.702 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:07.725 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:07.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:07.827 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:07.849 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:07.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:07.951 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:07.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:07 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:07.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:07 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:07.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:07 smithi204 ceph-mon[87071]: osdmap e220: 8 total, 8 up, 7 in 2024-09-13T14:48:07.971 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:07.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:08.073 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:08.096 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:08.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:08.198 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:08.220 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:08.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:08.265 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:48:08.322 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:08.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:48:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:48:07] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.51.0" 2024-09-13T14:48:08.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:07 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:08.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:07 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:08.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:07 smithi096 ceph-mon[98901]: osdmap e220: 8 total, 8 up, 7 in 2024-09-13T14:48:08.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:07 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:08.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:07 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:08.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:07 smithi096 ceph-mon[102928]: osdmap e220: 8 total, 8 up, 7 in 2024-09-13T14:48:08.344 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:08.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:08.446 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:08.469 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:08.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:08.527 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:08.572 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:08.594 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:08.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:08.697 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:08.720 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:08.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:08.821 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:08.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.844 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:08.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:08.945 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:08.968 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:08.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:09.070 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:09.091 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:09.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:09.193 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:09.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.216 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:09.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:09.317 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:09.338 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:09.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:09.440 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:09.460 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:09.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:09.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:09 smithi204 ceph-mon[87071]: pgmap v607: 109 pgs: 109 active+clean; 73 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s 2024-09-13T14:48:09.487 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:09 smithi096 ceph-mon[98901]: pgmap v607: 109 pgs: 109 active+clean; 73 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s 2024-09-13T14:48:09.488 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:09 smithi096 ceph-mon[102928]: pgmap v607: 109 pgs: 109 active+clean; 73 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s 2024-09-13T14:48:09.562 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:09.585 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:09.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:09.687 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:09.708 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:09.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:09.779 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:48:09.779 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:48:09.796 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:48:09.796 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:48:09.810 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:09.812 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:48:09.812 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:48:09.828 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:48:09.828 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:48:09.832 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:09.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:09.843 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:48:09.844 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:48:09.852 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:09.862 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:48:09.863 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:48:09.886 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:48:09.886 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:48:09.904 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:48:09.905 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:48:09.933 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:09.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.957 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:09.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:10.059 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:10.082 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:10.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:10.174 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:10 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4239792015' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:10.179 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:48:10.179 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:48:10.179 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T14:48:10.179 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T14:48:10.179 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:48:10.185 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:10.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.208 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:10.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:10.309 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:10.330 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:10.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:10.432 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.453 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:10.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:10.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:48:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:48:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:10.554 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:10.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:10 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4239792015' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:10.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:10 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4239792015' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:10.576 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:10.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:10.678 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:10.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.700 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:10.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:10.734 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:48:10.799 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:48:10.801 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:10.822 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:10.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:10.924 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:10.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.946 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:10.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:11.046 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:11.048 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:11.070 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:11.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:11.172 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:11.193 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:11.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:11.295 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:11.317 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:11.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:11.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:11 smithi096 ceph-mon[102928]: pgmap v608: 109 pgs: 109 active+clean; 21 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:48:11.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:11 smithi096 ceph-mon[98901]: pgmap v608: 109 pgs: 109 active+clean; 21 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:48:11.419 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:11.439 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:11.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:11.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:11 smithi204 ceph-mon[87071]: pgmap v608: 109 pgs: 109 active+clean; 21 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:48:11.541 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:11.562 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:11.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:11.626 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T14:48:11.626 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T14:48:11.626 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T14:48:11.626 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T14:48:11.626 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T14:48:11.627 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T14:48:11.627 INFO:teuthology.orchestra.run.smithi096.stderr:bench [] [] [] [] 2024-09-13T14:48:11.627 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:48:11.627 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T14:48:11.627 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T14:48:11.627 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore allocator dump block 2024-09-13T14:48:11.627 INFO:teuthology.orchestra.run.smithi096.stderr:Error EINVAL: invalid command 2024-09-13T14:48:11.664 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:11.686 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:11.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:11.789 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:11.810 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:11.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:11.912 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:11.933 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:11.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:12.035 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:12.057 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:12.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:12.159 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:12.180 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:12.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:12.282 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:12.303 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:12.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:12.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:12 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:48:12.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:12 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:48:12.405 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:12.426 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:12.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:12.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:12 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:48:12.529 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:12.551 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:12.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:12.653 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:12.656 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:12.675 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:12.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:12.777 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:12.798 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:12.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:12.900 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:12.922 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:12.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:12.952 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-09-13T14:48:12.952 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-09-13T14:48:13.024 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:13.047 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:13.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:13.109 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:48:13.110 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:48:13.148 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:13.170 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:13.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:13.262 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:48:13.262 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T14:48:13.262 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T14:48:13.262 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:48:13.262 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.262 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:136882 6.2 6:4d112999:::benchmark_data_smithi096_167087_object19654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-09-13T14:48:13.262 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:02.049745+0000", 2024-09-13T14:48:13.263 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 131.208701236, 2024-09-13T14:48:13.263 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.150460636, 2024-09-13T14:48:13.263 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.263 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:48:13.263 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.263 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.263 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.263 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 136882 2024-09-13T14:48:13.263 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.263 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.263 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.263 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.263 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049745+0000", 2024-09-13T14:48:13.264 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.264 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.264 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.264 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.264 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049745+0000", 2024-09-13T14:48:13.264 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.264 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.264 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.264 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.264 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049744+0000", 2024-09-13T14:48:13.264 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:13.264 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.264 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.264 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.265 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049749+0000", 2024-09-13T14:48:13.265 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8510000000000004e-06 2024-09-13T14:48:13.265 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.265 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.265 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.265 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049763+0000", 2024-09-13T14:48:13.265 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3518e-05 2024-09-13T14:48:13.265 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.265 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.265 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.265 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049769+0000", 2024-09-13T14:48:13.265 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0009999999999998e-06 2024-09-13T14:48:13.266 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.266 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.266 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.266 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049809+0000", 2024-09-13T14:48:13.266 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0071000000000003e-05 2024-09-13T14:48:13.266 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.266 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.266 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.266 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049959+0000", 2024-09-13T14:48:13.266 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015019400000000001 2024-09-13T14:48:13.266 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.266 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.267 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:48:13.267 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.050081+0000", 2024-09-13T14:48:13.267 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000121418 2024-09-13T14:48:13.267 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.267 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.267 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.267 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.053462+0000", 2024-09-13T14:48:13.267 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0033818400000000001 2024-09-13T14:48:13.267 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.267 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.267 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.267 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068368+0000", 2024-09-13T14:48:13.267 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.014905295000000001 2024-09-13T14:48:13.267 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.268 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.268 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.268 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.081052+0000", 2024-09-13T14:48:13.268 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012684012999999999 2024-09-13T14:48:13.268 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.268 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.268 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.268 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.081061+0000", 2024-09-13T14:48:13.268 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8999999999999995e-06 2024-09-13T14:48:13.268 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.268 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.268 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.268 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082907+0000", 2024-09-13T14:48:13.269 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0018459310000001 2024-09-13T14:48:13.269 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.269 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.269 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.269 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082914+0000", 2024-09-13T14:48:13.269 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8520000000000004e-06 2024-09-13T14:48:13.269 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.269 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.269 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.269 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.200133+0000", 2024-09-13T14:48:13.269 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.117218641 2024-09-13T14:48:13.269 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.269 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.270 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.270 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.200205+0000", 2024-09-13T14:48:13.270 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2381999999999999e-05 2024-09-13T14:48:13.270 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.270 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.270 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.270 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.270 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.270 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:136895 6.2 6:487a132d:::benchmark_data_smithi096_167087_object19667:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-09-13T14:48:13.270 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:02.068487+0000", 2024-09-13T14:48:13.270 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 131.18995917300001, 2024-09-13T14:48:13.270 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1320098289999998, 2024-09-13T14:48:13.270 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.271 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:48:13.271 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.271 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.271 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.271 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 136895 2024-09-13T14:48:13.271 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.271 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.271 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.271 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.271 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068487+0000", 2024-09-13T14:48:13.271 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.271 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.271 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.271 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.272 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068487+0000", 2024-09-13T14:48:13.272 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.272 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.272 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.272 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.272 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068485+0000", 2024-09-13T14:48:13.272 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:48:13.272 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.272 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.272 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.272 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068492+0000", 2024-09-13T14:48:13.272 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1040000000000001e-06 2024-09-13T14:48:13.272 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.272 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.273 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.273 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068507+0000", 2024-09-13T14:48:13.273 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4707e-05 2024-09-13T14:48:13.273 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.273 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.273 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.273 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068512+0000", 2024-09-13T14:48:13.273 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7830000000000004e-06 2024-09-13T14:48:13.273 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.273 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.273 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.273 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068560+0000", 2024-09-13T14:48:13.273 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7141000000000001e-05 2024-09-13T14:48:13.274 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.274 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.274 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.274 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068680+0000", 2024-09-13T14:48:13.274 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000120183 2024-09-13T14:48:13.274 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.274 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.274 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:48:13.274 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.069244+0000", 2024-09-13T14:48:13.274 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00056373199999999995 2024-09-13T14:48:13.274 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.274 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.274 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.274 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.072489+0000", 2024-09-13T14:48:13.275 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0032459580000000002 2024-09-13T14:48:13.275 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.275 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.275 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.275 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.081086+0000", 2024-09-13T14:48:13.275 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0085960910000000001 2024-09-13T14:48:13.275 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.275 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.275 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.275 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.081092+0000", 2024-09-13T14:48:13.275 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7970000000000003e-06 2024-09-13T14:48:13.275 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.275 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.276 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.276 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082944+0000", 2024-09-13T14:48:13.276 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.001852076 2024-09-13T14:48:13.276 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.276 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.276 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.276 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082952+0000", 2024-09-13T14:48:13.276 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2660000000000004e-06 2024-09-13T14:48:13.276 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.276 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.276 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.276 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.200444+0000", 2024-09-13T14:48:13.276 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.117492039 2024-09-13T14:48:13.277 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.277 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.277 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.277 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.200497+0000", 2024-09-13T14:48:13.277 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2961999999999999e-05 2024-09-13T14:48:13.277 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.277 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.277 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.277 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.277 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.277 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:136900 6.2 6:4c459d5b:::benchmark_data_smithi096_167087_object19672:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-09-13T14:48:13.277 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:02.077300+0000", 2024-09-13T14:48:13.277 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 131.18114590600001, 2024-09-13T14:48:13.278 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1405926819999999, 2024-09-13T14:48:13.278 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.278 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.278 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.278 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.278 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.278 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 136900 2024-09-13T14:48:13.278 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.278 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.278 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.278 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.278 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.077300+0000", 2024-09-13T14:48:13.278 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.279 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.279 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.279 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.279 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.077300+0000", 2024-09-13T14:48:13.279 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.279 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.279 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.279 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.279 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.077298+0000", 2024-09-13T14:48:13.279 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:48:13.279 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.279 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.279 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.279 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.077305+0000", 2024-09-13T14:48:13.280 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9770000000000003e-06 2024-09-13T14:48:13.280 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.280 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.280 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.280 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.077317+0000", 2024-09-13T14:48:13.280 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.155e-05 2024-09-13T14:48:13.280 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.280 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.280 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.280 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.077323+0000", 2024-09-13T14:48:13.280 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9449999999999998e-06 2024-09-13T14:48:13.280 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.280 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.281 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.281 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.083184+0000", 2024-09-13T14:48:13.281 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0058613329999999998 2024-09-13T14:48:13.281 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.281 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.281 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.281 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.083193+0000", 2024-09-13T14:48:13.281 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1379999999999996e-06 2024-09-13T14:48:13.281 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.281 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.281 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.281 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082972+0000", 2024-09-13T14:48:13.281 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99977896499999996 2024-09-13T14:48:13.282 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.282 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.282 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.282 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082979+0000", 2024-09-13T14:48:13.282 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5819999999999998e-06 2024-09-13T14:48:13.282 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.282 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.282 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.282 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.200530+0000", 2024-09-13T14:48:13.282 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1175511789999999 2024-09-13T14:48:13.282 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.282 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.282 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.282 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.200642+0000", 2024-09-13T14:48:13.283 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000112376 2024-09-13T14:48:13.283 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.283 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.283 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T14:48:13.283 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.201279+0000", 2024-09-13T14:48:13.283 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00063697600000000004 2024-09-13T14:48:13.283 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.283 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.283 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.283 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.209711+0000", 2024-09-13T14:48:13.283 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0084313359999999993 2024-09-13T14:48:13.283 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.283 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.284 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.284 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.209770+0000", 2024-09-13T14:48:13.284 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8968000000000002e-05 2024-09-13T14:48:13.284 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.284 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.284 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.284 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.212282+0000", 2024-09-13T14:48:13.284 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0025126110000000001 2024-09-13T14:48:13.284 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.284 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.284 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.284 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.217709+0000", 2024-09-13T14:48:13.284 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.005426859 2024-09-13T14:48:13.284 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.285 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.285 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.285 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.217766+0000", 2024-09-13T14:48:13.285 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7182000000000002e-05 2024-09-13T14:48:13.285 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.285 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.285 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.285 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.217893+0000", 2024-09-13T14:48:13.285 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012656200000000001 2024-09-13T14:48:13.285 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.285 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.285 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.285 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.286 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.286 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:136908 6.e 6:740111d7:::benchmark_data_smithi096_167087_object19680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:48:13.286 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:02.112700+0000", 2024-09-13T14:48:13.286 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 131.145746323, 2024-09-13T14:48:13.286 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1197194330000002, 2024-09-13T14:48:13.286 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.286 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.286 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.286 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.286 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.286 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 136908 2024-09-13T14:48:13.286 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.286 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.286 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.287 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.287 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112700+0000", 2024-09-13T14:48:13.287 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.287 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.287 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.287 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.287 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112700+0000", 2024-09-13T14:48:13.287 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.287 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.287 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.287 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.287 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112699+0000", 2024-09-13T14:48:13.287 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:13.287 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.288 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.288 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.288 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112703+0000", 2024-09-13T14:48:13.288 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9089999999999999e-06 2024-09-13T14:48:13.288 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.288 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.288 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.288 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112714+0000", 2024-09-13T14:48:13.288 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1071e-05 2024-09-13T14:48:13.288 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.288 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.288 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.288 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112720+0000", 2024-09-13T14:48:13.289 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5470000000000004e-06 2024-09-13T14:48:13.289 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.289 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.289 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.289 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112782+0000", 2024-09-13T14:48:13.289 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2302000000000001e-05 2024-09-13T14:48:13.289 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.289 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.289 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.289 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112796+0000", 2024-09-13T14:48:13.289 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3495999999999999e-05 2024-09-13T14:48:13.289 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.289 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.289 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.290 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082520+0000", 2024-09-13T14:48:13.290 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.96972424199999996 2024-09-13T14:48:13.290 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.290 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.290 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.290 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082548+0000", 2024-09-13T14:48:13.290 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8402999999999999e-05 2024-09-13T14:48:13.290 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.290 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.290 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.290 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.218682+0000", 2024-09-13T14:48:13.291 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.136133767 2024-09-13T14:48:13.291 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.291 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.291 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.291 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:13.293 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.218802+0000", 2024-09-13T14:48:13.293 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119988 2024-09-13T14:48:13.293 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.293 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.293 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:48:13.293 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.219302+0000", 2024-09-13T14:48:13.293 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00050009499999999999 2024-09-13T14:48:13.293 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.293 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.293 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.293 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.224040+0000", 2024-09-13T14:48:13.293 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0047376420000000002 2024-09-13T14:48:13.294 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.294 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.294 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.294 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.229213+0000", 2024-09-13T14:48:13.294 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0051726039999999999 2024-09-13T14:48:13.294 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.294 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.294 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.294 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.231384+0000", 2024-09-13T14:48:13.294 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0021715990000000002 2024-09-13T14:48:13.294 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.294 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.294 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.294 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.232326+0000", 2024-09-13T14:48:13.295 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00094199799999999999 2024-09-13T14:48:13.295 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.295 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.295 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.295 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.232360+0000", 2024-09-13T14:48:13.295 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3411999999999998e-05 2024-09-13T14:48:13.295 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.295 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.295 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.295 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.232419+0000", 2024-09-13T14:48:13.295 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9555999999999997e-05 2024-09-13T14:48:13.295 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.296 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.296 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.296 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.296 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.296 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:136915 6.2 6:4a34972c:::benchmark_data_smithi096_167087_object19687:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:48:13.296 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:02.127704+0000", 2024-09-13T14:48:13.296 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 131.13074174900001, 2024-09-13T14:48:13.296 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.090359758, 2024-09-13T14:48:13.296 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.296 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.296 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.296 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.296 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.297 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 136915 2024-09-13T14:48:13.297 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.297 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.297 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.297 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.297 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127704+0000", 2024-09-13T14:48:13.297 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.297 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.297 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.297 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.297 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127704+0000", 2024-09-13T14:48:13.297 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.297 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.297 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.298 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.298 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127703+0000", 2024-09-13T14:48:13.298 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:13.298 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.298 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.298 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.298 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127708+0000", 2024-09-13T14:48:13.298 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3990000000000003e-06 2024-09-13T14:48:13.298 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.298 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.298 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.298 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127719+0000", 2024-09-13T14:48:13.298 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.065e-05 2024-09-13T14:48:13.299 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.299 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.299 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.299 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127726+0000", 2024-09-13T14:48:13.299 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8789999999999997e-06 2024-09-13T14:48:13.299 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.299 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.299 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.299 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127847+0000", 2024-09-13T14:48:13.299 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012154099999999999 2024-09-13T14:48:13.299 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.299 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.299 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.299 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127854+0000", 2024-09-13T14:48:13.300 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1570000000000003e-06 2024-09-13T14:48:13.300 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.300 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.300 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.300 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.083001+0000", 2024-09-13T14:48:13.300 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95514685899999996 2024-09-13T14:48:13.300 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.300 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.300 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.300 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.083008+0000", 2024-09-13T14:48:13.300 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4230000000000002e-06 2024-09-13T14:48:13.300 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.300 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.301 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.301 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.206475+0000", 2024-09-13T14:48:13.301 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.123467217 2024-09-13T14:48:13.301 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.301 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.301 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.301 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.206608+0000", 2024-09-13T14:48:13.301 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013340800000000001 2024-09-13T14:48:13.301 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.301 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.301 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T14:48:13.301 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.206693+0000", 2024-09-13T14:48:13.301 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4627000000000001e-05 2024-09-13T14:48:13.301 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.302 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.302 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.302 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.209759+0000", 2024-09-13T14:48:13.302 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00306646 2024-09-13T14:48:13.302 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.302 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.302 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.302 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.209777+0000", 2024-09-13T14:48:13.302 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7135000000000001e-05 2024-09-13T14:48:13.302 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.302 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.302 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.302 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.216584+0000", 2024-09-13T14:48:13.303 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0068075100000000001 2024-09-13T14:48:13.303 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.303 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.303 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.303 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.217946+0000", 2024-09-13T14:48:13.303 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001361791 2024-09-13T14:48:13.303 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.303 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.303 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.303 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.217981+0000", 2024-09-13T14:48:13.303 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4632999999999999e-05 2024-09-13T14:48:13.303 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.303 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.304 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.304 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.218064+0000", 2024-09-13T14:48:13.304 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3517999999999996e-05 2024-09-13T14:48:13.304 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.304 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.304 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.304 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.304 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.304 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:136925 6.2 6:47d3f3a2:::benchmark_data_smithi096_167087_object19697:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:48:13.304 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:02.144759+0000", 2024-09-13T14:48:13.304 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 131.113687489, 2024-09-13T14:48:13.304 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0783393989999999, 2024-09-13T14:48:13.304 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.304 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.305 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.305 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.305 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.305 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 136925 2024-09-13T14:48:13.305 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.305 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.305 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.305 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.305 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144759+0000", 2024-09-13T14:48:13.305 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.305 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.305 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.305 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.305 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144759+0000", 2024-09-13T14:48:13.306 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.306 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.306 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.306 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.306 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144757+0000", 2024-09-13T14:48:13.306 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:13.306 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.306 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.306 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.306 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144762+0000", 2024-09-13T14:48:13.306 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1549999999999998e-06 2024-09-13T14:48:13.306 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.306 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.306 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.306 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144771+0000", 2024-09-13T14:48:13.307 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1849999999999993e-06 2024-09-13T14:48:13.307 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.307 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.307 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.307 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144776+0000", 2024-09-13T14:48:13.307 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7419999999999997e-06 2024-09-13T14:48:13.307 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.307 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.307 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.307 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144820+0000", 2024-09-13T14:48:13.307 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4200999999999999e-05 2024-09-13T14:48:13.307 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.307 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.308 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.308 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144828+0000", 2024-09-13T14:48:13.308 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5989999999999996e-06 2024-09-13T14:48:13.308 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.308 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.308 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.308 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.083036+0000", 2024-09-13T14:48:13.308 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93820851599999999 2024-09-13T14:48:13.308 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.308 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.308 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.308 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.083043+0000", 2024-09-13T14:48:13.308 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6390000000000003e-06 2024-09-13T14:48:13.308 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.309 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.309 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.309 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.209811+0000", 2024-09-13T14:48:13.309 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1267674599999999 2024-09-13T14:48:13.309 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.309 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.309 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.309 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.209923+0000", 2024-09-13T14:48:13.310 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000112247 2024-09-13T14:48:13.310 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.310 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.310 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T14:48:13.310 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.210021+0000", 2024-09-13T14:48:13.310 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7788000000000006e-05 2024-09-13T14:48:13.310 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.310 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.311 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.311 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.216521+0000", 2024-09-13T14:48:13.311 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.006500151 2024-09-13T14:48:13.311 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.311 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.311 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.311 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.216531+0000", 2024-09-13T14:48:13.311 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0180000000000001e-05 2024-09-13T14:48:13.311 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.311 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.311 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.311 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.216620+0000", 2024-09-13T14:48:13.311 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.9054999999999994e-05 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.222936+0000", 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.006315668 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.222982+0000", 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6338999999999998e-05 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.223098+0000", 2024-09-13T14:48:13.313 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000115958 2024-09-13T14:48:13.313 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.313 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.313 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.313 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.313 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.313 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181248 6.17 6:eda41412:::benchmark_data_smithi096_167087_object64020:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:48:13.313 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.569149+0000", 2024-09-13T14:48:13.313 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 58.689296702, 2024-09-13T14:48:13.313 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.071124813, 2024-09-13T14:48:13.313 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.313 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.313 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.314 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.314 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.314 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181248 2024-09-13T14:48:13.314 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.314 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.314 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.314 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.314 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569149+0000", 2024-09-13T14:48:13.314 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.314 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.314 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.314 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.314 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569149+0000", 2024-09-13T14:48:13.314 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.315 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.315 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.315 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.315 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569148+0000", 2024-09-13T14:48:13.315 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:48:13.315 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.315 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.315 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.315 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569154+0000", 2024-09-13T14:48:13.315 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8190000000000003e-06 2024-09-13T14:48:13.315 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.315 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.315 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.315 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569178+0000", 2024-09-13T14:48:13.316 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4122e-05 2024-09-13T14:48:13.316 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.316 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.316 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.316 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569184+0000", 2024-09-13T14:48:13.316 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9100000000000002e-06 2024-09-13T14:48:13.316 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.316 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.316 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.316 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569235+0000", 2024-09-13T14:48:13.317 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0711000000000003e-05 2024-09-13T14:48:13.317 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.317 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.317 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.317 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569245+0000", 2024-09-13T14:48:13.317 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6770000000000007e-06 2024-09-13T14:48:13.317 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.317 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.317 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.317 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565701+0000", 2024-09-13T14:48:13.317 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99645585800000003 2024-09-13T14:48:13.317 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.317 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.318 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.318 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565711+0000", 2024-09-13T14:48:13.318 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9750000000000002e-06 2024-09-13T14:48:13.318 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.318 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.318 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.318 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.634822+0000", 2024-09-13T14:48:13.318 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0691110779999999 2024-09-13T14:48:13.318 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.318 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.318 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.318 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.634916+0000", 2024-09-13T14:48:13.318 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.3894999999999995e-05 2024-09-13T14:48:13.318 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.319 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.319 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:48:13.319 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.635198+0000", 2024-09-13T14:48:13.319 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00028242499999999999 2024-09-13T14:48:13.319 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.319 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.319 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.319 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640038+0000", 2024-09-13T14:48:13.319 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0048403659999999996 2024-09-13T14:48:13.319 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.319 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.319 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.319 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640075+0000", 2024-09-13T14:48:13.319 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6956000000000003e-05 2024-09-13T14:48:13.320 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.320 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.320 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.320 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640121+0000", 2024-09-13T14:48:13.320 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5466999999999997e-05 2024-09-13T14:48:13.320 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.320 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.320 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.320 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640168+0000", 2024-09-13T14:48:13.320 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7579e-05 2024-09-13T14:48:13.320 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.320 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.320 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.320 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640202+0000", 2024-09-13T14:48:13.321 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3290999999999997e-05 2024-09-13T14:48:13.321 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.321 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.321 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.321 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640274+0000", 2024-09-13T14:48:13.321 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2492999999999999e-05 2024-09-13T14:48:13.321 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.321 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.321 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.321 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.321 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.321 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181259 6.17 6:eec3e1cc:::benchmark_data_smithi096_167087_object64031:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:48:13.321 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.585855+0000", 2024-09-13T14:48:13.322 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 58.672591146000002, 2024-09-13T14:48:13.322 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0547445899999999, 2024-09-13T14:48:13.322 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.322 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.322 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.322 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.322 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.322 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181259 2024-09-13T14:48:13.322 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.322 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.322 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.322 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.322 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585855+0000", 2024-09-13T14:48:13.322 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.323 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.323 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.323 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.323 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585855+0000", 2024-09-13T14:48:13.323 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.323 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.323 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.323 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.323 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585853+0000", 2024-09-13T14:48:13.323 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:48:13.323 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.323 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.323 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.323 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585861+0000", 2024-09-13T14:48:13.324 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0360000000000006e-06 2024-09-13T14:48:13.324 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.324 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.324 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.324 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585878+0000", 2024-09-13T14:48:13.324 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6558e-05 2024-09-13T14:48:13.324 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.324 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.324 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.324 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585884+0000", 2024-09-13T14:48:13.324 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6490000000000002e-06 2024-09-13T14:48:13.324 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.324 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.324 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.325 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585953+0000", 2024-09-13T14:48:13.325 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8206000000000004e-05 2024-09-13T14:48:13.325 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.325 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.325 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.325 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585965+0000", 2024-09-13T14:48:13.325 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2094e-05 2024-09-13T14:48:13.325 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.325 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.325 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.325 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565736+0000", 2024-09-13T14:48:13.325 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97977119700000004 2024-09-13T14:48:13.325 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.326 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.326 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.326 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565744+0000", 2024-09-13T14:48:13.326 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8150000000000006e-06 2024-09-13T14:48:13.326 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.326 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.326 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.326 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.636443+0000", 2024-09-13T14:48:13.326 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.070699402 2024-09-13T14:48:13.326 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.326 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.326 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.326 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.636515+0000", 2024-09-13T14:48:13.326 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1812999999999998e-05 2024-09-13T14:48:13.327 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.327 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.327 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:48:13.327 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.636650+0000", 2024-09-13T14:48:13.327 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013464799999999999 2024-09-13T14:48:13.327 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.327 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.327 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.327 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640130+0000", 2024-09-13T14:48:13.327 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0034803210000000002 2024-09-13T14:48:13.327 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.327 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.327 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.327 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640331+0000", 2024-09-13T14:48:13.328 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00020073899999999999 2024-09-13T14:48:13.328 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.328 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.328 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.328 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640378+0000", 2024-09-13T14:48:13.328 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7766999999999999e-05 2024-09-13T14:48:13.328 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.328 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.328 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.328 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640429+0000", 2024-09-13T14:48:13.328 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0692999999999997e-05 2024-09-13T14:48:13.328 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.328 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.328 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.329 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640482+0000", 2024-09-13T14:48:13.329 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3282999999999998e-05 2024-09-13T14:48:13.329 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.329 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.329 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.329 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640599+0000", 2024-09-13T14:48:13.329 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011709099999999999 2024-09-13T14:48:13.329 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.329 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.329 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.329 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.329 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.329 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181267 6.17 6:ef08088c:::benchmark_data_smithi096_167087_object64039:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:48:13.329 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.591094+0000", 2024-09-13T14:48:13.330 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 58.667352057000002, 2024-09-13T14:48:13.330 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.052348388, 2024-09-13T14:48:13.330 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.330 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.330 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.330 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.330 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.330 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181267 2024-09-13T14:48:13.330 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.330 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.330 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.330 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.330 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591094+0000", 2024-09-13T14:48:13.330 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.331 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.331 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.331 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.331 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591094+0000", 2024-09-13T14:48:13.331 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.331 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.331 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.331 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.331 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591092+0000", 2024-09-13T14:48:13.331 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:48:13.331 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.331 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.331 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.331 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591099+0000", 2024-09-13T14:48:13.332 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9190000000000002e-06 2024-09-13T14:48:13.332 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.332 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.332 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.332 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591113+0000", 2024-09-13T14:48:13.332 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4099e-05 2024-09-13T14:48:13.332 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.332 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.332 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.332 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591119+0000", 2024-09-13T14:48:13.332 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5849999999999999e-06 2024-09-13T14:48:13.332 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.332 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.333 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.333 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591182+0000", 2024-09-13T14:48:13.333 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3466999999999994e-05 2024-09-13T14:48:13.333 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.333 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.333 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.333 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591196+0000", 2024-09-13T14:48:13.333 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4435e-05 2024-09-13T14:48:13.333 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.333 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.333 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.333 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565767+0000", 2024-09-13T14:48:13.333 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97457061199999995 2024-09-13T14:48:13.334 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.334 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.334 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.334 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565775+0000", 2024-09-13T14:48:13.334 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7409999999999992e-06 2024-09-13T14:48:13.334 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.334 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.334 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.334 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.637678+0000", 2024-09-13T14:48:13.334 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.071902879 2024-09-13T14:48:13.334 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.334 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.334 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.334 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.637765+0000", 2024-09-13T14:48:13.335 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.6873000000000007e-05 2024-09-13T14:48:13.335 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.335 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.335 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:48:13.335 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.637918+0000", 2024-09-13T14:48:13.335 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015319899999999999 2024-09-13T14:48:13.335 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.335 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.335 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.335 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640137+0000", 2024-09-13T14:48:13.335 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0022196059999999998 2024-09-13T14:48:13.335 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.336 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.336 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.336 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640638+0000", 2024-09-13T14:48:13.336 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00050057400000000005 2024-09-13T14:48:13.336 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.336 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.336 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.336 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640683+0000", 2024-09-13T14:48:13.336 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4583999999999999e-05 2024-09-13T14:48:13.336 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.336 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.336 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.336 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643279+0000", 2024-09-13T14:48:13.337 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0025964400000000002 2024-09-13T14:48:13.337 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.337 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.337 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.337 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643331+0000", 2024-09-13T14:48:13.337 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1684000000000002e-05 2024-09-13T14:48:13.337 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.337 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.337 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.337 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643442+0000", 2024-09-13T14:48:13.337 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000111742 2024-09-13T14:48:13.337 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.337 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.338 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.338 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.338 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.338 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181270 6.17 6:ea4ae503:::benchmark_data_smithi096_167087_object64042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:48:13.338 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.594659+0000", 2024-09-13T14:48:13.338 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 58.663786868999999, 2024-09-13T14:48:13.338 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.048986019, 2024-09-13T14:48:13.338 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.338 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.338 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.338 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.338 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.338 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181270 2024-09-13T14:48:13.339 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.339 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.339 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.339 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.339 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594659+0000", 2024-09-13T14:48:13.339 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.339 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.339 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.339 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.339 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594659+0000", 2024-09-13T14:48:13.339 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.339 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.339 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.339 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.340 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594657+0000", 2024-09-13T14:48:13.340 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:48:13.340 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.340 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.340 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594663+0000", 2024-09-13T14:48:13.340 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0480000000000004e-06 2024-09-13T14:48:13.340 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.340 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.340 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594677+0000", 2024-09-13T14:48:13.340 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3688e-05 2024-09-13T14:48:13.340 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.341 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.341 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594684+0000", 2024-09-13T14:48:13.341 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3890000000000002e-06 2024-09-13T14:48:13.341 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.341 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.341 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.341 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594723+0000", 2024-09-13T14:48:13.341 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8982e-05 2024-09-13T14:48:13.341 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.341 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.341 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.341 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594731+0000", 2024-09-13T14:48:13.341 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1380000000000007e-06 2024-09-13T14:48:13.342 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.342 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.342 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.342 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565798+0000", 2024-09-13T14:48:13.342 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97106689300000004 2024-09-13T14:48:13.342 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.343 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.343 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.343 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565805+0000", 2024-09-13T14:48:13.343 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8210000000000002e-06 2024-09-13T14:48:13.343 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.343 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.343 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.343 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.638755+0000", 2024-09-13T14:48:13.343 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0729500999999999 2024-09-13T14:48:13.343 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.343 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.343 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.343 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.638832+0000", 2024-09-13T14:48:13.343 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6369e-05 2024-09-13T14:48:13.344 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.344 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.344 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:48:13.344 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.638985+0000", 2024-09-13T14:48:13.344 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015271699999999999 2024-09-13T14:48:13.344 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.344 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.344 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.344 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640145+0000", 2024-09-13T14:48:13.344 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001160269 2024-09-13T14:48:13.344 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.344 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.344 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.344 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640711+0000", 2024-09-13T14:48:13.345 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00056574800000000003 2024-09-13T14:48:13.345 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.345 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.345 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.345 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640744+0000", 2024-09-13T14:48:13.345 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3510000000000003e-05 2024-09-13T14:48:13.346 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.346 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.346 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.346 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643496+0000", 2024-09-13T14:48:13.346 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0027522599999999999 2024-09-13T14:48:13.346 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.346 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.346 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.346 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643541+0000", 2024-09-13T14:48:13.346 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5124999999999999e-05 2024-09-13T14:48:13.346 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.346 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.346 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.346 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643645+0000", 2024-09-13T14:48:13.347 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000103733 2024-09-13T14:48:13.347 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.347 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.347 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.347 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.347 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.347 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181272 6.17 6:ef450286:::benchmark_data_smithi096_167087_object64044:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:48:13.347 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.595486+0000", 2024-09-13T14:48:13.347 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 58.662960275000003, 2024-09-13T14:48:13.347 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0541925989999998, 2024-09-13T14:48:13.347 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.347 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.347 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.347 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.348 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.348 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181272 2024-09-13T14:48:13.348 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.348 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.348 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.348 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.348 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595486+0000", 2024-09-13T14:48:13.348 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.348 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.348 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.348 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.348 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595486+0000", 2024-09-13T14:48:13.348 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.349 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.349 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.349 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.349 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595485+0000", 2024-09-13T14:48:13.349 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:48:13.349 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.349 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.349 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.349 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595489+0000", 2024-09-13T14:48:13.349 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0160000000000002e-06 2024-09-13T14:48:13.349 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.349 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.349 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.349 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595499+0000", 2024-09-13T14:48:13.350 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.006e-05 2024-09-13T14:48:13.350 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.350 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.350 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.350 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595504+0000", 2024-09-13T14:48:13.350 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6689999999999997e-06 2024-09-13T14:48:13.350 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.350 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.350 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.350 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595539+0000", 2024-09-13T14:48:13.350 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5673000000000002e-05 2024-09-13T14:48:13.350 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.350 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.350 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.351 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595547+0000", 2024-09-13T14:48:13.351 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.515e-06 2024-09-13T14:48:13.351 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.351 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.351 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.351 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565827+0000", 2024-09-13T14:48:13.351 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97028026899999997 2024-09-13T14:48:13.351 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.351 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.351 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.351 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565834+0000", 2024-09-13T14:48:13.351 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7440000000000001e-06 2024-09-13T14:48:13.351 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.352 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.352 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.352 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640765+0000", 2024-09-13T14:48:13.352 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.074930806 2024-09-13T14:48:13.352 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.352 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.352 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.352 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640833+0000", 2024-09-13T14:48:13.352 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7933000000000002e-05 2024-09-13T14:48:13.352 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.352 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.352 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:48:13.352 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640974+0000", 2024-09-13T14:48:13.353 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014157500000000001 2024-09-13T14:48:13.353 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.353 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.353 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.353 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643458+0000", 2024-09-13T14:48:13.353 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0024839609999999998 2024-09-13T14:48:13.353 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.353 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.353 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.353 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.645516+0000", 2024-09-13T14:48:13.353 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0020581850000000001 2024-09-13T14:48:13.353 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.353 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.353 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.354 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649470+0000", 2024-09-13T14:48:13.354 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0039536559999999998 2024-09-13T14:48:13.354 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.354 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.354 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.354 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649506+0000", 2024-09-13T14:48:13.354 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5920999999999997e-05 2024-09-13T14:48:13.354 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.354 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.354 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.354 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649554+0000", 2024-09-13T14:48:13.354 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7545e-05 2024-09-13T14:48:13.354 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.354 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.355 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.355 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649678+0000", 2024-09-13T14:48:13.355 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012479599999999999 2024-09-13T14:48:13.355 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.355 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.355 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.355 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.355 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.355 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181418 6.17 6:e9041057:::benchmark_data_smithi096_167087_object64190:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:48:13.355 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.860748+0000", 2024-09-13T14:48:13.355 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 58.397697772999997, 2024-09-13T14:48:13.355 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7891968889999998, 2024-09-13T14:48:13.355 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.355 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.356 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.356 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.356 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.356 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181418 2024-09-13T14:48:13.356 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.356 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.356 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.356 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.356 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860748+0000", 2024-09-13T14:48:13.356 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.356 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.356 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.356 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.356 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860748+0000", 2024-09-13T14:48:13.357 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.357 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.357 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.357 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.357 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860747+0000", 2024-09-13T14:48:13.357 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:13.357 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.357 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.357 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.357 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860755+0000", 2024-09-13T14:48:13.357 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5299999999999996e-06 2024-09-13T14:48:13.357 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.357 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.357 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.358 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860767+0000", 2024-09-13T14:48:13.358 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1935e-05 2024-09-13T14:48:13.358 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.358 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.358 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.358 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860772+0000", 2024-09-13T14:48:13.358 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0459999999999996e-06 2024-09-13T14:48:13.358 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.358 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.358 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.358 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860813+0000", 2024-09-13T14:48:13.358 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0963000000000001e-05 2024-09-13T14:48:13.358 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.359 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.359 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.359 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860821+0000", 2024-09-13T14:48:13.359 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.391e-06 2024-09-13T14:48:13.359 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.359 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.359 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.359 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565856+0000", 2024-09-13T14:48:13.359 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.70503548599999999 2024-09-13T14:48:13.359 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.359 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.359 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.359 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565864+0000", 2024-09-13T14:48:13.359 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5399999999999998e-06 2024-09-13T14:48:13.360 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.360 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.360 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.360 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.641841+0000", 2024-09-13T14:48:13.360 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.07597739 2024-09-13T14:48:13.360 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.360 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.360 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.360 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.641962+0000", 2024-09-13T14:48:13.360 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012125600000000001 2024-09-13T14:48:13.360 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.360 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.360 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:48:13.361 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.642177+0000", 2024-09-13T14:48:13.361 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000214916 2024-09-13T14:48:13.361 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.361 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.361 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.361 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643655+0000", 2024-09-13T14:48:13.361 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0014778770000000001 2024-09-13T14:48:13.361 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.361 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.361 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.361 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.645548+0000", 2024-09-13T14:48:13.361 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001892864 2024-09-13T14:48:13.361 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.362 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.362 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.362 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649754+0000", 2024-09-13T14:48:13.362 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0042061920000000001 2024-09-13T14:48:13.362 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.362 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.362 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.362 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649790+0000", 2024-09-13T14:48:13.362 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5929000000000001e-05 2024-09-13T14:48:13.362 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.362 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.362 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.362 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649839+0000", 2024-09-13T14:48:13.362 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8649999999999997e-05 2024-09-13T14:48:13.363 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.363 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.363 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.363 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649945+0000", 2024-09-13T14:48:13.363 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000106252 2024-09-13T14:48:13.363 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.363 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.363 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.363 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.363 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.363 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181419 6.17 6:ecfb7c25:::benchmark_data_smithi096_167087_object64191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:48:13.363 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.861452+0000", 2024-09-13T14:48:13.363 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 58.396993799999997, 2024-09-13T14:48:13.364 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7981438779999999, 2024-09-13T14:48:13.364 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.364 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.364 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.364 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.364 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.364 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181419 2024-09-13T14:48:13.364 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.364 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.364 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.364 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.364 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861452+0000", 2024-09-13T14:48:13.364 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.365 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.365 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.365 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.365 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861452+0000", 2024-09-13T14:48:13.365 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.365 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.365 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.365 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.365 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861451+0000", 2024-09-13T14:48:13.365 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:48:13.365 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.365 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.365 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.366 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861459+0000", 2024-09-13T14:48:13.366 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.379e-06 2024-09-13T14:48:13.366 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.366 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.366 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.366 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861470+0000", 2024-09-13T14:48:13.366 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1311000000000001e-05 2024-09-13T14:48:13.366 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.366 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.366 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.366 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861475+0000", 2024-09-13T14:48:13.366 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1510000000000003e-06 2024-09-13T14:48:13.366 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.366 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.367 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.367 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861500+0000", 2024-09-13T14:48:13.367 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.423e-05 2024-09-13T14:48:13.367 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.367 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.367 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.367 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861505+0000", 2024-09-13T14:48:13.367 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9930000000000003e-06 2024-09-13T14:48:13.367 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.367 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.367 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.367 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565910+0000", 2024-09-13T14:48:13.367 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.70440582600000001 2024-09-13T14:48:13.367 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.368 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.368 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.368 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565918+0000", 2024-09-13T14:48:13.368 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9370000000000004e-06 2024-09-13T14:48:13.368 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.368 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.368 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.368 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643685+0000", 2024-09-13T14:48:13.368 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0777663710000001 2024-09-13T14:48:13.368 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.368 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.368 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.368 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643798+0000", 2024-09-13T14:48:13.369 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000112866 2024-09-13T14:48:13.369 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.369 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.369 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:48:13.369 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.644019+0000", 2024-09-13T14:48:13.369 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00022168000000000001 2024-09-13T14:48:13.369 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.369 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.369 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.369 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649720+0000", 2024-09-13T14:48:13.369 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0057008689999999999 2024-09-13T14:48:13.369 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.369 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.370 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.370 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649982+0000", 2024-09-13T14:48:13.370 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00026224400000000002 2024-09-13T14:48:13.370 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.370 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.370 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.370 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.650014+0000", 2024-09-13T14:48:13.370 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2039999999999998e-05 2024-09-13T14:48:13.370 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.370 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.370 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.370 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.659434+0000", 2024-09-13T14:48:13.370 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0094201530000000006 2024-09-13T14:48:13.370 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.370 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.371 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.371 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.659489+0000", 2024-09-13T14:48:13.371 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4169999999999998e-05 2024-09-13T14:48:13.371 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.371 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.371 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.371 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.659596+0000", 2024-09-13T14:48:13.371 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000107458 2024-09-13T14:48:13.371 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.371 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.371 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.371 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.371 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.372 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181456 6.17 6:eccfdfe3:::benchmark_data_smithi096_167087_object64228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:48:13.372 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.892981+0000", 2024-09-13T14:48:13.372 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 58.365464541000001, 2024-09-13T14:48:13.372 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.766763437, 2024-09-13T14:48:13.372 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.372 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.372 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.372 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.372 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.372 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181456 2024-09-13T14:48:13.372 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.372 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.372 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.372 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.373 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.892981+0000", 2024-09-13T14:48:13.373 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.373 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.373 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.373 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.373 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.892981+0000", 2024-09-13T14:48:13.373 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.373 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.373 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.373 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.373 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.892980+0000", 2024-09-13T14:48:13.373 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:13.373 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.373 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.374 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.374 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.892985+0000", 2024-09-13T14:48:13.374 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6650000000000002e-06 2024-09-13T14:48:13.374 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.374 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.374 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.374 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.892994+0000", 2024-09-13T14:48:13.374 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8140000000000007e-06 2024-09-13T14:48:13.374 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.374 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.374 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.374 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.892999+0000", 2024-09-13T14:48:13.374 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8369999999999996e-06 2024-09-13T14:48:13.374 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.375 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.375 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.375 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.893047+0000", 2024-09-13T14:48:13.375 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8371999999999997e-05 2024-09-13T14:48:13.375 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.375 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.375 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.375 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.893057+0000", 2024-09-13T14:48:13.375 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9879999999999999e-06 2024-09-13T14:48:13.375 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.375 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.375 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.375 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565958+0000", 2024-09-13T14:48:13.375 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.67290108000000004 2024-09-13T14:48:13.376 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.376 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.376 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.376 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565966+0000", 2024-09-13T14:48:13.376 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1079999999999993e-06 2024-09-13T14:48:13.376 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.376 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.376 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.376 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.645611+0000", 2024-09-13T14:48:13.376 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.079645027 2024-09-13T14:48:13.376 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.376 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.376 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.376 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.645720+0000", 2024-09-13T14:48:13.377 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000108997 2024-09-13T14:48:13.377 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.377 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.377 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:48:13.377 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.645984+0000", 2024-09-13T14:48:13.377 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00026369700000000002 2024-09-13T14:48:13.377 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.377 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.377 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.377 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.650045+0000", 2024-09-13T14:48:13.377 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0040617220000000002 2024-09-13T14:48:13.377 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.377 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.377 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.378 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.650092+0000", 2024-09-13T14:48:13.378 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6533999999999999e-05 2024-09-13T14:48:13.378 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.378 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.378 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.378 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.657060+0000", 2024-09-13T14:48:13.378 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0069675910000000004 2024-09-13T14:48:13.378 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.378 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.378 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.379 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:13 smithi096 ceph-mon[98901]: pgmap v609: 109 pgs: 109 active+clean; 10 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 932 op/s 2024-09-13T14:48:13.380 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:13 smithi096 ceph-mon[102928]: pgmap v609: 109 pgs: 109 active+clean; 10 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 932 op/s 2024-09-13T14:48:13.380 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.659605+0000", 2024-09-13T14:48:13.380 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0025451279999999998 2024-09-13T14:48:13.380 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.380 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.380 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.380 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.659644+0000", 2024-09-13T14:48:13.381 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9165000000000002e-05 2024-09-13T14:48:13.381 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.381 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.381 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.381 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.659745+0000", 2024-09-13T14:48:13.381 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000101017 2024-09-13T14:48:13.381 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.381 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.381 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.381 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.381 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.381 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181475 6.17 6:e9f35c4f:::benchmark_data_smithi096_167087_object64247:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:48:13.381 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.910141+0000", 2024-09-13T14:48:13.382 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 58.348305490000001, 2024-09-13T14:48:13.382 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.749748391, 2024-09-13T14:48:13.382 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.382 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.382 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.382 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.382 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.382 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181475 2024-09-13T14:48:13.382 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.382 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.382 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.382 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.382 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.910141+0000", 2024-09-13T14:48:13.382 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.383 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.383 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.383 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.383 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.910141+0000", 2024-09-13T14:48:13.383 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.383 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.383 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.383 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.383 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.910139+0000", 2024-09-13T14:48:13.383 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:13.383 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.383 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.383 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.384 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.910145+0000", 2024-09-13T14:48:13.384 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8030000000000002e-06 2024-09-13T14:48:13.384 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.384 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.384 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.384 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.910157+0000", 2024-09-13T14:48:13.384 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1863e-05 2024-09-13T14:48:13.384 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.384 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.384 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.384 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.910161+0000", 2024-09-13T14:48:13.384 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5249999999999999e-06 2024-09-13T14:48:13.384 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.384 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.385 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.385 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.910202+0000", 2024-09-13T14:48:13.385 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0847999999999999e-05 2024-09-13T14:48:13.385 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.385 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.385 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.385 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.910210+0000", 2024-09-13T14:48:13.385 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7589999999999997e-06 2024-09-13T14:48:13.385 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.385 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.385 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.385 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565990+0000", 2024-09-13T14:48:13.385 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.65577958700000005 2024-09-13T14:48:13.386 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.386 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.386 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.386 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565997+0000", 2024-09-13T14:48:13.386 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6539999999999999e-06 2024-09-13T14:48:13.386 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.386 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.386 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.386 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.647717+0000", 2024-09-13T14:48:13.386 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0817198889999999 2024-09-13T14:48:13.386 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.386 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.386 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.386 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.647854+0000", 2024-09-13T14:48:13.387 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013676000000000001 2024-09-13T14:48:13.387 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.387 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.387 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:48:13.387 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.648140+0000", 2024-09-13T14:48:13.387 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00028588099999999998 2024-09-13T14:48:13.387 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.387 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.387 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.387 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.657091+0000", 2024-09-13T14:48:13.387 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0089514429999999999 2024-09-13T14:48:13.387 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.387 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.387 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.388 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.657120+0000", 2024-09-13T14:48:13.388 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8238e-05 2024-09-13T14:48:13.388 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.388 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.388 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.388 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.657147+0000", 2024-09-13T14:48:13.388 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7480000000000001e-05 2024-09-13T14:48:13.388 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.388 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.388 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.388 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.659758+0000", 2024-09-13T14:48:13.388 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0026108720000000002 2024-09-13T14:48:13.388 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.389 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.389 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.389 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.659796+0000", 2024-09-13T14:48:13.389 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8497000000000001e-05 2024-09-13T14:48:13.389 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.389 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.389 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.389 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.659889+0000", 2024-09-13T14:48:13.389 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.2571999999999994e-05 2024-09-13T14:48:13.389 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.389 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.389 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.389 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.389 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.390 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181511 6.17 6:ebedf956:::benchmark_data_smithi096_167087_object64283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:48:13.390 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.940611+0000", 2024-09-13T14:48:13.390 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 58.317835524000003, 2024-09-13T14:48:13.390 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7194262300000001, 2024-09-13T14:48:13.390 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.390 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.390 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.390 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.390 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.390 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181511 2024-09-13T14:48:13.390 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.390 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.390 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.391 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.391 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.940611+0000", 2024-09-13T14:48:13.391 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.391 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.391 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.391 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.391 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.940611+0000", 2024-09-13T14:48:13.391 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.391 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.391 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.391 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.391 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.940610+0000", 2024-09-13T14:48:13.391 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:48:13.391 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.392 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.392 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.392 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.940614+0000", 2024-09-13T14:48:13.392 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7119999999999999e-06 2024-09-13T14:48:13.392 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.392 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.392 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.392 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.940625+0000", 2024-09-13T14:48:13.392 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0851e-05 2024-09-13T14:48:13.392 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.392 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.392 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.392 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.940631+0000", 2024-09-13T14:48:13.393 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0650000000000004e-06 2024-09-13T14:48:13.393 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.393 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.393 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.393 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.940685+0000", 2024-09-13T14:48:13.393 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3257000000000002e-05 2024-09-13T14:48:13.393 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.393 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.393 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.393 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.940696+0000", 2024-09-13T14:48:13.393 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1476e-05 2024-09-13T14:48:13.393 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.393 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.393 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.394 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.566020+0000", 2024-09-13T14:48:13.394 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.62532397200000001 2024-09-13T14:48:13.394 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.394 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.394 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.394 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.566028+0000", 2024-09-13T14:48:13.394 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.4989999999999997e-06 2024-09-13T14:48:13.394 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.394 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.394 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.394 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.650114+0000", 2024-09-13T14:48:13.394 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.084086452 2024-09-13T14:48:13.394 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.395 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.395 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.395 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.650221+0000", 2024-09-13T14:48:13.395 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010751000000000001 2024-09-13T14:48:13.395 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.395 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.395 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:48:13.395 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.650504+0000", 2024-09-13T14:48:13.395 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00028206200000000002 2024-09-13T14:48:13.395 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.395 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.395 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.395 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.657174+0000", 2024-09-13T14:48:13.396 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0066708590000000003 2024-09-13T14:48:13.396 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.396 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.396 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.396 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.657204+0000", 2024-09-13T14:48:13.396 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9581e-05 2024-09-13T14:48:13.396 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.396 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.396 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.396 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.657348+0000", 2024-09-13T14:48:13.396 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000144349 2024-09-13T14:48:13.396 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.396 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.396 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.397 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.659900+0000", 2024-09-13T14:48:13.397 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0025515360000000001 2024-09-13T14:48:13.397 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.397 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.397 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.397 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.659944+0000", 2024-09-13T14:48:13.397 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3650999999999999e-05 2024-09-13T14:48:13.397 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.397 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.397 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.397 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.660037+0000", 2024-09-13T14:48:13.397 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.3228999999999998e-05 2024-09-13T14:48:13.397 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.398 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.398 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.398 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.398 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.398 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181568 6.17 6:e9568d99:::benchmark_data_smithi096_167087_object64340:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:48:13.398 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:15.088994+0000", 2024-09-13T14:48:13.398 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 58.169452139000001, 2024-09-13T14:48:13.398 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.571387667, 2024-09-13T14:48:13.398 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.398 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.398 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.398 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.398 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.398 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181568 2024-09-13T14:48:13.399 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.399 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.399 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.399 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.399 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.088994+0000", 2024-09-13T14:48:13.399 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.399 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.399 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.399 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.399 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.088994+0000", 2024-09-13T14:48:13.399 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.399 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.399 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.400 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.400 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.088992+0000", 2024-09-13T14:48:13.400 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:13.400 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.400 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.400 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.400 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.088997+0000", 2024-09-13T14:48:13.400 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0520000000000001e-06 2024-09-13T14:48:13.400 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.400 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.400 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.400 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.089010+0000", 2024-09-13T14:48:13.400 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3021e-05 2024-09-13T14:48:13.401 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.401 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.401 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.401 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.089016+0000", 2024-09-13T14:48:13.401 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4469999999999997e-06 2024-09-13T14:48:13.401 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.401 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.401 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.401 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.089072+0000", 2024-09-13T14:48:13.401 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6521999999999999e-05 2024-09-13T14:48:13.401 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.401 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.401 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.401 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.089085+0000", 2024-09-13T14:48:13.402 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.287e-05 2024-09-13T14:48:13.402 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.402 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.402 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.402 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.566051+0000", 2024-09-13T14:48:13.402 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.47696532400000002 2024-09-13T14:48:13.402 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.402 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.402 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.402 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.566058+0000", 2024-09-13T14:48:13.402 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6210000000000004e-06 2024-09-13T14:48:13.402 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.402 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.402 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.403 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.654958+0000", 2024-09-13T14:48:13.403 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.088899549 2024-09-13T14:48:13.403 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.403 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.403 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.403 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.655063+0000", 2024-09-13T14:48:13.403 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010555100000000001 2024-09-13T14:48:13.403 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.403 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.403 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:48:13.403 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.655339+0000", 2024-09-13T14:48:13.403 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00027563300000000001 2024-09-13T14:48:13.403 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.404 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.404 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.404 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.660049+0000", 2024-09-13T14:48:13.404 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0047095640000000003 2024-09-13T14:48:13.404 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.404 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.404 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.404 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.660111+0000", 2024-09-13T14:48:13.404 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2271000000000001e-05 2024-09-13T14:48:13.404 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.404 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.404 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.404 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.660179+0000", 2024-09-13T14:48:13.404 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8634999999999996e-05 2024-09-13T14:48:13.405 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.405 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.405 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.405 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.660213+0000", 2024-09-13T14:48:13.405 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3914000000000001e-05 2024-09-13T14:48:13.405 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.405 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.405 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.405 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.660271+0000", 2024-09-13T14:48:13.405 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7175000000000001e-05 2024-09-13T14:48:13.405 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.405 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.405 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.406 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.660382+0000", 2024-09-13T14:48:13.406 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011105800000000001 2024-09-13T14:48:13.406 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.406 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.406 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.406 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.406 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.406 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181601 6.17 6:e9aaf03f:::benchmark_data_smithi096_167087_object64373:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:48:13.406 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:15.126359+0000", 2024-09-13T14:48:13.406 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 58.132086678, 2024-09-13T14:48:13.406 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.539280491, 2024-09-13T14:48:13.406 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.406 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.406 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.407 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.407 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.407 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181601 2024-09-13T14:48:13.407 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.407 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.407 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.407 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.407 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.126359+0000", 2024-09-13T14:48:13.407 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.407 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.407 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.407 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.407 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.126359+0000", 2024-09-13T14:48:13.407 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.408 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.408 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.408 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.408 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.126358+0000", 2024-09-13T14:48:13.408 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:13.408 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.408 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.408 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.408 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.126367+0000", 2024-09-13T14:48:13.408 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5470000000000002e-06 2024-09-13T14:48:13.408 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.408 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.408 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.409 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.126381+0000", 2024-09-13T14:48:13.409 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4154999999999999e-05 2024-09-13T14:48:13.409 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.409 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.409 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.409 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.126388+0000", 2024-09-13T14:48:13.409 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3400000000000003e-06 2024-09-13T14:48:13.409 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.409 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.409 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.409 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.126445+0000", 2024-09-13T14:48:13.409 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6820000000000001e-05 2024-09-13T14:48:13.410 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.410 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.410 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.410 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.126456+0000", 2024-09-13T14:48:13.410 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1812e-05 2024-09-13T14:48:13.410 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.410 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.410 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.410 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.566081+0000", 2024-09-13T14:48:13.410 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.43962478999999999 2024-09-13T14:48:13.410 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.410 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.410 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.410 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.566089+0000", 2024-09-13T14:48:13.411 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2269999999999993e-06 2024-09-13T14:48:13.411 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.411 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.411 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.411 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.657372+0000", 2024-09-13T14:48:13.411 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0912829239999999 2024-09-13T14:48:13.411 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.411 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.411 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.411 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.657500+0000", 2024-09-13T14:48:13.411 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012792399999999999 2024-09-13T14:48:13.411 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.411 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.412 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:48:13.412 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.657837+0000", 2024-09-13T14:48:13.412 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00033657799999999997 2024-09-13T14:48:13.412 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.412 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.412 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.412 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.660497+0000", 2024-09-13T14:48:13.412 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0026604580000000001 2024-09-13T14:48:13.412 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.412 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.412 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.412 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.660537+0000", 2024-09-13T14:48:13.412 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9641000000000002e-05 2024-09-13T14:48:13.412 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.413 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.413 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.413 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.660571+0000", 2024-09-13T14:48:13.413 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3578999999999998e-05 2024-09-13T14:48:13.413 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.413 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.413 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.413 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.665497+0000", 2024-09-13T14:48:13.413 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.004926522 2024-09-13T14:48:13.413 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.413 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.413 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.413 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.665529+0000", 2024-09-13T14:48:13.414 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1639e-05 2024-09-13T14:48:13.414 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.414 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.414 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.414 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.665640+0000", 2024-09-13T14:48:13.414 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011118199999999999 2024-09-13T14:48:13.414 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.414 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.414 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.414 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.414 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.414 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181667 6.17 6:ec0e8660:::benchmark_data_smithi096_167087_object64439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:48:13.414 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:15.205022+0000", 2024-09-13T14:48:13.415 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 58.053424016000001, 2024-09-13T14:48:13.415 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4761209690000001, 2024-09-13T14:48:13.415 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.415 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.415 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.415 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.415 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.415 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181667 2024-09-13T14:48:13.415 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.415 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.415 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.415 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.415 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.205022+0000", 2024-09-13T14:48:13.415 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.416 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.416 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.416 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.416 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.205022+0000", 2024-09-13T14:48:13.416 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.416 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.416 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.416 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.416 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.205021+0000", 2024-09-13T14:48:13.416 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:13.416 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.417 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.417 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.417 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.205027+0000", 2024-09-13T14:48:13.417 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0759999999999999e-06 2024-09-13T14:48:13.417 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.417 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.417 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.417 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.205043+0000", 2024-09-13T14:48:13.417 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5733e-05 2024-09-13T14:48:13.417 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.417 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.417 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.417 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.205049+0000", 2024-09-13T14:48:13.418 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3260000000000001e-06 2024-09-13T14:48:13.418 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.418 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.418 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.418 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.205108+0000", 2024-09-13T14:48:13.418 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9196e-05 2024-09-13T14:48:13.418 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.418 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.418 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.418 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.205120+0000", 2024-09-13T14:48:13.418 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.187e-05 2024-09-13T14:48:13.418 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.418 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.419 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.419 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.566119+0000", 2024-09-13T14:48:13.419 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.36099901299999998 2024-09-13T14:48:13.419 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.419 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.419 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.419 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.566127+0000", 2024-09-13T14:48:13.419 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9400000000000002e-06 2024-09-13T14:48:13.419 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.419 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.419 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.419 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.660594+0000", 2024-09-13T14:48:13.419 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.094467404 2024-09-13T14:48:13.419 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.420 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.420 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.420 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.660720+0000", 2024-09-13T14:48:13.420 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012609700000000001 2024-09-13T14:48:13.420 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.420 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.420 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:48:13.420 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.661049+0000", 2024-09-13T14:48:13.420 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00032867900000000002 2024-09-13T14:48:13.420 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.420 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.420 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.420 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.665650+0000", 2024-09-13T14:48:13.421 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0046013950000000003 2024-09-13T14:48:13.421 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.421 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.421 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.421 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.665700+0000", 2024-09-13T14:48:13.421 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9733e-05 2024-09-13T14:48:13.421 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.421 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.421 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.421 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.674346+0000", 2024-09-13T14:48:13.421 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0086455830000000001 2024-09-13T14:48:13.421 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.421 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.421 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.421 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.680888+0000", 2024-09-13T14:48:13.422 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0065421940000000003 2024-09-13T14:48:13.422 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.422 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.422 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.422 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.681032+0000", 2024-09-13T14:48:13.422 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014456000000000001 2024-09-13T14:48:13.422 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.422 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.422 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.422 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.681143+0000", 2024-09-13T14:48:13.422 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000110493 2024-09-13T14:48:13.422 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.422 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.422 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.423 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.423 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.423 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181681 6.17 6:edb6794d:::benchmark_data_smithi096_167087_object64453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:48:13.423 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:15.225202+0000", 2024-09-13T14:48:13.423 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 58.033243876, 2024-09-13T14:48:13.423 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.460923763, 2024-09-13T14:48:13.423 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:13.423 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:13.423 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:13.423 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:13.423 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:13.423 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181681 2024-09-13T14:48:13.423 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.424 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:13.424 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.424 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:13.424 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.225202+0000", 2024-09-13T14:48:13.424 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.424 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.424 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.424 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:13.424 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.225202+0000", 2024-09-13T14:48:13.424 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:13.424 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.424 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.424 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:13.424 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.225200+0000", 2024-09-13T14:48:13.425 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:13.425 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.425 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.425 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:13.425 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.225207+0000", 2024-09-13T14:48:13.425 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1689999999999997e-06 2024-09-13T14:48:13.425 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.425 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.425 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:13.425 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.225220+0000", 2024-09-13T14:48:13.425 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3202e-05 2024-09-13T14:48:13.425 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.425 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.426 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:13.426 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.225226+0000", 2024-09-13T14:48:13.426 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8300000000000001e-06 2024-09-13T14:48:13.426 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.426 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.426 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.426 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.225282+0000", 2024-09-13T14:48:13.426 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6214000000000001e-05 2024-09-13T14:48:13.426 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.426 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.426 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.426 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.225294+0000", 2024-09-13T14:48:13.426 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1674999999999999e-05 2024-09-13T14:48:13.427 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.427 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.427 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.427 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.566153+0000", 2024-09-13T14:48:13.427 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.34085950500000001 2024-09-13T14:48:13.427 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.427 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.427 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:13.427 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.566161+0000", 2024-09-13T14:48:13.427 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6550000000000004e-06 2024-09-13T14:48:13.427 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.427 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.427 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:13.427 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.662718+0000", 2024-09-13T14:48:13.428 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.096556917 2024-09-13T14:48:13.428 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.428 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.428 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:13.428 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.662837+0000", 2024-09-13T14:48:13.428 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119546 2024-09-13T14:48:13.428 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.428 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.428 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:48:13.428 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.664087+0000", 2024-09-13T14:48:13.428 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001249495 2024-09-13T14:48:13.428 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.428 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.429 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.429 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.674288+0000", 2024-09-13T14:48:13.429 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010201465999999999 2024-09-13T14:48:13.429 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.429 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.429 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:13.429 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.674356+0000", 2024-09-13T14:48:13.429 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8109999999999997e-05 2024-09-13T14:48:13.429 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.429 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.429 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.429 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.674428+0000", 2024-09-13T14:48:13.429 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1432000000000003e-05 2024-09-13T14:48:13.430 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.430 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.430 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:13.430 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.685967+0000", 2024-09-13T14:48:13.430 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011539181000000001 2024-09-13T14:48:13.430 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.430 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.430 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:13.430 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.686015+0000", 2024-09-13T14:48:13.430 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7984e-05 2024-09-13T14:48:13.430 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:13.430 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:13.430 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:13.431 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.686126+0000", 2024-09-13T14:48:13.431 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011105 2024-09-13T14:48:13.431 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.431 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.431 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.431 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:13.431 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:13.431 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:48:13.431 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:13 smithi204 ceph-mon[87071]: pgmap v609: 109 pgs: 109 active+clean; 10 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 932 op/s 2024-09-13T14:48:13.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.433 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:13.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:13.534 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:13.555 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:13.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:13.657 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:13.677 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:13.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:13.779 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:13.800 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:13.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:13.901 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:13.922 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:13.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:14.024 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:14.045 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:14.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:14.147 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:14.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.169 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:14.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:14.270 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:14.291 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:14.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:14.354 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:48:14.365 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:48:14.393 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:14.420 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:14.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:14.522 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:14.543 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:14.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:14.645 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:14.668 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:14.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:14.770 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:14.791 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:14.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:14.893 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:14.915 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:14.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:15.017 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:15.038 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:15.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:15.133 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:15.141 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:15.165 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:15.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:15.265 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:15.267 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:15.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.293 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:15.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:15.387 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:15 smithi096 ceph-mon[98901]: pgmap v610: 109 pgs: 109 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 537 op/s 2024-09-13T14:48:15.387 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:15 smithi096 ceph-mon[102928]: pgmap v610: 109 pgs: 109 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 537 op/s 2024-09-13T14:48:15.394 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:15.404 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:15 smithi204 ceph-mon[87071]: pgmap v610: 109 pgs: 109 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 537 op/s 2024-09-13T14:48:15.415 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:15.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:15.517 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:15.542 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:15.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:15.644 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:15.667 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:15.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:15.769 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:15.791 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:15.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:15.893 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:15.915 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:15.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:16.017 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:16.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.040 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:16.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:16.141 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:16.163 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:16.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:16.173 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:16.264 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:16.285 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:16.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:16.387 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:16.408 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:16.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:16.510 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:16.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.532 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:16.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:16.633 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:16.654 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:16.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:16.756 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:16.777 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:16.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:16.879 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:16.901 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:16.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:17.003 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:17.065 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:17.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:17.146 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:48:17.167 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:17.189 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:17.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:17.291 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:17.312 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:17.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:17.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:17 smithi096 ceph-mon[102928]: pgmap v611: 109 pgs: 109 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 522 op/s 2024-09-13T14:48:17.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:17 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/80092038' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:17.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:17 smithi096 ceph-mon[98901]: pgmap v611: 109 pgs: 109 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 522 op/s 2024-09-13T14:48:17.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:17 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/80092038' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:17.414 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:17.436 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:17.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:17.459 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:17 smithi204 ceph-mon[87071]: pgmap v611: 109 pgs: 109 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 522 op/s 2024-09-13T14:48:17.459 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:17 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/80092038' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:17.471 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:48:17.471 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:48:17.471 INFO:teuthology.orchestra.run.smithi204.stdout: "num_ops": 0 2024-09-13T14:48:17.471 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:48:17.538 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:17.559 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:17.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:17.661 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:17.682 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:17.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:17.784 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:17.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.806 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:17.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:17.907 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:17.929 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:17.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:18.031 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:18.053 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:18.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:18.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:48:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:48:17] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.51.0" 2024-09-13T14:48:18.111 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:48:18.111 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:48:18.111 INFO:tasks.thrashosds.thrasher:Adding osd 6 2024-09-13T14:48:18.111 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd in 6 2024-09-13T14:48:18.148 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:48:18.155 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:18.214 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:18.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:18.316 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:18.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.338 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:18.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:18.433 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:18 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:18.433 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:18 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/80092038' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:48:18.433 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:18 smithi204 ceph-mon[87071]: osdmap e221: 8 total, 8 up, 7 in 2024-09-13T14:48:18.440 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:18.463 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:18.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:18.556 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:18 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:18.556 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:18 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/80092038' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:48:18.556 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:18 smithi096 ceph-mon[98901]: osdmap e221: 8 total, 8 up, 7 in 2024-09-13T14:48:18.557 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:18 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:18.557 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:18 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/80092038' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:48:18.557 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:18 smithi096 ceph-mon[102928]: osdmap e221: 8 total, 8 up, 7 in 2024-09-13T14:48:18.564 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:18.587 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:18.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:18.689 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:18.712 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:18.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:18.814 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:18.835 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:18.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:18.937 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:18.959 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:18.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:19.061 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:19.082 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:19.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:19.157 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'unique_pool_1' removed 2024-09-13T14:48:19.184 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:19.219 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:19.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:19.301 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:19.321 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:19.342 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:19.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:19.427 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:19 smithi204 ceph-mon[87071]: pgmap v613: 109 pgs: 109 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 109 op/s 2024-09-13T14:48:19.427 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:19 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2627092952' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:48:19.427 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:19 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:48:19.444 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:19.467 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:19.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:19.494 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:19 smithi096 ceph-mon[98901]: pgmap v613: 109 pgs: 109 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 109 op/s 2024-09-13T14:48:19.494 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:19 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2627092952' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:48:19.494 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:19 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:48:19.494 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:19 smithi096 ceph-mon[102928]: pgmap v613: 109 pgs: 109 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 109 op/s 2024-09-13T14:48:19.494 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:19 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2627092952' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:48:19.494 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:19 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:48:19.570 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:19.591 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:19.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:19.693 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:19.713 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:19.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:19.815 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:19.837 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:19.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:19.939 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:19.962 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:19.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:19.979 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:48:19.979 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:48:19.980 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-09-13T14:48:19.980 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_2 16 2024-09-13T14:48:20.065 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:20.086 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:20.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:20.188 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:20.210 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:20.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:20.312 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:20.322 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:48:20.334 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:20.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:20.366 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:20 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:48:20.366 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:20 smithi096 ceph-mon[98901]: osdmap e222: 8 total, 8 up, 7 in 2024-09-13T14:48:20.366 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:20 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:48:20.366 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:20 smithi096 ceph-mon[102928]: osdmap e222: 8 total, 8 up, 7 in 2024-09-13T14:48:20.436 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:20.458 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:20.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:20.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:20 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:48:20.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:20 smithi204 ceph-mon[87071]: osdmap e222: 8 total, 8 up, 7 in 2024-09-13T14:48:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:48:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:48:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:20.484 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:48:20.484 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:48:20.485 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:48:20.485 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:48:20.499 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:48:20.499 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:48:20.517 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:48:20.517 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:48:20.533 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:48:20.533 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:48:20.549 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:48:20.549 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:48:20.560 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:20.564 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:48:20.565 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:48:20.581 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:48:20.581 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:48:20.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.583 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:20.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:20.684 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:20.706 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:20.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:20.809 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:20.831 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:20.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:20.933 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:20.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.955 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:20.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:21.056 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:21.081 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:21.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:21.183 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:21.205 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:21.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:21.307 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:21.319 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:21 smithi096 ceph-mon[98901]: pgmap v615: 83 pgs: 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:21.319 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:21 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2352907771' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:21.319 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:21.319 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:21 smithi096 ceph-mon[102928]: pgmap v615: 83 pgs: 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:21.320 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:21 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2352907771' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:21.320 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:21.330 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:21.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:21.421 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:21 smithi204 ceph-mon[87071]: pgmap v615: 83 pgs: 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:21.421 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:21 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2352907771' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:21.421 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:21.431 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:21.453 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:21.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:21.555 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:21.576 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:21.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:21.678 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:21.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.701 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:21.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:21.802 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:21.826 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:21.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:21.905 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:21.905 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:21.928 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:21.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.950 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:21.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:22.052 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:22.074 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:22.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:22.166 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:22.171 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:22.175 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:22.204 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:22.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:22.306 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:22.327 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:22.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:22.430 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:22.443 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:22.453 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:22.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:22.554 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:22.576 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:22.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:22.678 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:22.702 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:22.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:22.804 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:22.826 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:22.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:22.924 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:48:22.928 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:22.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.951 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:22.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:23.052 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:23.072 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:23.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:23.174 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:23.197 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:23.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:23.203 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:23.299 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:23.321 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:23.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:23.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:23 smithi096 ceph-mon[98901]: pgmap v616: 83 pgs: 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:23.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:23 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:23.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:23 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:23.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:23 smithi096 ceph-mon[102928]: pgmap v616: 83 pgs: 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:23.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:23 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:23.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:23 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:23.423 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:23.445 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:23.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:23.449 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:23 smithi204 ceph-mon[87071]: pgmap v616: 83 pgs: 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:23.449 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:23 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:23.449 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:23 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:23.546 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:23.570 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:23.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:23.671 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:23.695 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:23.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:23.797 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:23.820 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:23.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:23.922 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:23.944 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:23.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:24.046 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:24.072 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:24.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:24.121 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:48:24.174 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:24.197 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:24.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:24.298 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:24.320 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:24.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:24.422 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:24.443 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:24.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:24.545 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:24.566 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:24.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:24.668 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:24.690 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:24.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:24.792 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:24.813 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:24.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:24.880 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:24.880 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:24.915 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:24.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.937 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:24.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:25.038 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:25.059 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:25.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:25.161 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:25.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.183 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:25.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:25.284 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:25.304 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:25.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:25.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:25 smithi096 ceph-mon[98901]: pgmap v617: 83 pgs: 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:25.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:25 smithi096 ceph-mon[102928]: pgmap v617: 83 pgs: 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:25.406 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:25.428 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:25.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:25.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:25 smithi204 ceph-mon[87071]: pgmap v617: 83 pgs: 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:25.508 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:48:25.529 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:25.550 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:25.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:25.653 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:25.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.675 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:25.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:25.776 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:25.797 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:25.799 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:25.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:25.901 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:25.923 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:25.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:26.025 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:26.046 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:26.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:26.147 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:26.169 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:26.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:26.273 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:26.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.296 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:26.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:26.397 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:26.420 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:26.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:26.522 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:26.543 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:26.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:26.645 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:26.671 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:26.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:26.773 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:26.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.796 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:26.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:26.897 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:26.920 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:26.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:27.022 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:27.043 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:27.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:27.145 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:27.169 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:27.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:27.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:27 smithi204 ceph-mon[87071]: pgmap v618: 83 pgs: 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:27.271 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:27.293 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:27.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:27.374 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:27 smithi096 ceph-mon[98901]: pgmap v618: 83 pgs: 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:27.375 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:27 smithi096 ceph-mon[102928]: pgmap v618: 83 pgs: 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:27.395 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:27.416 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:27.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:27.491 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:27.491 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:27.518 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:27.540 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:27.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:27.642 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:27.663 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:27.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:27.765 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:27.787 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:27.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:27.889 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:27.910 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:27.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:27.970 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:48:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:48:27] "GET /metrics HTTP/1.1" 200 34201 "" "Prometheus/2.51.0" 2024-09-13T14:48:28.012 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:28.034 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:28.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:28.135 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:28.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.159 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:28.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:28.260 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:28.282 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:28.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:28.384 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:28.405 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:28.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:28.508 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:28.517 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:28.531 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:28.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:28.587 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:48:28.633 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:28.656 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:28.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:28.757 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:28.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.781 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:28.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:28.841 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:28.882 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:28.904 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:28.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:29.006 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:29.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.030 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:29.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:29.131 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:29.153 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:29.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:29.173 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:48:29.173 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'unique_pool_2' created 2024-09-13T14:48:29.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:29 smithi204 ceph-mon[87071]: pgmap v619: 83 pgs: 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:29 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3729429357' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:29 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3995908850' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:48:29.254 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:29.275 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:29.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:29 smithi096 ceph-mon[98901]: pgmap v619: 83 pgs: 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:29 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3729429357' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:29 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3995908850' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:48:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:29 smithi096 ceph-mon[102928]: pgmap v619: 83 pgs: 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:29 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3729429357' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:29 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3995908850' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:48:29.377 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:29.398 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:29.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:29.500 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:29.522 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:29.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:29.624 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:29.645 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:29.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:29.746 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:29.768 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:29.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:29.870 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:29.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.893 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:29.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:29.994 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:30.016 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:30.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:30.118 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:30.140 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:30.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:30.215 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:30 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:48:30.215 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:30 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:48:30.215 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:30 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3729429357' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:30.215 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:30 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3995908850' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-13T14:48:30.215 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:30 smithi204 ceph-mon[87071]: osdmap e223: 8 total, 8 up, 7 in 2024-09-13T14:48:30.242 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:30.263 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:30.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:30.305 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:30 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:48:30.305 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:30 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:48:30.305 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:30 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3729429357' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:30.306 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:30 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3995908850' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-13T14:48:30.306 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:30 smithi096 ceph-mon[98901]: osdmap e223: 8 total, 8 up, 7 in 2024-09-13T14:48:30.306 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:30 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:48:30.306 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:30 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:48:30.306 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:30 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3729429357' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:30.306 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:30 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3995908850' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-13T14:48:30.306 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:30 smithi096 ceph-mon[102928]: osdmap e223: 8 total, 8 up, 7 in 2024-09-13T14:48:30.365 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:30.386 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:30.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:30.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:48:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:48:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:30.488 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:30.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.511 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:30.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:30.537 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:30.538 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:30.612 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:30.636 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:30.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:30.737 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:30.758 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:30.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:30.860 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:30.881 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:30.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:30.983 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:31.004 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:31.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:31.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:30 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a[98876]: 2024-09-13T14:48:30.815+0000 7f16aeda4640 -1 mon.a@0(leader).osd e224 definitely_dead 0 2024-09-13T14:48:31.106 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:31.127 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:31.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:31.149 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:48:31.184 INFO:teuthology.orchestra.run.smithi096.stderr:marked in osd.6. 2024-09-13T14:48:31.229 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:31.249 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:31.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:31.351 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:31.374 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:31.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:31.402 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:31.445 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:31 smithi096 ceph-mon[98901]: pgmap v621: 99 pgs: 2 creating+peering, 14 unknown, 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:31.445 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:31 smithi096 ceph-mon[98901]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:48:31.446 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:31 smithi096 ceph-mon[98901]: osdmap e224: 8 total, 8 up, 7 in 2024-09-13T14:48:31.446 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:31 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2873696161' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-13T14:48:31.446 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:31 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-13T14:48:31.446 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:31 smithi096 ceph-mon[102928]: pgmap v621: 99 pgs: 2 creating+peering, 14 unknown, 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:31.446 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:31 smithi096 ceph-mon[102928]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:48:31.446 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:31 smithi096 ceph-mon[102928]: osdmap e224: 8 total, 8 up, 7 in 2024-09-13T14:48:31.446 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:31 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2873696161' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-13T14:48:31.446 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:31 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-13T14:48:31.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:31 smithi204 ceph-mon[87071]: pgmap v621: 99 pgs: 2 creating+peering, 14 unknown, 83 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:48:31.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:31 smithi204 ceph-mon[87071]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:48:31.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:31 smithi204 ceph-mon[87071]: osdmap e224: 8 total, 8 up, 7 in 2024-09-13T14:48:31.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:31 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2873696161' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-13T14:48:31.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:31 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-13T14:48:31.476 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:31.498 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:31.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:31.600 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:31.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.622 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:31.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:31.723 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:31.745 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:31.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:31.847 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:31.869 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:31.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:31.971 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:31.992 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:31.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:32.094 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:32.116 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:32.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:32.218 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:32.240 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:32.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:32.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:32 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-13T14:48:32.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:32 smithi096 ceph-mon[98901]: osdmap e225: 8 total, 8 up, 8 in 2024-09-13T14:48:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:32 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-13T14:48:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:32 smithi096 ceph-mon[102928]: osdmap e225: 8 total, 8 up, 8 in 2024-09-13T14:48:32.342 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:32.363 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:32.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:32.465 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:32.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:32 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-13T14:48:32.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:32 smithi204 ceph-mon[87071]: osdmap e225: 8 total, 8 up, 8 in 2024-09-13T14:48:32.487 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:32.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:32.589 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:32.612 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:32.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:32.622 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-09-13T14:48:32.715 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:32.736 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:32.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:32.838 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:32.861 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:32.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:32.965 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:32.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.987 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:32.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:33.089 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:33.097 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:33.097 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:33.112 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:33.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:33.214 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:33.236 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:33.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:33.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:33 smithi096 ceph-mon[98901]: pgmap v624: 99 pgs: 22 peering, 2 creating+peering, 9 unknown, 66 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:48:33.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:33 smithi096 ceph-mon[98901]: osdmap e226: 8 total, 8 up, 8 in 2024-09-13T14:48:33.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:33.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:33.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:33 smithi096 ceph-mon[102928]: pgmap v624: 99 pgs: 22 peering, 2 creating+peering, 9 unknown, 66 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:48:33.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:33 smithi096 ceph-mon[102928]: osdmap e226: 8 total, 8 up, 8 in 2024-09-13T14:48:33.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:33.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:33.338 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:33.356 INFO:tasks.thrashosds.thrasher:Added osd 6 2024-09-13T14:48:33.359 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:33.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:33.461 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:33 smithi204 ceph-mon[87071]: pgmap v624: 99 pgs: 22 peering, 2 creating+peering, 9 unknown, 66 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:48:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:33 smithi204 ceph-mon[87071]: osdmap e226: 8 total, 8 up, 8 in 2024-09-13T14:48:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:33.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.484 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:33.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:33.585 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:33.592 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:48:33.592 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:48:33.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.608 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:33.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:33.609 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:48:33.609 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:48:33.626 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:48:33.626 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:48:33.644 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:33.644 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:48:33.645 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:48:33.660 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:48:33.660 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:48:33.676 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:48:33.677 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:48:33.692 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:48:33.692 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:48:33.707 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:48:33.708 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:48:33.709 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:33.732 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:33.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:33.758 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:48:33.822 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:48:33.835 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:33.886 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:33.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:33.988 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:34.010 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:34.010 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:34.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:34.112 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:34.134 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:34.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:34.236 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:34.258 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:34.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:34.360 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:34.381 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:34.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:34.387 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:34 smithi204 ceph-mon[87071]: osdmap e227: 8 total, 8 up, 8 in 2024-09-13T14:48:34.387 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:34 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:48:34.387 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:34 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:48:34.387 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:34 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:34.388 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:34 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3009789254' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:34.388 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:34 smithi204 ceph-mon[87071]: osdmap e228: 8 total, 8 up, 8 in 2024-09-13T14:48:34.483 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:34.505 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:34.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:34.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:34 smithi096 ceph-mon[98901]: osdmap e227: 8 total, 8 up, 8 in 2024-09-13T14:48:34.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:34 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:48:34.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:34 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:48:34.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:34 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:34.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:34 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3009789254' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:34.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:34 smithi096 ceph-mon[98901]: osdmap e228: 8 total, 8 up, 8 in 2024-09-13T14:48:34.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:34 smithi096 ceph-mon[102928]: osdmap e227: 8 total, 8 up, 8 in 2024-09-13T14:48:34.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:34 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:48:34.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:34 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:48:34.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:34 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:34.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:34 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3009789254' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:34.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:34 smithi096 ceph-mon[102928]: osdmap e228: 8 total, 8 up, 8 in 2024-09-13T14:48:34.608 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:34.630 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:34.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:34.641 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:34.732 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:34.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.755 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:34.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:34.856 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:34.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.877 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:34.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:34.978 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:34.999 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:35.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:35.101 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:35.124 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:35.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:35.226 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:35.248 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:35.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:35.349 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:35.371 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:35.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:35.395 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:35 smithi204 ceph-mon[87071]: pgmap v627: 99 pgs: 28 peering, 2 creating+peering, 2 remapped+peering, 67 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:35.395 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:35 smithi204 ceph-mon[87071]: 1.7 scrub starts 2024-09-13T14:48:35.395 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:35 smithi204 ceph-mon[87071]: 1.7 scrub ok 2024-09-13T14:48:35.396 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:35 smithi204 ceph-mon[87071]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:48:35.396 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:35 smithi204 ceph-mon[87071]: osdmap e229: 8 total, 8 up, 8 in 2024-09-13T14:48:35.472 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:35.493 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:35.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:35.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:35 smithi096 ceph-mon[98901]: pgmap v627: 99 pgs: 28 peering, 2 creating+peering, 2 remapped+peering, 67 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:35.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:35 smithi096 ceph-mon[98901]: 1.7 scrub starts 2024-09-13T14:48:35.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:35 smithi096 ceph-mon[98901]: 1.7 scrub ok 2024-09-13T14:48:35.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:35 smithi096 ceph-mon[98901]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:48:35.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:35 smithi096 ceph-mon[98901]: osdmap e229: 8 total, 8 up, 8 in 2024-09-13T14:48:35.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:35 smithi096 ceph-mon[102928]: pgmap v627: 99 pgs: 28 peering, 2 creating+peering, 2 remapped+peering, 67 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:35.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:35 smithi096 ceph-mon[102928]: 1.7 scrub starts 2024-09-13T14:48:35.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:35 smithi096 ceph-mon[102928]: 1.7 scrub ok 2024-09-13T14:48:35.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:35 smithi096 ceph-mon[102928]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:48:35.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:35 smithi096 ceph-mon[102928]: osdmap e229: 8 total, 8 up, 8 in 2024-09-13T14:48:35.595 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:35.616 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:35.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:35.620 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:48:35.621 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:48:35.621 INFO:teuthology.orchestra.run.smithi204.stdout: "num_ops": 0 2024-09-13T14:48:35.621 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:48:35.718 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:35.740 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:35.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:35.842 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:35.864 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:35.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:35.965 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:35.988 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:35.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:36.090 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:36.112 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:36.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:36.214 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:36.235 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:36.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:36.317 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:48:36.337 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:36.382 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:36.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:36.484 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:36.506 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:36.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:36.566 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:36.608 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:36.629 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:36.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:36.731 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:36.752 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:36.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:36.853 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:36.875 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:36.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:36.942 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:36 smithi204 ceph-mon[87071]: pgmap v630: 99 pgs: 28 peering, 2 remapped+peering, 69 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:36.942 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:36.943 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:36.943 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:36 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2679962777' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:48:36.943 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:36 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:48:36.977 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:37.001 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:37.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:37.102 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:37.125 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:37.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:37.167 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:48:37.189 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:36 smithi096 ceph-mon[98901]: pgmap v630: 99 pgs: 28 peering, 2 remapped+peering, 69 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:37.189 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:37.189 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:37.189 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:36 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2679962777' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:48:37.190 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:36 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:48:37.190 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:36 smithi096 ceph-mon[102928]: pgmap v630: 99 pgs: 28 peering, 2 remapped+peering, 69 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:37.190 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:37.190 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:37.190 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:36 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2679962777' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:48:37.190 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:36 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:48:37.203 INFO:teuthology.orchestra.run.smithi096.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-09-13T14:48:37.227 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:37.249 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:37.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:37.352 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:37.373 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:37.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:37.475 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:37.497 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:37.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:37.600 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:37.621 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:37.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:37.723 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:37.745 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:37.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:37.846 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:37.868 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:37.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:37.970 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:37.979 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:48:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:48:37] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.51.0" 2024-09-13T14:48:37.992 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:37.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:38.094 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:38.116 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:38.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:38.218 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:38.239 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:38.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:38.241 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:48:38.241 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:48:38.241 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T14:48:38.241 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T14:48:38.241 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:48:38.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:38 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:48:38.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:38 smithi096 ceph-mon[98901]: osdmap e230: 8 total, 8 up, 8 in 2024-09-13T14:48:38.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:38 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:48:38.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:38 smithi096 ceph-mon[102928]: osdmap e230: 8 total, 8 up, 8 in 2024-09-13T14:48:38.341 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:38.357 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:48:38.357 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:48:38.357 INFO:tasks.thrashosds.thrasher:reset_purged_snaps_last 2024-09-13T14:48:38.357 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.0 reset_purged_snaps_last 2024-09-13T14:48:38.362 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:38.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:38.465 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:38.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:38 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:48:38.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:38 smithi204 ceph-mon[87071]: osdmap e230: 8 total, 8 up, 8 in 2024-09-13T14:48:38.486 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:38.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:38.588 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:38.611 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:38.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:38.651 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:38.684 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:38.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:38.734 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:38.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:38.835 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:38.858 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:38.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:38.893 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:48:38.960 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:38.998 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:38.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:39.099 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:39.121 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:39.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:39.182 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:39.211 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:48:39.222 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:39.243 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:39.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:39.277 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:39 smithi096 ceph-mon[98901]: pgmap v632: 99 pgs: 99 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 886 KiB/s, 10 objects/s recovering 2024-09-13T14:48:39.278 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:39 smithi096 ceph-mon[98901]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:48:39.278 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:39 smithi096 ceph-mon[98901]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:48:39.278 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:39 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:48:39.278 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:39 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/36361057' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:48:39.278 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:39 smithi096 ceph-mon[102928]: pgmap v632: 99 pgs: 99 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 886 KiB/s, 10 objects/s recovering 2024-09-13T14:48:39.278 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:39 smithi096 ceph-mon[102928]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:48:39.278 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:39 smithi096 ceph-mon[102928]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:48:39.278 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:39 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:48:39.278 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:39 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/36361057' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:48:39.345 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:39.366 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:39.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:39.468 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:39.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:39 smithi204 ceph-mon[87071]: pgmap v632: 99 pgs: 99 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 886 KiB/s, 10 objects/s recovering 2024-09-13T14:48:39.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:39 smithi204 ceph-mon[87071]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:48:39.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:39 smithi204 ceph-mon[87071]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:48:39.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:39 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:48:39.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:39 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/36361057' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:48:39.490 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:39.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:39.592 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:39.613 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:39.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:39.622 DEBUG:teuthology.orchestra.run.smithi096:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-09-13T14:48:39.658 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:48:39.658 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:48:39.702 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:hints = 1 2024-09-13T14:48:39.702 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:48:39.702 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Object prefix: benchmark_data_smithi096_196313 2024-09-13T14:48:39.703 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:48:39.703 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:48:39.715 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:39.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.758 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:39.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:39.860 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:39.898 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:39.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:40.000 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:40.043 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:40.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:40.145 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:40.180 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:40.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:40.282 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:40.295 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:48:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:48:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:40.296 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:40 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:40.296 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:40 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/36361057' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:48:40.296 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:40 smithi204 ceph-mon[87071]: osdmap e231: 8 total, 8 up, 8 in 2024-09-13T14:48:40.314 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:40 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:40.314 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:40 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/36361057' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:48:40.314 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:40 smithi096 ceph-mon[98901]: osdmap e231: 8 total, 8 up, 8 in 2024-09-13T14:48:40.315 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:40 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:40.315 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:40 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/36361057' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:48:40.315 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:40 smithi096 ceph-mon[102928]: osdmap e231: 8 total, 8 up, 8 in 2024-09-13T14:48:40.317 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:40.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:40.419 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:40.461 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:40.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:40.563 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:40.602 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:40.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:40.703 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 1 16 3632 3616 225.987 226 0.00274467 0.00441082 2024-09-13T14:48:40.704 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:40.739 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:40.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:40.841 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:40.870 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:40.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:40.972 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:41.012 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:41.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:41.114 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:41.120 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:48:41.120 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T14:48:41.120 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T14:48:41.120 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:48:41.120 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.120 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35049.0:68069 5.4 5:20652590:::benchmark_data_smithi096_138841_object68068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-13T14:48:41.120 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:40:25.511751+0000", 2024-09-13T14:48:41.120 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 495.60160122299999, 2024-09-13T14:48:41.120 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.166519957, 2024-09-13T14:48:41.120 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.120 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:41.121 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:41.121 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35049", 2024-09-13T14:48:41.121 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4217909210", 2024-09-13T14:48:41.121 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 68069 2024-09-13T14:48:41.121 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.121 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.121 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.121 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.121 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.511751+0000", 2024-09-13T14:48:41.121 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.122 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.122 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.122 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.122 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.511751+0000", 2024-09-13T14:48:41.122 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.122 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.122 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.122 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.122 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.511750+0000", 2024-09-13T14:48:41.122 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:48:41.122 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.122 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.122 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.123 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.511839+0000", 2024-09-13T14:48:41.123 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8355000000000004e-05 2024-09-13T14:48:41.123 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.123 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.123 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.123 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.511853+0000", 2024-09-13T14:48:41.123 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4687e-05 2024-09-13T14:48:41.123 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.123 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.123 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.123 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.511859+0000", 2024-09-13T14:48:41.123 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5099999999999998e-06 2024-09-13T14:48:41.123 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.123 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.124 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.124 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.513682+0000", 2024-09-13T14:48:41.124 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001823437 2024-09-13T14:48:41.124 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.124 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.124 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.124 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.513803+0000", 2024-09-13T14:48:41.124 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000121128 2024-09-13T14:48:41.124 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.124 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.124 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:48:41.124 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.513923+0000", 2024-09-13T14:48:41.125 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119542 2024-09-13T14:48:41.125 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.125 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.125 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:41.125 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.518669+0000", 2024-09-13T14:48:41.125 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0047461439999999999 2024-09-13T14:48:41.125 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.125 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.125 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.125 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.519128+0000", 2024-09-13T14:48:41.125 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000459004 2024-09-13T14:48:41.125 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.125 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.125 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.126 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.678206+0000", 2024-09-13T14:48:41.126 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15907775399999999 2024-09-13T14:48:41.126 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.126 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.126 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:41.126 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.678244+0000", 2024-09-13T14:48:41.126 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8247000000000002e-05 2024-09-13T14:48:41.126 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.126 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.127 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.127 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:40:25.678271+0000", 2024-09-13T14:48:41.127 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7084e-05 2024-09-13T14:48:41.127 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.127 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.127 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.127 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.127 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.127 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:34135 6.3 6:c2de75f9:::benchmark_data_smithi096_167087_object34134:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:48:41.127 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.699103+0000", 2024-09-13T14:48:41.127 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 258.41424975699999, 2024-09-13T14:48:41.127 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19865819200000001, 2024-09-13T14:48:41.127 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.128 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:41.128 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:41.128 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:41.128 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:41.128 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 34135 2024-09-13T14:48:41.128 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.128 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.128 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.128 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.128 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699103+0000", 2024-09-13T14:48:41.128 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.128 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.128 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.129 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.129 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699103+0000", 2024-09-13T14:48:41.129 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.129 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.129 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.129 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.129 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699102+0000", 2024-09-13T14:48:41.129 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:48:41.129 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.129 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.129 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.129 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699154+0000", 2024-09-13T14:48:41.129 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2114000000000003e-05 2024-09-13T14:48:41.130 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.130 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.130 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.130 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699161+0000", 2024-09-13T14:48:41.130 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1899999999999998e-06 2024-09-13T14:48:41.130 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.130 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.130 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.130 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699165+0000", 2024-09-13T14:48:41.130 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.597e-06 2024-09-13T14:48:41.130 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.130 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.130 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.131 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699198+0000", 2024-09-13T14:48:41.131 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3531000000000001e-05 2024-09-13T14:48:41.131 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.131 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.131 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.131 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699264+0000", 2024-09-13T14:48:41.131 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5819000000000001e-05 2024-09-13T14:48:41.131 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.131 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.131 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:48:41.131 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699341+0000", 2024-09-13T14:48:41.131 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6790000000000002e-05 2024-09-13T14:48:41.131 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.132 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.132 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:41.132 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.701095+0000", 2024-09-13T14:48:41.132 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001754434 2024-09-13T14:48:41.132 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.132 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.132 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.132 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897558+0000", 2024-09-13T14:48:41.132 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19646306199999999 2024-09-13T14:48:41.132 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.132 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.132 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.132 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897605+0000", 2024-09-13T14:48:41.132 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7012000000000003e-05 2024-09-13T14:48:41.133 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.133 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.133 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:41.133 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897648+0000", 2024-09-13T14:48:41.133 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3058e-05 2024-09-13T14:48:41.133 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.133 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.133 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.133 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897761+0000", 2024-09-13T14:48:41.133 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011252999999999999 2024-09-13T14:48:41.133 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.133 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.133 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.134 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.134 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.134 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:34139 6.3 6:cd715ee8:::benchmark_data_smithi096_167087_object34138:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:48:41.134 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.699339+0000", 2024-09-13T14:48:41.134 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 258.414013512, 2024-09-13T14:48:41.134 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19895058299999999, 2024-09-13T14:48:41.134 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.134 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:41.134 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:41.134 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:41.134 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:41.134 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 34139 2024-09-13T14:48:41.134 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.135 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.135 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.135 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.135 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699339+0000", 2024-09-13T14:48:41.135 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.135 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.135 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.135 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.135 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699339+0000", 2024-09-13T14:48:41.135 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.135 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.136 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.136 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.136 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699338+0000", 2024-09-13T14:48:41.136 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:48:41.136 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.136 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.136 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.136 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699447+0000", 2024-09-13T14:48:41.136 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000109077 2024-09-13T14:48:41.136 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.136 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.136 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.136 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699456+0000", 2024-09-13T14:48:41.137 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3520000000000007e-06 2024-09-13T14:48:41.137 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.137 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.137 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.137 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699460+0000", 2024-09-13T14:48:41.137 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6920000000000001e-06 2024-09-13T14:48:41.137 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.137 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.137 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.137 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.700283+0000", 2024-09-13T14:48:41.137 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000822742 2024-09-13T14:48:41.137 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.137 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.138 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.138 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.700351+0000", 2024-09-13T14:48:41.138 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8039000000000004e-05 2024-09-13T14:48:41.138 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.138 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.138 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:48:41.138 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.700433+0000", 2024-09-13T14:48:41.138 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2398999999999995e-05 2024-09-13T14:48:41.138 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.138 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.138 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:41.138 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.701174+0000", 2024-09-13T14:48:41.138 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00074037599999999995 2024-09-13T14:48:41.138 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897584+0000", 2024-09-13T14:48:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19641035100000001 2024-09-13T14:48:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.898214+0000", 2024-09-13T14:48:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00063014600000000005 2024-09-13T14:48:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.898255+0000", 2024-09-13T14:48:41.139 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1083e-05 2024-09-13T14:48:41.140 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.140 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.140 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.140 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.898289+0000", 2024-09-13T14:48:41.140 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4069000000000002e-05 2024-09-13T14:48:41.140 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.140 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.140 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.140 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.140 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.140 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:34136 6.c e150/145 6:3e63118a:::benchmark_data_smithi096_167087_object34135:head v 150'2081, mlcod=150'2079)", 2024-09-13T14:48:41.140 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.703607+0000", 2024-09-13T14:48:41.140 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 258.40974522499999, 2024-09-13T14:48:41.141 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19718830400000001, 2024-09-13T14:48:41.141 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.141 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:41.141 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.141 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.141 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.141 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703607+0000", 2024-09-13T14:48:41.141 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.141 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.141 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.141 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.141 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703607+0000", 2024-09-13T14:48:41.141 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.142 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.142 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.142 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.142 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703606+0000", 2024-09-13T14:48:41.142 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:41.142 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.142 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.142 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.142 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703652+0000", 2024-09-13T14:48:41.142 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6789000000000002e-05 2024-09-13T14:48:41.142 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.142 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.142 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.142 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703674+0000", 2024-09-13T14:48:41.143 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1297e-05 2024-09-13T14:48:41.143 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.143 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.143 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.143 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703683+0000", 2024-09-13T14:48:41.143 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5559999999999996e-06 2024-09-13T14:48:41.143 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.143 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.143 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.143 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897235+0000", 2024-09-13T14:48:41.143 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19355149599999999 2024-09-13T14:48:41.143 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.143 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.143 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.144 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897275+0000", 2024-09-13T14:48:41.144 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0275000000000003e-05 2024-09-13T14:48:41.144 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.144 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.144 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:41.144 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.900770+0000", 2024-09-13T14:48:41.144 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0034945810000000001 2024-09-13T14:48:41.144 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.144 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.144 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.144 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.900795+0000", 2024-09-13T14:48:41.144 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5803e-05 2024-09-13T14:48:41.144 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.145 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.145 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.145 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.145 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.145 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.45167.0:34135 6.3 e150/145 ondisk, result = 0)", 2024-09-13T14:48:41.145 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.704116+0000", 2024-09-13T14:48:41.145 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 258.409236265, 2024-09-13T14:48:41.145 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19344439799999999, 2024-09-13T14:48:41.145 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.145 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:48:41.145 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.145 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.145 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.146 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704116+0000", 2024-09-13T14:48:41.146 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.146 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.146 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.146 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.146 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704116+0000", 2024-09-13T14:48:41.146 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.146 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.146 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.146 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.146 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704115+0000", 2024-09-13T14:48:41.146 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:48:41.146 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.147 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.147 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.147 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704119+0000", 2024-09-13T14:48:41.147 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9960000000000004e-06 2024-09-13T14:48:41.147 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.147 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.147 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.147 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897303+0000", 2024-09-13T14:48:41.147 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19318347899999999 2024-09-13T14:48:41.147 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.147 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.147 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.147 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897312+0000", 2024-09-13T14:48:41.148 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8969999999999997e-06 2024-09-13T14:48:41.148 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.148 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.148 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.148 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897545+0000", 2024-09-13T14:48:41.148 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00023321799999999999 2024-09-13T14:48:41.148 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.148 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.148 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.148 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897552+0000", 2024-09-13T14:48:41.148 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.396e-06 2024-09-13T14:48:41.148 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.148 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.148 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.149 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897560+0000", 2024-09-13T14:48:41.149 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0050000000000004e-06 2024-09-13T14:48:41.149 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.149 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.149 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.149 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.149 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.149 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:34141 6.b e150/145 6:d45fd98b:::benchmark_data_smithi096_167087_object34140:head v 150'2166, mlcod=150'2163)", 2024-09-13T14:48:41.149 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.704766+0000", 2024-09-13T14:48:41.150 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 258.40858602200001, 2024-09-13T14:48:41.150 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.197159529, 2024-09-13T14:48:41.150 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.150 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:41.150 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.150 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.150 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.151 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704766+0000", 2024-09-13T14:48:41.151 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.151 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.151 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.151 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.151 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704766+0000", 2024-09-13T14:48:41.151 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.151 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.151 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.151 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.151 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704766+0000", 2024-09-13T14:48:41.151 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:48:41.151 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.151 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.152 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.152 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704832+0000", 2024-09-13T14:48:41.152 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5964999999999996e-05 2024-09-13T14:48:41.152 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.152 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.152 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.152 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897456+0000", 2024-09-13T14:48:41.152 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.192624657 2024-09-13T14:48:41.152 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.152 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.152 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.152 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897466+0000", 2024-09-13T14:48:41.152 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5510000000000005e-06 2024-09-13T14:48:41.153 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.153 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.153 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.153 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897783+0000", 2024-09-13T14:48:41.153 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00031745699999999999 2024-09-13T14:48:41.153 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.153 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.153 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.153 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897800+0000", 2024-09-13T14:48:41.153 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6619000000000001e-05 2024-09-13T14:48:41.153 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.153 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.153 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:41.154 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.901888+0000", 2024-09-13T14:48:41.154 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0040879230000000003 2024-09-13T14:48:41.154 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.154 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.154 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.154 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.901926+0000", 2024-09-13T14:48:41.154 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7832e-05 2024-09-13T14:48:41.154 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.154 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.154 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.154 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.154 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.154 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:37471 6.9 e150/145 6:92dc42f2:::benchmark_data_smithi096_167087_object37470:head v 150'2391, mlcod=150'2389)", 2024-09-13T14:48:41.154 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:25.099939+0000", 2024-09-13T14:48:41.155 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 256.01341358500002, 2024-09-13T14:48:41.155 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16909158199999999, 2024-09-13T14:48:41.155 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.155 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:41.155 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.155 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.155 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.155 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.099939+0000", 2024-09-13T14:48:41.155 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.155 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.155 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.155 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.155 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.099939+0000", 2024-09-13T14:48:41.156 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.156 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.156 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.156 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.156 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.099938+0000", 2024-09-13T14:48:41.156 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:48:41.156 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.156 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.156 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.156 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.099998+0000", 2024-09-13T14:48:41.156 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9787999999999997e-05 2024-09-13T14:48:41.156 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.156 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.156 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.157 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100005+0000", 2024-09-13T14:48:41.157 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5900000000000002e-06 2024-09-13T14:48:41.157 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.157 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.157 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.157 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100009+0000", 2024-09-13T14:48:41.157 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5839999999999999e-06 2024-09-13T14:48:41.157 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.157 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.157 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.157 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100043+0000", 2024-09-13T14:48:41.157 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3726000000000002e-05 2024-09-13T14:48:41.157 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.157 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.158 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.158 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100060+0000", 2024-09-13T14:48:41.158 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6731999999999998e-05 2024-09-13T14:48:41.158 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.158 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.158 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:41.158 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.268975+0000", 2024-09-13T14:48:41.158 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16891590300000001 2024-09-13T14:48:41.158 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.158 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.158 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.158 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269030+0000", 2024-09-13T14:48:41.158 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4947000000000001e-05 2024-09-13T14:48:41.159 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.159 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.159 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.159 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.159 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.159 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:37473 6.6 e150/145 6:6dedaf54:::benchmark_data_smithi096_167087_object37472:head v 150'2324, mlcod=150'2322)", 2024-09-13T14:48:41.159 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:25.100530+0000", 2024-09-13T14:48:41.159 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 256.01282197299997, 2024-09-13T14:48:41.159 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.17121808399999999, 2024-09-13T14:48:41.159 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.159 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:41.159 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.159 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.159 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.160 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100530+0000", 2024-09-13T14:48:41.160 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.160 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.160 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.160 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.160 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100530+0000", 2024-09-13T14:48:41.160 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.160 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.160 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.160 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.160 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100530+0000", 2024-09-13T14:48:41.160 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:48:41.160 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.161 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.161 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.161 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100650+0000", 2024-09-13T14:48:41.161 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011967 2024-09-13T14:48:41.161 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.161 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.161 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.161 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100656+0000", 2024-09-13T14:48:41.161 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1380000000000004e-06 2024-09-13T14:48:41.161 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.161 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.161 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.161 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100659+0000", 2024-09-13T14:48:41.162 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9249999999999999e-06 2024-09-13T14:48:41.162 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.162 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.162 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.162 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100689+0000", 2024-09-13T14:48:41.162 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0114999999999999e-05 2024-09-13T14:48:41.162 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.162 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.162 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.162 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100704+0000", 2024-09-13T14:48:41.162 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5502000000000002e-05 2024-09-13T14:48:41.162 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.162 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.162 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:41.163 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271712+0000", 2024-09-13T14:48:41.163 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.17100812600000001 2024-09-13T14:48:41.163 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.163 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.163 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.163 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271748+0000", 2024-09-13T14:48:41.163 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6117e-05 2024-09-13T14:48:41.163 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.163 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.163 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.163 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.163 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.163 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:37477 6.b e150/145 6:d0ca37d0:::benchmark_data_smithi096_167087_object37476:head v 150'2365, mlcod=150'2363)", 2024-09-13T14:48:41.164 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:25.101869+0000", 2024-09-13T14:48:41.164 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 256.01148358699999, 2024-09-13T14:48:41.164 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.169716322, 2024-09-13T14:48:41.164 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.164 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:41.164 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.164 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.164 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.164 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.101869+0000", 2024-09-13T14:48:41.164 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.164 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.164 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.164 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.165 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.101869+0000", 2024-09-13T14:48:41.165 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.165 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.165 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.165 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.165 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.101868+0000", 2024-09-13T14:48:41.165 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:48:41.165 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.165 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.165 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.165 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.101944+0000", 2024-09-13T14:48:41.165 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6200999999999998e-05 2024-09-13T14:48:41.165 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.165 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.166 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.166 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269038+0000", 2024-09-13T14:48:41.166 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.167093726 2024-09-13T14:48:41.166 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.166 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.166 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.166 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269044+0000", 2024-09-13T14:48:41.166 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2729999999999999e-06 2024-09-13T14:48:41.166 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.166 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.166 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.166 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269122+0000", 2024-09-13T14:48:41.166 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7550000000000001e-05 2024-09-13T14:48:41.166 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.167 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.167 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.167 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269141+0000", 2024-09-13T14:48:41.167 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9026999999999999e-05 2024-09-13T14:48:41.167 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.167 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.167 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:41.167 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271551+0000", 2024-09-13T14:48:41.167 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0024101700000000001 2024-09-13T14:48:41.167 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.167 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.167 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.167 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271585+0000", 2024-09-13T14:48:41.167 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3859000000000002e-05 2024-09-13T14:48:41.168 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.168 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.168 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.168 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.168 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.168 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:37481 6.1 e150/145 6:86b51d11:::benchmark_data_smithi096_167087_object37480:head v 150'2387, mlcod=150'2385)", 2024-09-13T14:48:41.168 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:25.103719+0000", 2024-09-13T14:48:41.168 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 256.00963295100001, 2024-09-13T14:48:41.168 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16811326800000001, 2024-09-13T14:48:41.168 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.168 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:41.168 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.168 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.169 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.169 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.103719+0000", 2024-09-13T14:48:41.169 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.169 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.169 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.169 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.169 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.103719+0000", 2024-09-13T14:48:41.169 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.169 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.169 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.169 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.169 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.103719+0000", 2024-09-13T14:48:41.169 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:48:41.170 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.170 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.170 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.170 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.103768+0000", 2024-09-13T14:48:41.170 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9070000000000003e-05 2024-09-13T14:48:41.170 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.170 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.170 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.170 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.268962+0000", 2024-09-13T14:48:41.170 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.165193749 2024-09-13T14:48:41.170 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.170 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.170 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.170 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.268973+0000", 2024-09-13T14:48:41.171 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1504e-05 2024-09-13T14:48:41.171 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.171 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.171 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.171 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269062+0000", 2024-09-13T14:48:41.171 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8554999999999995e-05 2024-09-13T14:48:41.171 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.171 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.171 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.171 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.269087+0000", 2024-09-13T14:48:41.171 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5772999999999999e-05 2024-09-13T14:48:41.171 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.171 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.171 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:41.172 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271798+0000", 2024-09-13T14:48:41.172 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0027102630000000001 2024-09-13T14:48:41.172 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.172 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.172 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.172 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271833+0000", 2024-09-13T14:48:41.172 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5077000000000003e-05 2024-09-13T14:48:41.172 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.172 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.172 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.172 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.172 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.172 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156611 6.3 6.ccd62063 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-13T14:48:41.173 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.006819+0000", 2024-09-13T14:48:41.173 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 129.10653338500001, 2024-09-13T14:48:41.173 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.30103220400000003, 2024-09-13T14:48:41.173 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.173 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:48:41.173 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:41.173 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:41.173 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:41.173 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156611 2024-09-13T14:48:41.173 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.173 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.173 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.173 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.173 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006819+0000", 2024-09-13T14:48:41.174 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.174 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.174 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.174 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.174 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006819+0000", 2024-09-13T14:48:41.174 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.174 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.174 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.174 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.174 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006816+0000", 2024-09-13T14:48:41.174 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999971 2024-09-13T14:48:41.174 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.174 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.175 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.175 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006831+0000", 2024-09-13T14:48:41.175 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5e-05 2024-09-13T14:48:41.175 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.175 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.175 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.175 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006848+0000", 2024-09-13T14:48:41.175 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6429000000000001e-05 2024-09-13T14:48:41.175 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.175 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.175 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.175 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006856+0000", 2024-09-13T14:48:41.175 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8420000000000005e-06 2024-09-13T14:48:41.175 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.176 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.176 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.176 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.307840+0000", 2024-09-13T14:48:41.176 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.30098485899999999 2024-09-13T14:48:41.176 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.176 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.176 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.176 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.307851+0000", 2024-09-13T14:48:41.176 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0763e-05 2024-09-13T14:48:41.176 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.176 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.176 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.176 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.177 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.177 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:156604 6.9 e173/169)", 2024-09-13T14:48:41.177 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.013215+0000", 2024-09-13T14:48:41.177 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 129.10013766399999, 2024-09-13T14:48:41.177 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.26933052299999999, 2024-09-13T14:48:41.177 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.177 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:48:41.177 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.177 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.177 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.177 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013215+0000", 2024-09-13T14:48:41.177 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.177 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.177 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.178 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.178 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013215+0000", 2024-09-13T14:48:41.178 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.178 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.178 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.178 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.178 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013214+0000", 2024-09-13T14:48:41.178 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:48:41.178 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.178 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.178 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.178 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013223+0000", 2024-09-13T14:48:41.178 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.0750000000000004e-06 2024-09-13T14:48:41.178 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.179 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.179 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.179 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013236+0000", 2024-09-13T14:48:41.179 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2925000000000001e-05 2024-09-13T14:48:41.179 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.179 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.179 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.179 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013242+0000", 2024-09-13T14:48:41.179 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0839999999999997e-06 2024-09-13T14:48:41.179 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.179 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.179 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.179 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282529+0000", 2024-09-13T14:48:41.179 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.26928756599999998 2024-09-13T14:48:41.180 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.180 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.180 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.180 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282545+0000", 2024-09-13T14:48:41.180 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5888000000000001e-05 2024-09-13T14:48:41.180 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.180 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.180 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.180 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.180 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.180 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:156612 6.9 e173/169)", 2024-09-13T14:48:41.180 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.023006+0000", 2024-09-13T14:48:41.180 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 129.09034606200001, 2024-09-13T14:48:41.181 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.25955990099999998, 2024-09-13T14:48:41.181 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.181 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:48:41.181 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.181 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.181 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.181 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023006+0000", 2024-09-13T14:48:41.181 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.181 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.181 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.181 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.181 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023006+0000", 2024-09-13T14:48:41.181 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.181 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.182 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.182 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.182 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023004+0000", 2024-09-13T14:48:41.182 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999976 2024-09-13T14:48:41.182 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.182 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.182 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.182 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023023+0000", 2024-09-13T14:48:41.182 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9327999999999999e-05 2024-09-13T14:48:41.182 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.182 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.182 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.182 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023054+0000", 2024-09-13T14:48:41.182 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0972000000000001e-05 2024-09-13T14:48:41.183 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.183 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.183 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.183 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023064+0000", 2024-09-13T14:48:41.183 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0198e-05 2024-09-13T14:48:41.183 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.183 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.183 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.183 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282559+0000", 2024-09-13T14:48:41.183 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.25949471600000001 2024-09-13T14:48:41.183 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.183 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.183 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.184 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282566+0000", 2024-09-13T14:48:41.184 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1570000000000003e-06 2024-09-13T14:48:41.184 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.184 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.184 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.184 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.184 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.184 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156611 6.3 6:c6046b33:::benchmark_data_smithi096_167087_object39383:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:48:41.184 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.023013+0000", 2024-09-13T14:48:41.184 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 129.09033935299999, 2024-09-13T14:48:41.184 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0343409539999999, 2024-09-13T14:48:41.184 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.184 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:41.185 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:41.185 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:41.185 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:41.185 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156611 2024-09-13T14:48:41.185 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.185 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.185 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.185 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.185 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023013+0000", 2024-09-13T14:48:41.185 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.185 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.185 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.185 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.186 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023013+0000", 2024-09-13T14:48:41.186 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.186 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.186 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.186 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.186 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023004+0000", 2024-09-13T14:48:41.186 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999909 2024-09-13T14:48:41.186 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.186 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.186 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.186 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023024+0000", 2024-09-13T14:48:41.186 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9893000000000001e-05 2024-09-13T14:48:41.186 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.186 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.187 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.187 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.039994+0000", 2024-09-13T14:48:41.187 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.016970180000000001 2024-09-13T14:48:41.187 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.187 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.187 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.187 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040019+0000", 2024-09-13T14:48:41.187 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4919e-05 2024-09-13T14:48:41.187 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.187 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.187 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.187 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.308192+0000", 2024-09-13T14:48:41.187 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.26817356999999997 2024-09-13T14:48:41.187 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.188 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.188 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:41.188 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.308207+0000", 2024-09-13T14:48:41.188 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4960000000000001e-05 2024-09-13T14:48:41.188 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.188 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.188 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.188 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.043149+0000", 2024-09-13T14:48:41.188 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.73494118500000005 2024-09-13T14:48:41.188 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.188 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.188 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.188 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.043279+0000", 2024-09-13T14:48:41.189 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013044200000000001 2024-09-13T14:48:41.189 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.189 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.189 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:48:41.189 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.043839+0000", 2024-09-13T14:48:41.189 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00056042099999999997 2024-09-13T14:48:41.189 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.189 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.189 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:41.189 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.047415+0000", 2024-09-13T14:48:41.189 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035758959999999998 2024-09-13T14:48:41.189 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.189 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.189 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.190 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.052702+0000", 2024-09-13T14:48:41.190 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0052870019999999998 2024-09-13T14:48:41.190 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.190 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.190 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.190 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.057268+0000", 2024-09-13T14:48:41.190 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0045653200000000003 2024-09-13T14:48:41.190 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.190 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.190 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:41.190 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.057316+0000", 2024-09-13T14:48:41.190 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8798000000000003e-05 2024-09-13T14:48:41.190 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.190 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.191 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.191 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.057354+0000", 2024-09-13T14:48:41.191 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7571999999999999e-05 2024-09-13T14:48:41.191 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.191 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.191 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.191 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.191 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.191 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156616 6.15 6:ac5310a6:::benchmark_data_smithi096_167087_object39388:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:48:41.191 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.040042+0000", 2024-09-13T14:48:41.191 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 129.07331071499999, 2024-09-13T14:48:41.191 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1222388599999999, 2024-09-13T14:48:41.191 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.192 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:41.192 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:41.192 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:41.192 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:41.192 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156616 2024-09-13T14:48:41.192 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.192 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.192 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.192 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.192 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040042+0000", 2024-09-13T14:48:41.192 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.192 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.192 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.193 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.193 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040042+0000", 2024-09-13T14:48:41.193 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.193 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.193 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.193 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.193 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040039+0000", 2024-09-13T14:48:41.193 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999976 2024-09-13T14:48:41.193 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.193 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.195 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.195 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040051+0000", 2024-09-13T14:48:41.195 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2446000000000001e-05 2024-09-13T14:48:41.195 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.195 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.195 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.195 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040068+0000", 2024-09-13T14:48:41.195 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6059e-05 2024-09-13T14:48:41.195 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.195 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.195 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.195 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040072+0000", 2024-09-13T14:48:41.196 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2370000000000003e-06 2024-09-13T14:48:41.196 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.196 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.196 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.196 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.309500+0000", 2024-09-13T14:48:41.196 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.26942850299999999 2024-09-13T14:48:41.196 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.196 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.196 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:41.196 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.309506+0000", 2024-09-13T14:48:41.196 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.049e-06 2024-09-13T14:48:41.196 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.196 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.196 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.197 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.158349+0000", 2024-09-13T14:48:41.197 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.84884293700000002 2024-09-13T14:48:41.197 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.197 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.197 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.197 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.158451+0000", 2024-09-13T14:48:41.197 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000101679 2024-09-13T14:48:41.197 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.197 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.197 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:48:41.197 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.158782+0000", 2024-09-13T14:48:41.197 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00033148900000000002 2024-09-13T14:48:41.197 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.197 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.198 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:41.198 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.160769+0000", 2024-09-13T14:48:41.198 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0019864119999999999 2024-09-13T14:48:41.198 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.198 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.198 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.198 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.161676+0000", 2024-09-13T14:48:41.198 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00090709000000000004 2024-09-13T14:48:41.198 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.199 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.199 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.199 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.162206+0000", 2024-09-13T14:48:41.199 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00053007 2024-09-13T14:48:41.199 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.199 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.199 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:41.199 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.162249+0000", 2024-09-13T14:48:41.199 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3250000000000001e-05 2024-09-13T14:48:41.199 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.199 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.199 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.199 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.162281+0000", 2024-09-13T14:48:41.199 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1241000000000001e-05 2024-09-13T14:48:41.200 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.200 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.200 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.200 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.200 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.200 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156618 6.8 6:13ebd03e:::benchmark_data_smithi096_167087_object39390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:48:41.200 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.040084+0000", 2024-09-13T14:48:41.200 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 129.07326841, 2024-09-13T14:48:41.200 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.023429146, 2024-09-13T14:48:41.200 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.200 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:41.200 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:41.200 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:41.201 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:41.201 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156618 2024-09-13T14:48:41.201 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.201 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.201 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.201 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.201 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040084+0000", 2024-09-13T14:48:41.201 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.201 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.201 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.201 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.201 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040084+0000", 2024-09-13T14:48:41.201 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.201 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.202 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.202 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.202 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040084+0000", 2024-09-13T14:48:41.202 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:48:41.202 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.202 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.202 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.202 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040087+0000", 2024-09-13T14:48:41.202 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0809999999999998e-06 2024-09-13T14:48:41.202 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.202 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.202 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.202 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040095+0000", 2024-09-13T14:48:41.202 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9529999999999999e-06 2024-09-13T14:48:41.203 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.203 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.203 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.203 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040098+0000", 2024-09-13T14:48:41.203 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4599999999999999e-06 2024-09-13T14:48:41.203 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.203 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.203 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.203 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.304957+0000", 2024-09-13T14:48:41.203 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.26485901699999997 2024-09-13T14:48:41.203 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.203 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.203 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:41.204 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.304969+0000", 2024-09-13T14:48:41.204 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1610999999999999e-05 2024-09-13T14:48:41.204 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.204 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.204 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.204 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.046503+0000", 2024-09-13T14:48:41.204 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.74153483200000003 2024-09-13T14:48:41.204 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.204 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.204 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.204 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.046619+0000", 2024-09-13T14:48:41.204 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000115473 2024-09-13T14:48:41.204 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.205 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.205 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:48:41.205 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.047071+0000", 2024-09-13T14:48:41.205 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00045168200000000002 2024-09-13T14:48:41.205 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.205 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.205 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:41.205 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.049454+0000", 2024-09-13T14:48:41.205 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002383399 2024-09-13T14:48:41.205 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.205 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.205 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.205 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.057281+0000", 2024-09-13T14:48:41.205 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0078272889999999994 2024-09-13T14:48:41.206 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.206 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.206 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.206 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.063435+0000", 2024-09-13T14:48:41.206 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0061532130000000003 2024-09-13T14:48:41.206 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.206 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.206 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:41.206 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.063479+0000", 2024-09-13T14:48:41.206 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4530000000000002e-05 2024-09-13T14:48:41.206 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.206 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.206 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.206 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.063513+0000", 2024-09-13T14:48:41.207 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4041999999999997e-05 2024-09-13T14:48:41.207 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.207 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.207 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.207 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.207 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.207 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:156614 6.1 e173/169)", 2024-09-13T14:48:41.207 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.040126+0000", 2024-09-13T14:48:41.207 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 129.073226441, 2024-09-13T14:48:41.207 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.24246019299999999, 2024-09-13T14:48:41.207 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.207 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:48:41.207 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.207 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.208 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.208 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040126+0000", 2024-09-13T14:48:41.208 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.208 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.208 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.208 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.208 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040126+0000", 2024-09-13T14:48:41.208 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.208 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.208 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.208 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.208 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040125+0000", 2024-09-13T14:48:41.208 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:41.208 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.209 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.209 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.209 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040134+0000", 2024-09-13T14:48:41.209 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.3610000000000007e-06 2024-09-13T14:48:41.209 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.209 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.209 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.209 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040144+0000", 2024-09-13T14:48:41.209 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0173000000000001e-05 2024-09-13T14:48:41.209 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.209 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.209 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.209 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040148+0000", 2024-09-13T14:48:41.210 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6169999999999998e-06 2024-09-13T14:48:41.210 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.210 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.210 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.210 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282579+0000", 2024-09-13T14:48:41.210 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.242431116 2024-09-13T14:48:41.210 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.210 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.210 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.210 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282586+0000", 2024-09-13T14:48:41.210 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.216e-06 2024-09-13T14:48:41.210 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.210 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.210 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.211 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.211 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.211 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156624 6.13 6:c852d4ec:::benchmark_data_smithi096_167087_object39396:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:48:41.211 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.372301+0000", 2024-09-13T14:48:41.211 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 128.741051763, 2024-09-13T14:48:41.211 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.7720032, 2024-09-13T14:48:41.211 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.211 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:41.211 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:41.211 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:41.211 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:41.211 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156624 2024-09-13T14:48:41.211 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.212 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.212 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.212 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.212 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372301+0000", 2024-09-13T14:48:41.212 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.212 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.212 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.212 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.212 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372301+0000", 2024-09-13T14:48:41.212 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.212 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.212 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.212 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.212 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372299+0000", 2024-09-13T14:48:41.213 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:41.213 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.213 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.213 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.213 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372306+0000", 2024-09-13T14:48:41.213 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.221e-06 2024-09-13T14:48:41.213 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.213 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.213 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.213 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372320+0000", 2024-09-13T14:48:41.213 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3293e-05 2024-09-13T14:48:41.213 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.213 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.213 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.214 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372325+0000", 2024-09-13T14:48:41.214 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7089999999999999e-06 2024-09-13T14:48:41.214 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.214 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.214 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.214 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372366+0000", 2024-09-13T14:48:41.214 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0308000000000001e-05 2024-09-13T14:48:41.214 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.214 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.214 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:41.214 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372372+0000", 2024-09-13T14:48:41.214 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8179999999999998e-06 2024-09-13T14:48:41.214 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.215 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.215 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.215 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.070086+0000", 2024-09-13T14:48:41.215 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.69771321799999997 2024-09-13T14:48:41.215 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.215 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.215 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.215 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.070229+0000", 2024-09-13T14:48:41.215 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014322 2024-09-13T14:48:41.215 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.215 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.215 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:48:41.215 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.070517+0000", 2024-09-13T14:48:41.216 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00028809400000000002 2024-09-13T14:48:41.216 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.216 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.216 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:41.216 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.072169+0000", 2024-09-13T14:48:41.216 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0016519390000000001 2024-09-13T14:48:41.216 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.216 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.216 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.216 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.073034+0000", 2024-09-13T14:48:41.216 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00086496100000000003 2024-09-13T14:48:41.216 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.216 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.217 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.217 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.144218+0000", 2024-09-13T14:48:41.217 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.071183739999999995 2024-09-13T14:48:41.217 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.217 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.217 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:41.217 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.144268+0000", 2024-09-13T14:48:41.217 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0076e-05 2024-09-13T14:48:41.217 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.217 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.217 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.217 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.144304+0000", 2024-09-13T14:48:41.217 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6125999999999999e-05 2024-09-13T14:48:41.217 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.218 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.218 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.218 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.218 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.218 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:182590 6.15 6:aa32eba5:::benchmark_data_smithi096_167087_object65362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2024-09-13T14:48:41.218 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:19.162570+0000", 2024-09-13T14:48:41.218 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 81.950782541999999, 2024-09-13T14:48:41.218 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95580658399999996, 2024-09-13T14:48:41.218 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.219 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:41.219 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:41.219 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:41.219 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:41.219 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 182590 2024-09-13T14:48:41.219 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.219 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.219 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.219 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.219 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162570+0000", 2024-09-13T14:48:41.219 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.219 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.219 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.220 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.220 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162570+0000", 2024-09-13T14:48:41.220 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.220 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.220 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.220 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.220 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162569+0000", 2024-09-13T14:48:41.220 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:48:41.220 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.220 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.220 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.220 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162574+0000", 2024-09-13T14:48:41.220 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3739999999999997e-06 2024-09-13T14:48:41.220 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.221 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.221 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.221 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162587+0000", 2024-09-13T14:48:41.221 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2448e-05 2024-09-13T14:48:41.221 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.221 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.221 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.221 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162593+0000", 2024-09-13T14:48:41.221 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.669e-06 2024-09-13T14:48:41.221 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.221 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.221 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.221 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162634+0000", 2024-09-13T14:48:41.222 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0627000000000003e-05 2024-09-13T14:48:41.222 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.222 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.222 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:41.222 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162642+0000", 2024-09-13T14:48:41.222 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2460000000000003e-06 2024-09-13T14:48:41.222 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.222 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.222 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.222 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.113644+0000", 2024-09-13T14:48:41.222 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95100172500000002 2024-09-13T14:48:41.222 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.222 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.222 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.223 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.113728+0000", 2024-09-13T14:48:41.223 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4036000000000007e-05 2024-09-13T14:48:41.223 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.223 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.223 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:48:41.223 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.113982+0000", 2024-09-13T14:48:41.223 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00025374799999999998 2024-09-13T14:48:41.223 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.223 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.223 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.223 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.116230+0000", 2024-09-13T14:48:41.223 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002248613 2024-09-13T14:48:41.223 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.223 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.224 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:41.224 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118234+0000", 2024-09-13T14:48:41.224 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0020033669999999998 2024-09-13T14:48:41.224 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.224 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.224 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.224 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118305+0000", 2024-09-13T14:48:41.224 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1443e-05 2024-09-13T14:48:41.224 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.224 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.224 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:41.224 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118344+0000", 2024-09-13T14:48:41.224 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8488999999999997e-05 2024-09-13T14:48:41.225 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.225 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.225 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.225 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118376+0000", 2024-09-13T14:48:41.225 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2659e-05 2024-09-13T14:48:41.225 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.225 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.225 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.225 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.225 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.225 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:182611 6.15 6:ae0b3e38:::benchmark_data_smithi096_167087_object65383:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2024-09-13T14:48:41.225 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:19.183753+0000", 2024-09-13T14:48:41.225 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 81.929599178999993, 2024-09-13T14:48:41.226 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93487196400000006, 2024-09-13T14:48:41.226 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:48:41.226 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:41.226 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:48:41.226 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:48:41.226 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:48:41.226 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 182611 2024-09-13T14:48:41.226 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.226 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:48:41.226 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.226 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:48:41.226 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183753+0000", 2024-09-13T14:48:41.226 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.226 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.227 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.227 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:48:41.227 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183753+0000", 2024-09-13T14:48:41.227 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:48:41.227 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.227 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.227 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:48:41.227 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183752+0000", 2024-09-13T14:48:41.227 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:48:41.227 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.227 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.227 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:48:41.227 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183757+0000", 2024-09-13T14:48:41.228 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0479999999999998e-06 2024-09-13T14:48:41.228 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.228 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.228 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:48:41.228 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183767+0000", 2024-09-13T14:48:41.228 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7410000000000004e-06 2024-09-13T14:48:41.228 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.228 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.228 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:48:41.228 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183771+0000", 2024-09-13T14:48:41.228 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1799999999999998e-06 2024-09-13T14:48:41.228 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.228 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.229 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.229 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183805+0000", 2024-09-13T14:48:41.229 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4193000000000003e-05 2024-09-13T14:48:41.229 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.229 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.229 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:48:41.229 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183812+0000", 2024-09-13T14:48:41.229 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1349999999999997e-06 2024-09-13T14:48:41.229 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.229 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.229 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:48:41.229 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.116254+0000", 2024-09-13T14:48:41.229 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93244249599999995 2024-09-13T14:48:41.229 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.230 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.230 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:48:41.230 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.116358+0000", 2024-09-13T14:48:41.230 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000104342 2024-09-13T14:48:41.230 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.230 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.230 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:48:41.230 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.116545+0000", 2024-09-13T14:48:41.230 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000186152 2024-09-13T14:48:41.230 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.230 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.230 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.230 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118435+0000", 2024-09-13T14:48:41.231 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0018904519999999999 2024-09-13T14:48:41.231 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.231 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.231 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:48:41.231 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118536+0000", 2024-09-13T14:48:41.231 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000101067 2024-09-13T14:48:41.231 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.231 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.231 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:41.231 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118571+0000", 2024-09-13T14:48:41.231 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4502000000000003e-05 2024-09-13T14:48:41.231 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.231 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.232 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:48:41.232 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118608+0000", 2024-09-13T14:48:41.232 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7051999999999997e-05 2024-09-13T14:48:41.232 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:48:41.232 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:48:41.232 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:48:41.232 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118625+0000", 2024-09-13T14:48:41.232 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7555999999999999e-05 2024-09-13T14:48:41.232 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.232 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.232 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.232 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:48:41.232 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:48:41.233 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:48:41.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.233 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:41.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:41.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:41 smithi096 ceph-mon[98901]: pgmap v634: 99 pgs: 99 active+clean; 7.5 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 528 KiB/s wr, 10 op/s; 719 KiB/s, 7 objects/s recovering 2024-09-13T14:48:41.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:41 smithi096 ceph-mon[102928]: pgmap v634: 99 pgs: 99 active+clean; 7.5 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 528 KiB/s wr, 10 op/s; 719 KiB/s, 7 objects/s recovering 2024-09-13T14:48:41.335 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:41.372 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:41.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:41.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:41 smithi204 ceph-mon[87071]: pgmap v634: 99 pgs: 99 active+clean; 7.5 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 528 KiB/s wr, 10 op/s; 719 KiB/s, 7 objects/s recovering 2024-09-13T14:48:41.473 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:41.506 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:41.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:41.608 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:41.645 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:41.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:41.703 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 2 16 6967 6951 217.201 208.438 0.00271742 0.00459696 2024-09-13T14:48:41.747 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:41.784 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:41.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:41.886 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:41.923 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:41.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:42.025 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:42.029 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:48:42.056 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:48:42.063 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:42.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:42.165 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:42.196 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:42.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:42.298 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:42.374 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:42.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:42.476 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:42.529 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:42.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:42.631 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:42.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.675 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:42.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:42.703 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 3 16 9532 9516 198.233 160.312 0.00697218 0.00503628 2024-09-13T14:48:42.776 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:42.794 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:48:42.795 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:48:42.804 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:42.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:42.829 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:48:42.829 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:48:42.863 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:48:42.864 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:48:42.897 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:48:42.897 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:48:42.906 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:42.929 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:48:42.930 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:48:42.948 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:42.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:42.963 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:48:42.963 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:48:42.998 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:48:42.998 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:48:43.030 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:48:43.030 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:48:43.050 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:43.076 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:43.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:43.178 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:43.207 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:43.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:43.309 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:43.340 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:43.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:43.442 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:43.477 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:43.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:43.563 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:43 smithi096 ceph-mon[98901]: pgmap v635: 99 pgs: 99 active+clean; 141 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s wr, 328 op/s; 611 KiB/s, 6 objects/s recovering 2024-09-13T14:48:43.563 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:43 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3081653250' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:43.563 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:43 smithi096 ceph-mon[102928]: pgmap v635: 99 pgs: 99 active+clean; 141 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s wr, 328 op/s; 611 KiB/s, 6 objects/s recovering 2024-09-13T14:48:43.563 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:43 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3081653250' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:43.580 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:43.618 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:43.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:43.703 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 4 16 12444 12428 194.171 182 0.00357838 0.00514527 2024-09-13T14:48:43.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:43 smithi204 ceph-mon[87071]: pgmap v635: 99 pgs: 99 active+clean; 141 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s wr, 328 op/s; 611 KiB/s, 6 objects/s recovering 2024-09-13T14:48:43.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:43 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3081653250' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:43.720 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:43.751 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:43.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:43.853 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:43.891 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:43.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:43.993 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:44.032 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:44.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:44.045 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.1 reset_purged_snaps_last 2024-09-13T14:48:44.133 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:44.160 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:44.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:44.262 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:44.291 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:44.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:44.393 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:44.437 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:44.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:44.539 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:44.570 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:44.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:44.672 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:44.703 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 5 16 15116 15100 188.733 167 0.00370507 0.00529241 2024-09-13T14:48:44.709 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:44.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:44.811 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:44.845 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:44.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:44.947 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:44.985 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:44.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:45.087 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:45.116 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:45.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:45.218 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:45.236 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:45.239 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:45.242 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:45.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:45.253 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:45.344 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:45.375 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:45 smithi096 ceph-mon[98901]: pgmap v636: 99 pgs: 99 active+clean; 519 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 512 KiB/s, 5 objects/s recovering 2024-09-13T14:48:45.376 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:45 smithi096 ceph-mon[102928]: pgmap v636: 99 pgs: 99 active+clean; 519 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 512 KiB/s, 5 objects/s recovering 2024-09-13T14:48:45.389 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:45.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:45.490 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:45.519 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:45.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:45.621 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:45.662 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:45.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:45.703 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 6 16 17655 17639 183.723 158.688 0.00327107 0.00543648 2024-09-13T14:48:45.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:45 smithi204 ceph-mon[87071]: pgmap v636: 99 pgs: 99 active+clean; 519 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 512 KiB/s, 5 objects/s recovering 2024-09-13T14:48:45.764 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:45.803 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:45.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:45.905 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:45.940 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:45.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:46.042 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:46.072 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:46.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:46.174 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:46.216 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:46.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:46.318 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:46.346 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:46.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:46.448 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:46.479 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:46.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:46.580 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:46.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.614 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:46.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:46.703 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 7 16 20308 20292 181.163 165.812 0.00357892 0.00551202 2024-09-13T14:48:46.715 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:46.743 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:46.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:46.845 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:46.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.885 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:46.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:46.987 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:47.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.021 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:47.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:47.122 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:47.152 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:47.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:47.167 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:48:47.254 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:47.281 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:47.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:47.285 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:47 smithi096 ceph-mon[98901]: pgmap v637: 99 pgs: 99 active+clean; 637 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 0 B/s, 1 objects/s recovering 2024-09-13T14:48:47.285 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:47 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:47.285 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:47 smithi096 ceph-mon[102928]: pgmap v637: 99 pgs: 99 active+clean; 637 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 0 B/s, 1 objects/s recovering 2024-09-13T14:48:47.286 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:47 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:47.383 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:47.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.423 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:47.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:47.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:47 smithi204 ceph-mon[87071]: pgmap v637: 99 pgs: 99 active+clean; 637 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 0 B/s, 1 objects/s recovering 2024-09-13T14:48:47.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:47 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:47.524 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:47.549 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:47.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:47.651 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:47.691 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:47.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:47.749 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 8 16 22633 22617 176.68 145.312 0.00652734 0.00565319 2024-09-13T14:48:47.850 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:47.879 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:47.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:47.981 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:48.012 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:48.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:48.113 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:48.144 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:48.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:48.248 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:48.284 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:48.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:48.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:48:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:48:47] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.51.0" 2024-09-13T14:48:48.386 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:48.420 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:48.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:48.521 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:48.559 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:48.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:48.661 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:48.697 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:48.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:48.703 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 9 16 24975 24959 173.311 146.375 0.00576144 0.00576334 2024-09-13T14:48:48.799 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:48.838 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:48.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:48.940 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:48.976 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:48.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:49.078 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:49.110 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:49.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:49.212 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:49.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.250 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:49.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:49.275 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:49 smithi096 ceph-mon[98901]: pgmap v638: 99 pgs: 99 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-09-13T14:48:49.276 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:49 smithi096 ceph-mon[102928]: pgmap v638: 99 pgs: 99 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-09-13T14:48:49.350 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:49.379 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:49.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:49 smithi204 ceph-mon[87071]: pgmap v638: 99 pgs: 99 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-09-13T14:48:49.481 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:49.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.524 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:49.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:49.625 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:49.640 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:49.640 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:49.651 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:49.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:49.703 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 10 16 27284 27268 170.41 144.312 0.00491519 0.00586326 2024-09-13T14:48:49.753 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:49.782 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:49.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:49.884 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:49.925 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:49.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:50.027 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:50.057 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:50.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:50.159 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:50.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.198 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:50.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:50.299 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:50.328 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:50.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:50.430 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:50.454 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:50.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:50.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:48:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:48:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:50.556 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:50.593 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:50.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:50.696 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:50.703 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 11 16 29306 29290 166.406 126.375 0.00423713 0.00600452 2024-09-13T14:48:50.726 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:50.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:50.828 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:50.867 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:50.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:50.969 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:51.010 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:51.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:51.112 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:51.142 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:51.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:51.245 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:51.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.282 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:51.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:51 smithi096 ceph-mon[98901]: pgmap v639: 99 pgs: 99 active+clean; 1.4 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2024-09-13T14:48:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:51 smithi096 ceph-mon[102928]: pgmap v639: 99 pgs: 99 active+clean; 1.4 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2024-09-13T14:48:51.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:51.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:51.383 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:51.414 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:51.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:51 smithi204 ceph-mon[87071]: pgmap v639: 99 pgs: 99 active+clean; 1.4 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2024-09-13T14:48:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:48:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:51.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:51.548 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:51.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:51.650 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:51.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.687 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:51.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:51.703 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 12 16 30994 30978 161.33 105.5 0.00526339 0.00619281 2024-09-13T14:48:51.789 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:51.816 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:51.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:51.918 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:51.957 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:51.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:52.060 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:52.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.100 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:52.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:52.201 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:52.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.229 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:52.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:52.332 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:52.361 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:52.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:52.463 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:52.500 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:52.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:52.602 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:52.633 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:52.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:52.704 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 13 16 32709 32693 157.164 107.188 0.00641644 0.00635371 2024-09-13T14:48:52.735 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:52.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.774 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:52.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:52.876 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:52.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.910 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:52.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:53.011 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:53.033 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:53.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:53.135 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:53.166 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:53.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:53.268 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:53.307 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:53.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:53.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:53 smithi096 ceph-mon[98901]: pgmap v640: 99 pgs: 99 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-09-13T14:48:53.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:53 smithi096 ceph-mon[102928]: pgmap v640: 99 pgs: 99 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-09-13T14:48:53.409 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:53.439 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:53.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:53.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:53 smithi204 ceph-mon[87071]: pgmap v640: 99 pgs: 99 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-09-13T14:48:53.541 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:53.576 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:53.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:53.678 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:53.704 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 14 16 34366 34350 153.335 103.562 0.00752341 0.0065167 2024-09-13T14:48:53.710 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:53.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:53.812 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:53.833 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:53.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:53.935 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:53.956 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:53.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:54.058 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:54.089 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:54.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:54.191 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:54.219 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:54.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:54.321 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:54.342 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:54.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:54.444 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:54.473 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:54.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:54.574 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:54.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.600 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:54.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:54.702 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:54.704 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 15 16 35679 35663 148.583 82.0625 0.00897073 0.00672496 2024-09-13T14:48:54.731 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:54.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:54.833 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:54.871 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:54.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:54.973 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:54.994 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:54.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:55.096 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:55.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.133 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:55.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:55.234 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:55.270 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:55.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:55.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:55 smithi096 ceph-mon[98901]: pgmap v641: 99 pgs: 99 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-09-13T14:48:55.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:55 smithi096 ceph-mon[102928]: pgmap v641: 99 pgs: 99 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-09-13T14:48:55.373 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:55.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.396 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:55.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:55 smithi204 ceph-mon[87071]: pgmap v641: 99 pgs: 99 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-09-13T14:48:55.497 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:55.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.527 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:55.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:55.628 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:55.657 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:55.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:55.704 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 16 16 37329 37313 145.741 103.125 0.0157205 0.00685565 2024-09-13T14:48:55.759 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:55.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.798 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:55.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:55.899 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:55.924 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:55.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:56.026 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:56.053 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:56.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:56.154 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:56.176 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:56.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:56.277 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:56.307 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:56.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:56.409 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:56.446 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:56.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:56.548 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:56.576 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:56.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:56.678 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:56.704 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:56.704 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 17 16 38589 38573 141.8 78.75 0.0125997 0.00704305 2024-09-13T14:48:56.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:56.806 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:56.843 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:56.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:56.945 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:56.982 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:56.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:57.043 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:48:57.084 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:57.119 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:57.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:57.221 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:57.241 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:57.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:57.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:57 smithi096 ceph-mon[98901]: pgmap v642: 99 pgs: 99 active+clean; 2.0 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-09-13T14:48:57.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:57 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3500412910' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:57.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:57 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:57.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:57 smithi096 ceph-mon[102928]: pgmap v642: 99 pgs: 99 active+clean; 2.0 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-09-13T14:48:57.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:57 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3500412910' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:57.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:57 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:57.343 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:57.371 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:57.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:57 smithi204 ceph-mon[87071]: pgmap v642: 99 pgs: 99 active+clean; 2.0 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-09-13T14:48:57.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:57 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3500412910' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:57.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:57 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:57.473 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:57.500 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:57.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:57.602 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:57.634 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:57.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:57.704 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 18 16 40432 40416 140.321 115.188 0.00581252 0.00712076 2024-09-13T14:48:57.736 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:57.764 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:57.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:57.866 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:57.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.897 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:57.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:57.998 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:58.038 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:58.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:58.140 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:58.179 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:58.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:58.281 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:58.284 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:58 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:58.284 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:58 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:58.284 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:58 smithi096 ceph-mon[98901]: osdmap e232: 8 total, 8 up, 8 in 2024-09-13T14:48:58.284 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:58 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:58.284 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:58 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:58.285 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:58 smithi096 ceph-mon[102928]: osdmap e232: 8 total, 8 up, 8 in 2024-09-13T14:48:58.285 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:48:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:48:57] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.51.0" 2024-09-13T14:48:58.319 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:58.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:58.421 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:58.453 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:58.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:58.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:58 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:58.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:58 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:58.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:58 smithi204 ceph-mon[87071]: osdmap e232: 8 total, 8 up, 8 in 2024-09-13T14:48:58.555 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:58.584 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:58.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:58.686 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:58.704 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 19 16 42313 42297 139.122 117.562 0.00660396 0.00718252 2024-09-13T14:48:58.724 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:58.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:58.825 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:58.854 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:58.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:58.955 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:58.982 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:48:58.984 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:58.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:59.086 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:59.127 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:59.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:59.228 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:59.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.274 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:59.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:59.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:48:59 smithi096 ceph-mon[102928]: pgmap v644: 99 pgs: 99 active+clean; 2.3 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-13T14:48:59.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:48:59 smithi096 ceph-mon[98901]: pgmap v644: 99 pgs: 99 active+clean; 2.3 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-13T14:48:59.375 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:59.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.406 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:59.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:48:59 smithi204 ceph-mon[87071]: pgmap v644: 99 pgs: 99 active+clean; 2.3 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-13T14:48:59.507 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:59.547 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:59.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:59.650 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:59.682 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:59.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:59.704 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:48:59.704675+0000 min lat: 0.00108557 max lat: 0.194678 avg lat: 0.007262 2024-09-13T14:48:59.705 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:48:59.705 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 20 16 44051 44035 137.596 108.625 0.00824303 0.007262 2024-09-13T14:48:59.784 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:59.821 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:48:59.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:59.923 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:59.952 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:48:59.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:59.980 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:00.054 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:00.055 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:00.079 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:00.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:00.181 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:00.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.226 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:00.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:00.327 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:00.357 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:00.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:00.459 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:49:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:49:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:00.495 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:00.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:00.597 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:00.629 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:00.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:00.705 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 21 16 45723 45707 136.019 104.5 0.00623634 0.00734678 2024-09-13T14:49:00.731 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:00.763 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:00.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:00.865 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:00.902 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:00.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:01.004 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:01.040 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:01.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:01.142 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:01.180 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:01.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:01.281 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:01.318 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:01.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:01.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:01 smithi096 ceph-mon[98901]: pgmap v645: 99 pgs: 99 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-13T14:49:01.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:01 smithi096 ceph-mon[102928]: pgmap v645: 99 pgs: 99 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-13T14:49:01.420 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:01.456 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:01.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:01.462 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:49:01.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:01 smithi204 ceph-mon[87071]: pgmap v645: 99 pgs: 99 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-13T14:49:01.558 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:01.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.600 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:01.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:01.701 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:01.709 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 22 16 47427 47411 134.648 106.5 0.00845471 0.00741999 2024-09-13T14:49:01.728 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.2 reset_purged_snaps_last 2024-09-13T14:49:01.762 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:01.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:01.865 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:01.894 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:01.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:01.996 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:02.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.043 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:02.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:02.143 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:02.168 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:02.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:02.270 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:02.298 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:02.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:02.400 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:02.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.437 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:02.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:02.538 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:02.576 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:02.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:02.677 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:02.706 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:02.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:02.709 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 23 16 49021 49005 133.125 99.625 0.0123628 0.00750661 2024-09-13T14:49:02.808 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:02.836 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:02.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:02.938 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:02.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.985 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:02.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:03.087 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:03.134 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:03.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:03.236 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:03.264 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:03.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:03.366 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:03.380 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:03 smithi096 ceph-mon[98901]: pgmap v646: 99 pgs: 99 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-09-13T14:49:03.380 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:03 smithi096 ceph-mon[102928]: pgmap v646: 99 pgs: 99 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-09-13T14:49:03.393 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:03.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:03.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:03 smithi204 ceph-mon[87071]: pgmap v646: 99 pgs: 99 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-09-13T14:49:03.495 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:03.531 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:03.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:03.601 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:03.633 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:03.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.679 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:03.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:03.711 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 24 16 50638 50622 131.782 101.062 0.0134584 0.00758338 2024-09-13T14:49:03.780 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:03.809 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:03.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:03.910 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:03.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.949 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:03.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:04.050 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:04.079 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:04.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:04.181 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:04.223 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:04.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:04.325 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:04.352 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:04.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:04.389 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:49:04.389 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:49:04.425 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:49:04.426 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:49:04.453 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:04.462 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:49:04.463 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:49:04.487 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:04.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:04.504 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:49:04.504 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:49:04.535 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:49:04.535 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:49:04.563 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:49:04.564 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:49:04.589 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:04.594 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:49:04.594 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:49:04.624 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:04.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:04.625 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:49:04.626 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:49:04.711 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 25 16 52144 52128 130.275 94.125 0.00627623 0.00767006 2024-09-13T14:49:04.727 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:04.760 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:04.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:04.862 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:04.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.898 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:04.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:04.999 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:05.043 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:05.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:05.146 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:05.176 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:05.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:05.278 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:05.308 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:05.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:05.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:05 smithi096 ceph-mon[98901]: pgmap v647: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T14:49:05.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:05 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3619955773' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:05.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:05 smithi096 ceph-mon[102928]: pgmap v647: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T14:49:05.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:05 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3619955773' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:05.410 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:05.451 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:05.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:05.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:05 smithi204 ceph-mon[87071]: pgmap v647: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T14:49:05.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:05 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3619955773' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:05.553 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:05.591 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:05.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:05.693 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:05.712 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 26 16 53745 53729 129.107 100.062 0.0114779 0.00774037 2024-09-13T14:49:05.721 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:05.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:05.823 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:05.849 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:05.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:05.951 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:05.994 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:05.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:06.096 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:06.123 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:06.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:06.225 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:06.279 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:06.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:06.381 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:06.391 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:06.391 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:06.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.428 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:06.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:06.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:06.531 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:06.565 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:06.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:06.667 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:06.694 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:06.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:06.712 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 27 16 55164 55148 127.61 88.6875 0.0155298 0.00783052 2024-09-13T14:49:06.796 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:06.834 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:06.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:06.936 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:06.962 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:06.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:07.063 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:07.090 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:07.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:07.192 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:07.216 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:07.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:07.223 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:07 smithi096 ceph-mon[98901]: pgmap v648: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-13T14:49:07.223 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:07 smithi096 ceph-mon[102928]: pgmap v648: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-13T14:49:07.249 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:49:07.249 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:49:07.249 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:49:07.249 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:49:07.249 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:49:07.318 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:07.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.362 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:07.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:07.463 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:07.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:07 smithi204 ceph-mon[87071]: pgmap v648: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-13T14:49:07.501 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:07.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:07.605 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:07.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.644 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:07.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:07.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 28 16 56383 56367 125.774 76.1875 0.00618073 0.00794525 2024-09-13T14:49:07.745 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:07.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.787 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:07.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:07.888 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:07.914 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:07.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:08.016 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:08.043 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:08.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:08.145 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:08.165 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:49:08.173 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:08.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:08.201 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:49:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:49:07] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.51.0" 2024-09-13T14:49:08.275 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:08.318 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:08.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:08.420 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:08.449 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:08.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:08.551 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:08.575 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:08.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:08.677 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:08.706 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:08.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:08.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 29 16 57720 57704 124.318 83.5625 0.0109907 0.00803886 2024-09-13T14:49:08.808 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:08.846 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:08.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:08.910 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:08.948 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:08.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.994 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:08.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:09.095 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:09.125 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:09.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:09.226 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:09.270 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:09.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:09.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:09 smithi096 ceph-mon[98901]: pgmap v649: 99 pgs: 99 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-09-13T14:49:09.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:09 smithi096 ceph-mon[102928]: pgmap v649: 99 pgs: 99 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-09-13T14:49:09.372 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:09.406 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:09.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:09.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:09 smithi204 ceph-mon[87071]: pgmap v649: 99 pgs: 99 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-09-13T14:49:09.508 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:09.543 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:09.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:09.645 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:09.683 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:09.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:09.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 30 15 59188 59173 123.235 91.8125 0.0072857 0.00811001 2024-09-13T14:49:09.785 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:09.814 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:09.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:09.887 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:49:09.916 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:09.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.946 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:09.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:10.047 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:10.072 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:10.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:10.174 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:10.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.221 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:10.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:10.322 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:10.351 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:10.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:10.454 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:10.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:49:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:49:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:10.480 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:10.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:10.582 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:10.625 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:10.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:10.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 31 16 60503 60487 121.909 82.125 0.00874162 0.00819726 2024-09-13T14:49:10.727 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:10.770 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:10.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:10.872 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:10.913 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:10.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:11.015 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:11.041 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:11.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:11.062 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:11.115 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:49:11.143 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:11.168 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:11.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:11.270 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:11.297 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:11.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:11.399 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:11.430 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:11.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:11.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:11 smithi204 ceph-mon[87071]: pgmap v650: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-09-13T14:49:11.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:11 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3550885464' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:49:11.496 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:11 smithi096 ceph-mon[98901]: pgmap v650: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-09-13T14:49:11.496 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:11 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3550885464' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:49:11.496 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:11 smithi096 ceph-mon[102928]: pgmap v650: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-09-13T14:49:11.496 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:11 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3550885464' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:49:11.532 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:11.572 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:11.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:11.674 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:11.702 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:11.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:11.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 32 16 61878 61862 120.785 85.9375 0.0109407 0.00827403 2024-09-13T14:49:11.804 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:11.842 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:11.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:11.944 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:11.984 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:11.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:12.086 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:12.112 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:12.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:12.214 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:12.243 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:12.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:12.345 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:12.375 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:12.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:12.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:12 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:12.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:12 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3550885464' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:49:12.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:12 smithi204 ceph-mon[87071]: osdmap e233: 8 total, 8 up, 8 in 2024-09-13T14:49:12.478 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:12.517 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:12.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:12.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:12 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:12.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:12 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3550885464' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:49:12.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:12 smithi096 ceph-mon[98901]: osdmap e233: 8 total, 8 up, 8 in 2024-09-13T14:49:12.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:12 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:12.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:12 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3550885464' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:49:12.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:12 smithi096 ceph-mon[102928]: osdmap e233: 8 total, 8 up, 8 in 2024-09-13T14:49:12.620 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:12.652 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:12.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:12.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 33 16 63353 63337 119.918 92.1875 0.0109431 0.00833432 2024-09-13T14:49:12.725 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:12.754 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:12.801 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:12.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:12.903 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:12.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.938 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:12.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:13.039 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:13.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.069 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:13.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:13.171 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:13.199 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:13.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:13.301 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:13.338 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:13.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:13.440 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:13.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:13 smithi204 ceph-mon[87071]: pgmap v652: 99 pgs: 99 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:49:13.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.480 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:13.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:13.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:13 smithi096 ceph-mon[98901]: pgmap v652: 99 pgs: 99 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:49:13.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:13 smithi096 ceph-mon[102928]: pgmap v652: 99 pgs: 99 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:49:13.581 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:13.620 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:13.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:13.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 34 16 64732 64716 118.926 86.1875 0.011769 0.00840296 2024-09-13T14:49:13.723 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:13.761 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:13.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:13.863 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:13.892 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:13.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:13.994 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:14.031 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:14.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:14.133 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:14.170 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:14.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:14.272 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:14.295 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:14.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:14.397 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:14.426 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:14.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:14.528 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:14.586 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:14.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:14.688 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:14.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 35 16 66106 66090 117.982 85.875 0.00741216 0.00847061 2024-09-13T14:49:14.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.731 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:14.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:14.833 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:14.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.875 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:14.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:14.976 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:15.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.013 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:15.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:15.114 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:15.137 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:15.182 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:49:15.239 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:15.268 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:15.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:15.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:15 smithi096 ceph-mon[98901]: pgmap v653: 99 pgs: 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-09-13T14:49:15.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:15 smithi096 ceph-mon[102928]: pgmap v653: 99 pgs: 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-09-13T14:49:15.369 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:15.394 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:15.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:15.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:15 smithi204 ceph-mon[87071]: pgmap v653: 99 pgs: 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-09-13T14:49:15.496 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:15.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.544 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:15.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:15.645 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:15.678 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:15.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:15.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 36 16 67294 67278 116.767 74.25 0.00725876 0.00855897 2024-09-13T14:49:15.779 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:15.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.801 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:15.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:15.902 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:15.934 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:15.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:16.036 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:16.091 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:16.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:16.193 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:16.219 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:16.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:16.324 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:16.362 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:16.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:16.464 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:16.493 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:16.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:16.513 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:49:16.513 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:49:16.547 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:49:16.548 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:49:16.594 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:49:16.594 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:49:16.595 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:16.635 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:16.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:16.637 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:49:16.637 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:49:16.674 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:49:16.675 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:49:16.703 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:49:16.703 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:49:16.713 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 37 16 68489 68473 115.63 74.6875 0.00723474 0.00864267 2024-09-13T14:49:16.735 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:49:16.736 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:49:16.737 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:16.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.782 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:16.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:16.783 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:49:16.784 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:49:16.884 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:16.926 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:16.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:17.027 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:17.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.069 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:17.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:17.170 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:17.213 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:17.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:17.315 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:17.347 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:17.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:17.449 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:17.452 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:17 smithi096 ceph-mon[98901]: pgmap v654: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-09-13T14:49:17.453 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:17 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/694924681' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:17.453 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:17 smithi096 ceph-mon[102928]: pgmap v654: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-09-13T14:49:17.453 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:17 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/694924681' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:17.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:17 smithi204 ceph-mon[87071]: pgmap v654: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-09-13T14:49:17.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:17 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/694924681' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:17.486 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:17.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:17.588 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:17.628 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:17.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:17.714 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 38 16 69775 69759 114.702 80.375 0.0135344 0.00871276 2024-09-13T14:49:17.730 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:17.758 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:17.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:17.860 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:17.896 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:17.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:17.998 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:18.028 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:18.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:18.130 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:18.169 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:18.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:18.271 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:18.300 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:18.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:18.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:49:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:49:17] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.51.0" 2024-09-13T14:49:18.402 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:18.462 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.3 reset_purged_snaps_last 2024-09-13T14:49:18.492 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:18.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:18.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:18.618 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:18.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:18.714 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 39 16 71097 71081 113.88 82.625 0.00806362 0.00877619 2024-09-13T14:49:18.720 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:18.746 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:18.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:18.848 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:18.875 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:18.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:18.977 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:19.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.017 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:19.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:19.118 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:19.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.169 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:19.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:19.270 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:19.299 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:19.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:19.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:19 smithi096 ceph-mon[98901]: pgmap v655: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:49:19.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:19 smithi096 ceph-mon[102928]: pgmap v655: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:49:19.401 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:19.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.423 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:19.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:19.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:19 smithi204 ceph-mon[87071]: pgmap v655: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:49:19.524 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:19.566 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:19.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:19.668 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:19.702 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:19.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:19.714 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:49:19.713839+0000 min lat: 0.00108557 max lat: 0.194678 avg lat: 0.00884721 2024-09-13T14:49:19.714 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:49:19.714 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 40 16 72332 72316 112.962 77.1875 0.00798569 0.00884721 2024-09-13T14:49:19.805 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:19.833 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:19.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:19.935 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:19.972 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:20.074 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:20.102 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:20.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:20.204 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:20.241 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:20.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:20.343 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:20.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.381 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:20.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:49:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:49:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:20.482 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:20.528 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:20.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:20.629 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:20.666 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:20.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:20.714 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 41 16 73669 73653 112.245 83.5625 0.0133515 0.00890331 2024-09-13T14:49:20.767 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:20.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.843 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:20.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:20.944 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:20.998 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:20.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:21.100 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:21.126 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:21.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:21.227 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:21.256 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:21.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:21.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:21 smithi096 ceph-mon[98901]: pgmap v656: 99 pgs: 99 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T14:49:21.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:21.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:21 smithi096 ceph-mon[102928]: pgmap v656: 99 pgs: 99 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T14:49:21.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:21.357 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:21.358 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:21.369 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:21.382 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:21.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:21.397 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:49:21.397 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:49:21.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:21 smithi204 ceph-mon[87071]: pgmap v656: 99 pgs: 99 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T14:49:21.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:21.484 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:21.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.516 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:21.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:21.617 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:21.652 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:21.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:21.714 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 42 16 74854 74838 111.336 74.0625 0.00598362 0.00897659 2024-09-13T14:49:21.754 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:21.783 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:21.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:21.885 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:21.913 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:21.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:22.015 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:22.043 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:22.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:22.145 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:22.173 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:22.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:22.275 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:22.315 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:22.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:22.417 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:22.445 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:22.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:22.470 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:49:22.546 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:22.570 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:22.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:22.672 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:22.714 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 43 16 76099 76083 110.555 77.8125 0.0135635 0.00904014 2024-09-13T14:49:22.720 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:22.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:22.805 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:49:22.822 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:22.900 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:22.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:23.001 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:23.027 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:23.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:23.129 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:23.156 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:23.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:23.258 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:23.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.306 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:23.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:23.407 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:23.437 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:23.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:23.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:23 smithi204 ceph-mon[87071]: pgmap v657: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:49:23.539 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:23.566 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:23.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:23.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:23 smithi096 ceph-mon[98901]: pgmap v657: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:49:23.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:23 smithi096 ceph-mon[102928]: pgmap v657: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:49:23.669 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:23.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.713 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:23.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:23.714 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 44 16 77265 77249 109.699 72.875 0.0176914 0.00911031 2024-09-13T14:49:23.814 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:23.842 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:23.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:23.947 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:23.981 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:23.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:24.083 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:24.108 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:24.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:24.210 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:24.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.232 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:24.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:24.333 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:24.368 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:24.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:24.470 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:24.497 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:24.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:24.599 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:24.637 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:24.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:24.715 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 45 16 78329 78313 108.739 66.5 0.0158416 0.00918473 2024-09-13T14:49:24.739 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:24.773 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:24.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:24.875 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:24.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.923 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:24.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:25.024 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:25.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.057 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:25.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:25.159 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:25.196 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:25.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:25.297 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:25.327 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:25.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:25.429 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:25.456 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:25.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:25 smithi204 ceph-mon[87071]: pgmap v658: 99 pgs: 99 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-13T14:49:25.557 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:25.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:25 smithi096 ceph-mon[98901]: pgmap v658: 99 pgs: 99 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-13T14:49:25.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:25 smithi096 ceph-mon[102928]: pgmap v658: 99 pgs: 99 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-13T14:49:25.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.606 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:25.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:25.707 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:25.715 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 46 16 79384 79368 107.808 65.9375 0.012257 0.00927065 2024-09-13T14:49:25.737 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:25.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:25.839 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:25.870 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:25.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:25.972 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:26.003 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:26.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:26.105 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:26.135 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:26.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:26.161 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:26.236 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:26.250 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:26.296 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:26.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:26.398 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:26.440 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:26.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:26.542 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:26.571 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:26.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:26.673 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:26.702 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:26.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:26.715 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 47 16 80539 80523 107.05 72.1875 0.00804032 0.00933581 2024-09-13T14:49:26.804 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:26.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.837 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:26.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:26.938 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:26.968 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:26.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:27.071 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:27.112 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:27.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:27.214 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:27.241 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:27.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:27.343 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:27.381 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:27.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:27.436 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:27 smithi096 ceph-mon[98901]: pgmap v659: 99 pgs: 99 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-13T14:49:27.436 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:27 smithi096 ceph-mon[102928]: pgmap v659: 99 pgs: 99 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-13T14:49:27.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:27 smithi204 ceph-mon[87071]: pgmap v659: 99 pgs: 99 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-13T14:49:27.483 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:27.518 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:27.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:27.621 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:27.648 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:27.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:27.715 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 48 15 81693 81678 106.324 72.1875 0.013736 0.00939959 2024-09-13T14:49:27.751 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:27.778 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:27.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:27.880 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:27.920 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:27.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:28.022 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:28.048 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:28.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:28.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:49:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:49:27] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.51.0" 2024-09-13T14:49:28.150 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:28.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.195 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:28.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:28.202 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:49:28.296 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:28.330 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:28.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:28.432 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:28.460 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:28.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:28.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:28 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2884580549' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:28.562 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:28.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:28 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2884580549' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:28.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:28 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2884580549' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:28.591 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:28.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:28.693 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:28.715 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 49 16 82833 82817 105.607 71.1875 0.0165815 0.0094644 2024-09-13T14:49:28.725 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:28.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:28.827 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:28.866 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:28.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:28.968 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:28.991 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:28.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:29.093 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:29.122 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:29.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:29.223 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:29.263 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:29.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:29.365 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:29.406 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:29.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:29 smithi204 ceph-mon[87071]: pgmap v660: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-13T14:49:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:29 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:49:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:29 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:49:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:29 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2884580549' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:49:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:29 smithi204 ceph-mon[87071]: osdmap e234: 8 total, 8 up, 8 in 2024-09-13T14:49:29.509 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:29.542 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:29.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:29.562 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:29 smithi096 ceph-mon[98901]: pgmap v660: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-13T14:49:29.562 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:29 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:49:29.562 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:29 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:49:29.562 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:29 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2884580549' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:49:29.562 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:29 smithi096 ceph-mon[98901]: osdmap e234: 8 total, 8 up, 8 in 2024-09-13T14:49:29.563 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:29 smithi096 ceph-mon[102928]: pgmap v660: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-13T14:49:29.563 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:29 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:49:29.563 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:29 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:49:29.563 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:29 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2884580549' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:49:29.563 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:29 smithi096 ceph-mon[102928]: osdmap e234: 8 total, 8 up, 8 in 2024-09-13T14:49:29.644 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:29.673 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:29.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:29.715 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 50 16 83990 83974 104.941 72.3125 0.00619323 0.00952367 2024-09-13T14:49:29.776 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:29.813 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:29.915 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:29.950 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:29.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:30.052 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:30.091 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:30.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:30.193 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:30.233 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:30.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:30.335 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:30.357 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:30.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:30.460 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:49:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:49:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:30.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.500 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:30.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:30.601 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:30.645 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:30.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:30.715 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 51 16 85138 85122 104.29 71.75 0.0104987 0.00958313 2024-09-13T14:49:30.747 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:30.785 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:30.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:30.887 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:30.913 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:30.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:31.015 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:31.044 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:31.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:31.146 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:31.177 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:31.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:31.279 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:31.307 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:31.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:31.409 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:31.435 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:31.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:31.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:31 smithi204 ceph-mon[87071]: pgmap v662: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:49:31.537 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:31.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:31 smithi096 ceph-mon[98901]: pgmap v662: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:49:31.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:31 smithi096 ceph-mon[102928]: pgmap v662: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:49:31.576 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:31.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:31.607 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:49:31.678 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:31.705 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:31.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:31.715 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 52 16 86109 86093 103.452 60.6875 0.0225402 0.00966163 2024-09-13T14:49:31.807 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:31.834 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:31.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:31.936 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:31.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.984 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:31.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:32.085 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:32.111 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:32.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:32.213 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:32.242 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:32.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:32.313 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.4 reset_purged_snaps_last 2024-09-13T14:49:32.344 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:32.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.423 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:32.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:32.524 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:32.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.553 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:32.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:32.654 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:32.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.699 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:32.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:32.717 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 53 16 87163 87147 102.741 65.875 0.00763282 0.009728 2024-09-13T14:49:32.800 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:32.841 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:32.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:32.943 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:32.992 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:32.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:33.094 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:33.126 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:33.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:33.229 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:33.262 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:33.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:33.363 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:33.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.399 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:33.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:33.419 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:33 smithi096 ceph-mon[98901]: pgmap v663: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:49:33.419 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:33 smithi096 ceph-mon[102928]: pgmap v663: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:49:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:33 smithi204 ceph-mon[87071]: pgmap v663: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:49:33.500 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:33.525 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:33.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:33.627 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:33.630 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:49:33.630 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:49:33.630 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:49:33.631 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:49:33.631 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:49:33.631 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:49:33.631 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:49:33.631 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:49:33.636 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:49:33.636 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:49:33.669 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:49:33.669 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:49:33.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.683 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:33.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:33.700 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:49:33.700 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:49:33.716 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 54 16 88173 88157 102.007 63.125 0.0109488 0.0097965 2024-09-13T14:49:33.734 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:49:33.734 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:49:33.784 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:33.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.834 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:33.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:33.916 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:49:33.916 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:49:33.916 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.916 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35607.0:88328 7.7 e234/225 7:e659c874:::benchmark_data_smithi096_196313_object88327:head v 234'5559, mlcod=234'5556)", 2024-09-13T14:49:33.916 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:49:33.909653+0000", 2024-09-13T14:49:33.916 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.004734345, 2024-09-13T14:49:33.917 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0047627219999999996, 2024-09-13T14:49:33.917 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:49:33.917 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:49:33.917 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:49:33.917 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:49:33.917 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.917 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:49:33.917 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909653+0000", 2024-09-13T14:49:33.917 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:49:33.917 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.917 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.917 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:49:33.917 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909653+0000", 2024-09-13T14:49:33.917 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:49:33.918 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.918 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.918 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:49:33.918 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909655+0000", 2024-09-13T14:49:33.918 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.085e-06 2024-09-13T14:49:33.918 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.918 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.918 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:49:33.918 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909695+0000", 2024-09-13T14:49:33.918 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0519999999999998e-05 2024-09-13T14:49:33.918 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.918 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.919 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:49:33.919 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909714+0000", 2024-09-13T14:49:33.919 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8487000000000001e-05 2024-09-13T14:49:33.919 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.919 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.919 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:49:33.919 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909721+0000", 2024-09-13T14:49:33.919 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.8399999999999995e-06 2024-09-13T14:49:33.919 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.919 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.919 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:49:33.919 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.910079+0000", 2024-09-13T14:49:33.919 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00035727900000000001 2024-09-13T14:49:33.920 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.920 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.920 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:49:33.920 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.910125+0000", 2024-09-13T14:49:33.920 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6665999999999997e-05 2024-09-13T14:49:33.920 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:49:33.920 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:49:33.920 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:49:33.920 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.920 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.920 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35607.0:88325 7.4 e234/223 7:2c8f97de:::benchmark_data_smithi096_196313_object88324:head v 234'5563, mlcod=234'5561)", 2024-09-13T14:49:33.920 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:49:33.909757+0000", 2024-09-13T14:49:33.921 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0046307989999999997, 2024-09-13T14:49:33.921 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0047174749999999996, 2024-09-13T14:49:33.921 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:49:33.921 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:49:33.921 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:49:33.921 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:49:33.921 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.921 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:49:33.921 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909757+0000", 2024-09-13T14:49:33.921 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:49:33.921 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.921 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.921 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:49:33.922 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909757+0000", 2024-09-13T14:49:33.922 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:49:33.922 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.922 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.922 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:49:33.922 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909758+0000", 2024-09-13T14:49:33.922 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7999999999999996e-07 2024-09-13T14:49:33.922 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.922 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.922 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:49:33.922 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909776+0000", 2024-09-13T14:49:33.922 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8128999999999999e-05 2024-09-13T14:49:33.922 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.922 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.923 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:49:33.923 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909783+0000", 2024-09-13T14:49:33.923 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3300000000000001e-06 2024-09-13T14:49:33.923 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.923 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.923 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:49:33.923 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909788+0000", 2024-09-13T14:49:33.923 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8679999999999998e-06 2024-09-13T14:49:33.923 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.923 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.923 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:49:33.923 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909855+0000", 2024-09-13T14:49:33.923 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7212000000000006e-05 2024-09-13T14:49:33.924 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.924 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.924 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:49:33.924 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909875+0000", 2024-09-13T14:49:33.924 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0367999999999999e-05 2024-09-13T14:49:33.924 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:49:33.924 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:49:33.924 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:49:33.924 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.924 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.924 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35607.0:88322 7.9 e234/223 7:94c520a7:::benchmark_data_smithi096_196313_object88321:head v 234'5431, mlcod=234'5429)", 2024-09-13T14:49:33.924 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:49:33.909804+0000", 2024-09-13T14:49:33.924 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0045838720000000001, 2024-09-13T14:49:33.924 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.004720475, 2024-09-13T14:49:33.924 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:49:33.925 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:49:33.925 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:49:33.925 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:49:33.925 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.925 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:49:33.925 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909804+0000", 2024-09-13T14:49:33.925 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:49:33.925 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.925 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.925 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:49:33.925 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909804+0000", 2024-09-13T14:49:33.925 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:49:33.925 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.925 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.926 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:49:33.926 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909804+0000", 2024-09-13T14:49:33.926 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6800000000000002e-07 2024-09-13T14:49:33.926 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.926 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.926 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:49:33.926 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909824+0000", 2024-09-13T14:49:33.926 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9819000000000001e-05 2024-09-13T14:49:33.926 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.926 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.926 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:49:33.926 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909830+0000", 2024-09-13T14:49:33.926 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7849999999999997e-06 2024-09-13T14:49:33.926 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.927 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.927 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:49:33.927 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909834+0000", 2024-09-13T14:49:33.927 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.121e-06 2024-09-13T14:49:33.927 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.927 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.927 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:49:33.927 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909876+0000", 2024-09-13T14:49:33.927 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2508e-05 2024-09-13T14:49:33.927 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.927 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.927 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:49:33.927 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.909892+0000", 2024-09-13T14:49:33.927 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6005e-05 2024-09-13T14:49:33.927 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:49:33.928 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:49:33.928 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:49:33.928 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.928 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.928 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35607.0:88331 7.d e234/225 7:bc0ca496:::benchmark_data_smithi096_196313_object88330:head v 234'5480, mlcod=234'5478)", 2024-09-13T14:49:33.928 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:49:33.911393+0000", 2024-09-13T14:49:33.928 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0029949590000000002, 2024-09-13T14:49:33.928 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0031835639999999998, 2024-09-13T14:49:33.928 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:49:33.928 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:49:33.928 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:49:33.928 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:49:33.929 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.929 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:49:33.929 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.911393+0000", 2024-09-13T14:49:33.929 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:49:33.929 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.929 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.929 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:49:33.929 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.911393+0000", 2024-09-13T14:49:33.929 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:49:33.929 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.929 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.929 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:49:33.929 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.911394+0000", 2024-09-13T14:49:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6400000000000001e-07 2024-09-13T14:49:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:49:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.911434+0000", 2024-09-13T14:49:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0318999999999998e-05 2024-09-13T14:49:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:49:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.911448+0000", 2024-09-13T14:49:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3920000000000001e-05 2024-09-13T14:49:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:49:33.930 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.911455+0000", 2024-09-13T14:49:33.931 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7290000000000003e-06 2024-09-13T14:49:33.931 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.931 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.931 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:49:33.931 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.913083+0000", 2024-09-13T14:49:33.931 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001628446 2024-09-13T14:49:33.931 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:49:33.931 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:49:33.931 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:49:33.931 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:49:33.914201+0000", 2024-09-13T14:49:33.931 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001117561 2024-09-13T14:49:33.931 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:49:33.931 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:49:33.931 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:49:33.932 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:49:33.932 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2024-09-13T14:49:33.932 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 4 2024-09-13T14:49:33.932 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:49:33.935 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:33.995 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:33.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:34.097 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:34.150 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:34.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:34.252 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:34.291 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:34.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:34.393 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:34.440 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:34.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:34.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:34 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1704224655' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:34.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:34 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:49:34.542 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:34.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.571 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:34.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:34.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:34 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1704224655' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:34.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:34 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:49:34.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:34 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1704224655' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:34.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:34 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:49:34.672 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:34.705 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:34.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:34.717 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 55 16 89183 89167 101.3 63.125 0.0153648 0.00986642 2024-09-13T14:49:34.808 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:34.847 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:34.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:34.948 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:34.985 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:34.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:35.087 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:35.126 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:35.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:35.228 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:35.262 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:35.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:35.364 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:35.391 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:35.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:35.454 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:35 smithi096 ceph-mon[98901]: pgmap v664: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:49:35.457 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:35 smithi096 ceph-mon[102928]: pgmap v664: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:49:35.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:35 smithi204 ceph-mon[87071]: pgmap v664: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:49:35.475 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:35.493 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:35.504 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:49:35.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.571 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:35.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:35.672 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:35.697 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:35.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:35.718 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 56 16 90136 90120 100.552 59.5625 0.0182347 0.00993859 2024-09-13T14:49:35.800 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:35.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.876 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:35.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:35.977 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:36.007 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:36.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:36.109 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:36.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.146 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:36.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:36.247 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:36.290 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:36.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:36.391 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:36.441 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:36.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:36.543 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:36.549 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:36.581 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:36.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:36.683 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:36.710 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:36.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:36.719 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 57 16 90997 90981 99.7303 53.8125 0.0126117 0.0100217 2024-09-13T14:49:36.765 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:49:36.812 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:36.863 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:36.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:36.965 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:37.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.006 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:37.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:37.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:49:37.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:36 smithi096 ceph-mon[98901]: pgmap v665: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:49:37.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:37.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:49:37.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:36 smithi096 ceph-mon[102928]: pgmap v665: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:49:37.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:37.107 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:37.133 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:37.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:37.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:49:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:36 smithi204 ceph-mon[87071]: pgmap v665: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:49:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:37.235 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:37.263 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:37.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:37.365 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:37.412 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:37.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:37.514 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:37.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.535 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:37.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:37.636 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:37.666 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:37.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:37.719 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 58 16 91957 91941 99.0454 60 0.00973273 0.0100911 2024-09-13T14:49:37.768 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:37.795 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:37.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:37.897 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:37.944 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:37.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:38.048 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:38.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:49:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:49:37] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.51.0" 2024-09-13T14:49:38.075 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:38.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:38.177 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:38.224 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:38.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:38.327 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:38.342 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:38.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.386 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:38.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:38.487 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:38.517 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:38.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:38.620 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:38.647 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:38.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:38.720 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 59 16 92978 92962 98.4482 63.8125 0.0141233 0.0101527 2024-09-13T14:49:38.748 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:38.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.788 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:38.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:38.889 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:38.915 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:38.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:39.017 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:39.066 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:39.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:39.158 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:38 smithi096 ceph-mon[98901]: pgmap v666: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:49:39.158 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:38 smithi096 ceph-mon[102928]: pgmap v666: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:49:39.168 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:39.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.210 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:39.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:38 smithi204 ceph-mon[87071]: pgmap v666: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:49:39.310 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:39.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.347 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:39.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:39.449 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:39.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.485 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:39.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:39.586 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:39.626 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:39.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:39.720 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:49:39.719828+0000 min lat: 0.00108557 max lat: 0.194678 avg lat: 0.0102114 2024-09-13T14:49:39.720 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:49:39.720 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 60 16 94008 93992 97.8804 64.375 0.0125578 0.0102114 2024-09-13T14:49:39.729 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:39.769 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:39.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:39.871 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:39.903 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:39.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:40.005 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:40.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.035 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:40.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:40.137 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:40.165 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:40.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:40.267 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:40.310 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:40.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:40.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:40 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:49:40.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:40 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:49:40.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:40 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:49:40.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:40 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:49:40.411 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:40 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:49:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:40 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:49:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:49:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:49:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:40.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.477 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:40.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:40.578 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:40.631 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:40.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:40.733 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:40.738 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 61 16 95041 95025 97.3046 64.5625 0.00409263 0.0102676 2024-09-13T14:49:40.770 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:40.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:40.874 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:40.907 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:40.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:41.009 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:41.038 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:41.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:41.140 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:41.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.186 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:41.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:41.288 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:41.302 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:41 smithi096 ceph-mon[98901]: pgmap v667: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:49:41.303 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:41 smithi096 ceph-mon[102928]: pgmap v667: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:49:41.314 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:41.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:41.416 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:41.443 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:41.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:41.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:41 smithi204 ceph-mon[87071]: pgmap v667: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:49:41.545 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:41.588 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:41.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:41.690 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:41.713 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:41.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:41.738 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 62 16 95943 95927 96.6447 56.375 0.0519181 0.0103355 2024-09-13T14:49:41.815 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:41.855 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:41.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:41.957 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:41.986 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:41.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:42.088 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:42.117 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:42.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:42.219 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:42.249 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:42.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:42.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:42 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2041973517' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:42.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:42 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:42.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:42 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2041973517' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:42.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:42 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:42.351 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:42.393 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:42.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:42.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:42 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2041973517' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:42.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:42 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:42.495 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:42.526 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:42.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:42.628 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:42.639 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:49:42.656 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:42.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:42.738 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 63 16 96923 96907 96.0831 61.25 0.00959669 0.0104028 2024-09-13T14:49:42.758 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:42.791 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:42.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:42.876 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:42.893 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:42.922 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:42.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:43.024 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:43.050 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:43.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:43.152 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:43.182 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:43.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:43.286 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:43.315 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:43.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:43.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:43 smithi096 ceph-mon[98901]: pgmap v668: 99 pgs: 99 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:49:43.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:43 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:43.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:43 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:49:43.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:43 smithi096 ceph-mon[98901]: osdmap e235: 8 total, 8 up, 8 in 2024-09-13T14:49:43.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:43 smithi096 ceph-mon[102928]: pgmap v668: 99 pgs: 99 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:49:43.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:43 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:43.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:43 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:49:43.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:43 smithi096 ceph-mon[102928]: osdmap e235: 8 total, 8 up, 8 in 2024-09-13T14:49:43.418 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:43.459 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:43.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:43.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:43 smithi204 ceph-mon[87071]: pgmap v668: 99 pgs: 99 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:49:43.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:43 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:43.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:43 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:49:43.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:43 smithi204 ceph-mon[87071]: osdmap e235: 8 total, 8 up, 8 in 2024-09-13T14:49:43.561 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:43.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.601 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:43.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:43.702 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:43.730 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:43.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:43.738 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 64 16 97964 97948 95.5985 65.0625 0.0213992 0.0104551 2024-09-13T14:49:43.834 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:43.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.911 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:43.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:44.012 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:44.055 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:44.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:44.157 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:44.182 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:44.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:44.284 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:44.311 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:44.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:44.413 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:44.453 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:44.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:44.555 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:44.586 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:44.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:44.688 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:44.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.727 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:44.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:44.739 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 65 16 98955 98939 95.0808 61.9375 0.0113859 0.0105117 2024-09-13T14:49:44.829 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:44.859 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:44.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:44.961 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:44.992 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:44.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:45.093 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:45.123 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:45.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:45.225 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:45.263 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:45.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:45.315 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:45 smithi096 ceph-mon[98901]: pgmap v670: 99 pgs: 99 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 986 op/s 2024-09-13T14:49:45.315 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:45 smithi096 ceph-mon[102928]: pgmap v670: 99 pgs: 99 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 986 op/s 2024-09-13T14:49:45.365 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:45.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.400 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:45.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:45.433 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:49:45.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:45 smithi204 ceph-mon[87071]: pgmap v670: 99 pgs: 99 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 986 op/s 2024-09-13T14:49:45.501 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:45.559 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:45.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:45.660 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:45.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.700 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:45.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:45.739 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 66 16 99883 99867 94.5187 58 0.014295 0.0105741 2024-09-13T14:49:45.802 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:45.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.849 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:45.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:45.881 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:49:45.881 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:49:45.881 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:49:45.881 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:49:45.881 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:49:45.951 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:45.978 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:45.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:46.080 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:46.109 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:46.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:46.211 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:46.252 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:46.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:46.354 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:46.380 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:46.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:46.482 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:46.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.522 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:46.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:46.623 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:46.652 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:46.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:46.739 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 67 16 100882 100866 94.04 62.4375 0.00956762 0.0106285 2024-09-13T14:49:46.754 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:46.791 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:46.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:46.894 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:46.919 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:46.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:47.021 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:47.057 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:47.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:47.159 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:47.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.199 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:47.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:47.300 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:47.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:47 smithi096 ceph-mon[98901]: pgmap v671: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-13T14:49:47.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:47 smithi096 ceph-mon[102928]: pgmap v671: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-13T14:49:47.339 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:47.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:47.440 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:47.467 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:47.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:47.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:47 smithi204 ceph-mon[87071]: pgmap v671: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-13T14:49:47.569 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:47.595 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:47.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:47.697 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:47.731 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:47.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:47.739 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 68 16 101795 101779 93.4964 57.0625 0.0129875 0.0106902 2024-09-13T14:49:47.833 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:47.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.862 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:47.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:47.963 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:47.993 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:47.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:48.095 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:48.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.125 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:48.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:48.226 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:48.261 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:48.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:48.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:49:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:49:47] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.51.0" 2024-09-13T14:49:48.363 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:48.405 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:48.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:48.507 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:48.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.548 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:48.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:48.649 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:48.676 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:48.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:48.739 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 69 16 102736 102720 92.9939 58.8125 0.00464242 0.0107476 2024-09-13T14:49:48.778 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:48.799 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:48.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:48.902 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:48.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.933 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:48.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:49.034 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:49.062 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:49.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:49.163 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:49.199 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:49.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:49.301 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:49 smithi096 ceph-mon[98901]: pgmap v672: 99 pgs: 99 active+clean; 6.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2024-09-13T14:49:49.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:49 smithi096 ceph-mon[102928]: pgmap v672: 99 pgs: 99 active+clean; 6.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2024-09-13T14:49:49.325 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:49.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:49.427 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:49.455 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:49.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:49 smithi204 ceph-mon[87071]: pgmap v672: 99 pgs: 99 active+clean; 6.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2024-09-13T14:49:49.557 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:49.597 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:49.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:49.699 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:49.739 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 70 16 103608 103592 92.4442 54.5 0.0155824 0.0108122 2024-09-13T14:49:49.744 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:49.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:49.846 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:49.882 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:49.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:49.986 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:50.014 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:50.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:50.117 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:50.142 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:50.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:50.152 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:49:50.181 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.5 reset_purged_snaps_last 2024-09-13T14:49:50.244 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:50.269 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:50.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:50.370 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:50.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.401 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:50.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:50.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:49:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:49:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:50.502 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:50.547 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:50.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:50.649 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:50.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.680 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:50.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:50.740 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 71 16 104466 104450 91.8976 53.625 0.0303549 0.0108766 2024-09-13T14:49:50.781 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:50.811 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:50.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:50.913 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:50.940 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:50.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:51.045 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:51.096 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:51.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:51.202 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:51.240 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:51.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:51.252 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:49:51.252 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:49:51.307 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:49:51.307 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:49:51.342 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:51.352 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:51 smithi096 ceph-mon[98901]: pgmap v673: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2024-09-13T14:49:51.352 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:49:51.352 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:51.352 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:51 smithi096 ceph-mon[102928]: pgmap v673: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2024-09-13T14:49:51.352 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:49:51.353 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:51.353 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:49:51.353 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:49:51.393 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:49:51.394 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:49:51.396 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:51.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:51.424 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:49:51.425 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:49:51.454 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:49:51.454 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:49:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:51 smithi204 ceph-mon[87071]: pgmap v673: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2024-09-13T14:49:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:49:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:51.488 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:49:51.489 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:49:51.498 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:51.518 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:49:51.519 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:49:51.531 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:51.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:51.633 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:51.661 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:51.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:51.740 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 72 16 105155 105139 91.2196 43.0625 0.00366234 0.0109442 2024-09-13T14:49:51.763 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:51.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.785 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:51.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:51.886 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:51.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.920 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:51.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:52.021 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:52.047 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:52.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:52.149 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:52.170 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:52.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:52.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:52 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3478183697' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:52.272 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:52.308 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:52.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:52.410 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:52.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.465 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:52.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:52.566 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:52.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:52 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3478183697' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:52.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:52 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3478183697' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:52.610 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:52.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:52.712 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:52.740 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 73 16 105871 105855 90.5832 44.75 0.0159502 0.0110345 2024-09-13T14:49:52.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.741 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:52.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:52.789 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:52.842 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:52.858 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:52.884 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:52.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:52.890 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:52.986 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:53.018 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:53.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:53.121 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:53.152 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:53.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:53.256 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:53.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.294 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:53.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:53.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:53 smithi096 ceph-mon[98901]: pgmap v674: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2024-09-13T14:49:53.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:53 smithi096 ceph-mon[102928]: pgmap v674: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2024-09-13T14:49:53.395 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:53.428 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:53.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:53.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:53 smithi204 ceph-mon[87071]: pgmap v674: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2024-09-13T14:49:53.530 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:53.561 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:53.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:53.662 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:53.692 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:53.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:53.740 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 74 16 106806 106790 90.1489 58.4375 0.0357778 0.0110875 2024-09-13T14:49:53.793 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:53.825 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:53.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:53.927 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:53.963 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:53.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:54.065 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:54.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.102 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:54.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:54.203 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:54.231 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:54.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:54.333 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:54.364 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:54.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:54.466 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:54.494 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:54.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:54.596 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:54.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.641 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:54.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:54.740 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 75 16 107780 107764 89.7587 60.875 0.00718697 0.0111354 2024-09-13T14:49:54.742 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:54.782 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:54.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:54.884 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:54.917 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:54.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:55.018 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:55.040 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:55.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:55.142 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:55.162 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:55.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:55.264 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:55.285 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:55.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:55.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:55 smithi096 ceph-mon[98901]: pgmap v675: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2024-09-13T14:49:55.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:55 smithi096 ceph-mon[102928]: pgmap v675: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2024-09-13T14:49:55.387 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:55.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.409 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:55.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:55.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:55 smithi204 ceph-mon[87071]: pgmap v675: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2024-09-13T14:49:55.489 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:49:55.510 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:55.575 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:55.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:55.676 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:55.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.712 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:55.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:55.740 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 76 16 108320 108304 89.022 33.75 0.0131646 0.0112282 2024-09-13T14:49:55.813 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:55.841 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:55.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:55.943 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:55.970 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:55.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:56.072 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:56.102 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:56.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:56.204 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:56.245 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:56.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:56.347 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:56.374 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:56.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:56.476 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:56.503 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:56.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:56.605 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:56.630 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:56.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:56.732 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:56.740 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 77 16 109275 109259 88.6411 59.6875 0.0108893 0.0112761 2024-09-13T14:49:56.763 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:56.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:56.865 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:56.889 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:56.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:56.991 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:57.018 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:57.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:57.120 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:57.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.158 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:57.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:57.259 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:57.284 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:57.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:57.386 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:57.427 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:57.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:57 smithi204 ceph-mon[87071]: pgmap v676: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s 2024-09-13T14:49:57.528 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:57.569 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:57.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:57.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:57 smithi096 ceph-mon[98901]: pgmap v676: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s 2024-09-13T14:49:57.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:57 smithi096 ceph-mon[102928]: pgmap v676: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s 2024-09-13T14:49:57.671 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:57.711 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:57.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:57.740 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 78 16 110192 110176 88.2395 57.3125 0.0145393 0.0113273 2024-09-13T14:49:57.813 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:57.838 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:57.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:57.939 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:57.968 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:57.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:58.071 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:58.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:49:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:49:57] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.51.0" 2024-09-13T14:49:58.101 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:58.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:58.203 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:58.229 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:58.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:58.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:58.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.368 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:58.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:58.469 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:58.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.524 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:58.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:58.625 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:58.652 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:58.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:58.740 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 79 16 111081 111065 87.826 55.5625 0.0210671 0.0113812 2024-09-13T14:49:58.754 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:58.781 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:58.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:58.883 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:58.907 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:58.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:59.009 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:59.039 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:59.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:59.142 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:59.175 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:59.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:59.277 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:59.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.318 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:49:59.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:59.390 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:49:59 smithi096 ceph-mon[98901]: pgmap v677: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2024-09-13T14:49:59.390 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:49:59 smithi096 ceph-mon[102928]: pgmap v677: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2024-09-13T14:49:59.419 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:59.447 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:59.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:49:59 smithi204 ceph-mon[87071]: pgmap v677: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2024-09-13T14:49:59.549 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:59.575 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:59.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:59.677 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:59.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.706 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:59.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:59.740 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:49:59.740437+0000 min lat: 0.00108557 max lat: 0.451783 avg lat: 0.0114363 2024-09-13T14:49:59.741 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:49:59.741 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 80 16 111932 111916 87.3931 53.1875 0.00935887 0.0114363 2024-09-13T14:49:59.807 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:59.834 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:59.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:59.935 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:59.961 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:49:59.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:00.063 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:00.086 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:00.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:00.147 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:50:00.187 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:00.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.231 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:00.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:00.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:00 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2917960945' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:00.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:00 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:00.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:00 smithi096 ceph-mon[98901]: Health detail: HEALTH_WARN noscrub flag(s) set 2024-09-13T14:50:00.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:00 smithi096 ceph-mon[98901]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-13T14:50:00.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:00 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2917960945' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:00.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:00 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:00.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:00 smithi096 ceph-mon[102928]: Health detail: HEALTH_WARN noscrub flag(s) set 2024-09-13T14:50:00.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:00 smithi096 ceph-mon[102928]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-13T14:50:00.333 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:00.369 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:00.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:50:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:50:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:00 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2917960945' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:00 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:00 smithi204 ceph-mon[87071]: Health detail: HEALTH_WARN noscrub flag(s) set 2024-09-13T14:50:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:00 smithi204 ceph-mon[87071]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-13T14:50:00.471 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:00.497 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:00.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:00.599 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:00.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.654 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:00.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:00.740 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 81 16 112824 112808 87.0025 55.75 0.0287238 0.011489 2024-09-13T14:50:00.755 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:00.793 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:00.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:00.894 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:00.930 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:00.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:01.032 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:01.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:01.163 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:01.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.192 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:01.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:01.293 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:01.323 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:01 smithi204 ceph-mon[87071]: pgmap v678: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s 2024-09-13T14:50:01.323 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:01 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:01.323 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:01 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:01.323 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:01 smithi204 ceph-mon[87071]: osdmap e236: 8 total, 8 up, 8 in 2024-09-13T14:50:01.324 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:01.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:01.426 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:01.468 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:01.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:01.569 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:01.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:01 smithi096 ceph-mon[98901]: pgmap v678: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s 2024-09-13T14:50:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:01 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:01 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:01 smithi096 ceph-mon[98901]: osdmap e236: 8 total, 8 up, 8 in 2024-09-13T14:50:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:01 smithi096 ceph-mon[102928]: pgmap v678: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s 2024-09-13T14:50:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:01 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:01 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:01 smithi096 ceph-mon[102928]: osdmap e236: 8 total, 8 up, 8 in 2024-09-13T14:50:01.603 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:01.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:01.705 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:01.738 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:01.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:01.740 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 82 16 113679 113663 86.5933 53.4375 0.00975892 0.0115425 2024-09-13T14:50:01.840 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:01.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.865 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:01.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:01.966 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:02.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.011 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:02.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:02.112 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:02.151 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:02.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:02.253 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:02.279 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:02.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:02.381 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:02.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.413 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:02.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:02.514 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:02.539 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:02.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:02.641 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:02.668 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:02.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:02.741 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 83 16 114459 114443 86.1375 48.75 0.0143122 0.0116039 2024-09-13T14:50:02.770 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:02.808 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:02.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:02.909 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:02.932 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:02.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:02.996 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:50:02.997 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:50:03.034 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:03.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.078 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:03.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:03.179 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:03.209 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:03.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:03.311 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:03.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:03 smithi096 ceph-mon[98901]: pgmap v680: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2024-09-13T14:50:03.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:03 smithi096 ceph-mon[102928]: pgmap v680: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2024-09-13T14:50:03.339 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:03.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:03.398 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.7 reset_purged_snaps_last 2024-09-13T14:50:03.441 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:03.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:03 smithi204 ceph-mon[87071]: pgmap v680: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2024-09-13T14:50:03.478 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:03.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:03.580 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:03.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.630 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:03.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:03.731 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:03.741 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 84 16 115303 115287 85.74 52.75 0.0270968 0.0116574 2024-09-13T14:50:03.784 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:03.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:03.885 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:03.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.922 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:03.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:04.023 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:04.049 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:04.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:04.151 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:04.198 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:04.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:04.300 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:04.338 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:04.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:04.440 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:04.462 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:50:04.489 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:04.508 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:04.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:04.544 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:04.610 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:04.635 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:04.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:04.644 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:50:04.737 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:04.741 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 85 16 116097 116081 85.3152 49.625 0.0407583 0.0117158 2024-09-13T14:50:04.851 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:04.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:04.932 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:04 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:50:04.932 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:04 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:50:04.932 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:04 smithi096 ceph-mon[98901]: pgmap v681: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2024-09-13T14:50:04.932 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:04 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:50:04.932 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:04 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:50:04.932 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:04 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:50:04.932 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:04 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:50:04.932 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:04 smithi096 ceph-mon[102928]: pgmap v681: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2024-09-13T14:50:04.933 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:04 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:50:04.933 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:04 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:50:04.958 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:05.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.022 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:05.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:05.123 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:05.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.166 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:05.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:05.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:04 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:50:05.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:04 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:50:05.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:04 smithi204 ceph-mon[87071]: pgmap v681: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2024-09-13T14:50:05.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:04 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:50:05.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:04 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:50:05.267 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:05.293 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:05.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:05.395 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:05.421 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:05.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:05.523 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:05.565 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:05.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:05.667 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:05.693 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:05.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:05.741 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 86 16 116851 116835 84.8712 47.125 0.0236585 0.0117773 2024-09-13T14:50:05.795 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:05.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.831 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:05.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:05.932 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:05.969 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:05.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:06.071 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:06.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:05 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:50:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:05 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:50:06.093 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:06.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:06.195 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:06.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:05 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:50:06.221 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:06.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:06.322 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:06.349 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:06.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:06.450 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:06.474 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:06.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:06.576 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:06.602 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:06.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:06.704 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:06.730 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:06.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:06.741 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 87 16 117643 117627 84.4647 49.5 0.0188191 0.0118339 2024-09-13T14:50:06.833 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:06.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.855 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:06.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:06.956 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:06.995 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:06.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:06 smithi096 ceph-mon[98901]: pgmap v682: 99 pgs: 99 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2024-09-13T14:50:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:06 smithi096 ceph-mon[102928]: pgmap v682: 99 pgs: 99 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2024-09-13T14:50:07.097 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:07.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.141 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:07.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:06 smithi204 ceph-mon[87071]: pgmap v682: 99 pgs: 99 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2024-09-13T14:50:07.243 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:07.267 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:07.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:07.369 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:07.395 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:07.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:07.497 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:07.520 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:07.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:07.622 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:07.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.658 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:07.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:07.741 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 88 16 118427 118411 84.0618 49 0.0125822 0.0118907 2024-09-13T14:50:07.760 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:07.799 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:07.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:07.902 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:07.927 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:07.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:08.029 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:08.054 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:08.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:08.156 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:08.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.198 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:08.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:08.225 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:50:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:50:07] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.51.0" 2024-09-13T14:50:08.299 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:08.325 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:08.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:08.428 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:08.458 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:08.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:08.560 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:08.587 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:08.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:08.689 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:08.717 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:08.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:08.741 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 89 16 119329 119313 83.7508 56.375 0.00913716 0.011935 2024-09-13T14:50:08.819 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:08.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.862 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:08.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:08.963 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:09.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.010 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:09.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:09.037 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:08 smithi096 ceph-mon[102928]: pgmap v683: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2024-09-13T14:50:09.040 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:08 smithi096 ceph-mon[98901]: pgmap v683: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2024-09-13T14:50:09.110 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:09.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.140 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:09.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:09.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:08 smithi204 ceph-mon[87071]: pgmap v683: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2024-09-13T14:50:09.241 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:09.265 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:09.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:09.367 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:09.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.403 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:09.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:09.504 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:09.538 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:09.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:09.640 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:09.680 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:09.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:09.741 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Total time run: 90.0097 2024-09-13T14:50:09.742 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Total writes made: 120128 2024-09-13T14:50:09.742 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Write size: 65536 2024-09-13T14:50:09.742 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Object size: 65536 2024-09-13T14:50:09.742 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Bandwidth (MB/sec): 83.4132 2024-09-13T14:50:09.742 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev Bandwidth: 37.3554 2024-09-13T14:50:09.742 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max bandwidth (MB/sec): 226 2024-09-13T14:50:09.742 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min bandwidth (MB/sec): 33.75 2024-09-13T14:50:09.742 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Average IOPS: 1334 2024-09-13T14:50:09.742 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev IOPS: 597.686 2024-09-13T14:50:09.742 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max IOPS: 3616 2024-09-13T14:50:09.742 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min IOPS: 540 2024-09-13T14:50:09.742 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Average Latency(s): 0.0119843 2024-09-13T14:50:09.743 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev Latency(s): 0.0108753 2024-09-13T14:50:09.743 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max latency(s): 0.451783 2024-09-13T14:50:09.743 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min latency(s): 0.00108557 2024-09-13T14:50:09.748 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:50:09.782 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:09.808 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:09.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:09.884 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:50:09.884 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:50:09.884 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:50:09.884 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:50:09.885 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:50:09.885 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:50:09.885 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:50:09.885 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:50:09.885 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:50:09.885 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:50:09.903 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:50:09.903 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:50:09.910 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:09.941 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:50:09.942 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:50:09.954 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:09.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:09.977 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:50:09.977 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:50:10.056 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:10.081 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:10.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:10.183 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:10.208 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:10.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:10.213 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:09 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1915899562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:10.310 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:10.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:09 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1915899562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:10.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:09 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1915899562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:10.331 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:10.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:10.433 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:10.455 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:10.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:10.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:50:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:50:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:10.556 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:10.583 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:10.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:10.685 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:10.710 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:10.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:10.812 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:10.844 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:10.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:10.946 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:10.980 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:10.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:11.084 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:11.111 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:11.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:11.213 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:11.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:10 smithi204 ceph-mon[87071]: pgmap v684: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2024-09-13T14:50:11.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.257 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:11.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:11.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:10 smithi096 ceph-mon[98901]: pgmap v684: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2024-09-13T14:50:11.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:10 smithi096 ceph-mon[102928]: pgmap v684: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2024-09-13T14:50:11.358 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:11.388 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:11.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:11.480 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.6 reset_purged_snaps_last 2024-09-13T14:50:11.492 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:11.562 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:11.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:11.664 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:11.704 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:11.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:11.807 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:11.835 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:11.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:11.937 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:11.977 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:11.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:12.079 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:12.104 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:12.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:12.204 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:12.206 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:12.208 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:12.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.230 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:12.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:12.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:12.374 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:12.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:12.476 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:12.511 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:12.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:12.613 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:12.640 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:12.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:12.742 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:12.769 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:12.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:12.871 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:12.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.899 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:12.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:13.000 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:13.047 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:13.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:13.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:12 smithi096 ceph-mon[98901]: pgmap v685: 99 pgs: 99 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 87 B/s rd, 49 MiB/s wr, 821 op/s 2024-09-13T14:50:13.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:12 smithi096 ceph-mon[102928]: pgmap v685: 99 pgs: 99 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 87 B/s rd, 49 MiB/s wr, 821 op/s 2024-09-13T14:50:13.149 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:13.178 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:13.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:12 smithi204 ceph-mon[87071]: pgmap v685: 99 pgs: 99 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 87 B/s rd, 49 MiB/s wr, 821 op/s 2024-09-13T14:50:13.279 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:13.317 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:13.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:13.419 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:13.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.454 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:13.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:13.554 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:13.583 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:13.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:13.685 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:13.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.714 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:13.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:13.785 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:13.815 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:13.878 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:13.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:13.980 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:14.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.010 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:14.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:14.112 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:14.149 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:14.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:14.236 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:50:14.251 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:14.279 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:14.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:14.381 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:14.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.456 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:14.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:14.557 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:14.583 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:14.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:14.685 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:14.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.724 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:14.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:14.825 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:14.850 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:14.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:14.952 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:15.006 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:15.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:15.034 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:14 smithi096 ceph-mon[102928]: pgmap v686: 99 pgs: 99 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 792 op/s 2024-09-13T14:50:15.034 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:14 smithi096 ceph-mon[98901]: pgmap v686: 99 pgs: 99 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 792 op/s 2024-09-13T14:50:15.108 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:15.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.141 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:15.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:15.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:14 smithi204 ceph-mon[87071]: pgmap v686: 99 pgs: 99 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 792 op/s 2024-09-13T14:50:15.242 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:15.269 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:15.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:15.370 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:15.394 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:15.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:15.496 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:15.520 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:15.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:15.622 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:15.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.670 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:15.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:15.771 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:15.796 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:15.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:15.903 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:15.938 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:15.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:16.040 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:16.075 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:16.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:16.177 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:16.214 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:16.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:16.317 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:16.338 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:16.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:16.440 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:16.461 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:16.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:16.562 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:16.583 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:16.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:16.685 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:16.711 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:16.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:16.813 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:16.854 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:16.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:16.956 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:16.989 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:16.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:17.091 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:17.116 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:17.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:17.218 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:17.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:16 smithi204 ceph-mon[87071]: pgmap v687: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 812 op/s 2024-09-13T14:50:17.246 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:17.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:17.289 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:16 smithi096 ceph-mon[98901]: pgmap v687: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 812 op/s 2024-09-13T14:50:17.289 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:16 smithi096 ceph-mon[102928]: pgmap v687: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 812 op/s 2024-09-13T14:50:17.347 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:17.374 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:17.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:17.477 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:17.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.518 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:17.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:17.619 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:17.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.655 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:17.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:17.756 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:17.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.796 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:17.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:17.897 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:17.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.950 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:17.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:18.051 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:18.079 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:18.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:18.127 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:50:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:50:17] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.51.0" 2024-09-13T14:50:18.130 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:50:18.130 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:50:18.130 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.130 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35607.0:127159 7.e 7:75c50ae6:::benchmark_data_smithi096_196313_object7028:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2024-09-13T14:50:18.130 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:50:18.100881+0000", 2024-09-13T14:50:18.131 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.024760832, 2024-09-13T14:50:18.131 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.024801383999999999, 2024-09-13T14:50:18.131 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:18.131 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:18.131 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:50:18.131 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:18.131 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35607", 2024-09-13T14:50:18.131 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:50:18.131 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 127159 2024-09-13T14:50:18.131 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.131 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:18.131 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.131 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:18.131 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.100881+0000", 2024-09-13T14:50:18.132 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:18.132 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.132 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.132 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:18.132 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.100881+0000", 2024-09-13T14:50:18.132 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:18.132 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.132 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.132 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:18.132 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.100882+0000", 2024-09-13T14:50:18.132 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4000000000000002e-07 2024-09-13T14:50:18.132 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.132 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.133 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:18.133 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.100885+0000", 2024-09-13T14:50:18.133 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.8399999999999999e-06 2024-09-13T14:50:18.133 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.133 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.133 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:18.133 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.100891+0000", 2024-09-13T14:50:18.133 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7059999999999998e-06 2024-09-13T14:50:18.133 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.133 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.133 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:18.133 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.100896+0000", 2024-09-13T14:50:18.133 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.831e-06 2024-09-13T14:50:18.133 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.134 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.134 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:18.134 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.101856+0000", 2024-09-13T14:50:18.134 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00095969200000000005 2024-09-13T14:50:18.134 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.134 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.134 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:18.134 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.101951+0000", 2024-09-13T14:50:18.134 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.5291999999999995e-05 2024-09-13T14:50:18.134 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.134 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.134 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:50:18.134 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.102043+0000", 2024-09-13T14:50:18.135 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.1803000000000002e-05 2024-09-13T14:50:18.135 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.135 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.135 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:18.135 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.110849+0000", 2024-09-13T14:50:18.135 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0088056709999999993 2024-09-13T14:50:18.135 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.135 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.135 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:18.135 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.110896+0000", 2024-09-13T14:50:18.135 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7695999999999999e-05 2024-09-13T14:50:18.135 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:18.135 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:18.136 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:18.136 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.136 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.136 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35607.0:127171 7.2 e236/223 7:4f88b6d4:::benchmark_data_smithi096_196313_object7040:head v 236'7951, mlcod=236'7949)", 2024-09-13T14:50:18.136 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:50:18.121484+0000", 2024-09-13T14:50:18.136 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.004158269, 2024-09-13T14:50:18.136 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0042764029999999998, 2024-09-13T14:50:18.136 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:18.136 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:18.136 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:50:18.136 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:18.136 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.136 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:18.136 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121484+0000", 2024-09-13T14:50:18.137 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:18.137 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.137 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.137 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:18.137 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121484+0000", 2024-09-13T14:50:18.137 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:18.137 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.137 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.137 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:18.137 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121485+0000", 2024-09-13T14:50:18.137 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.079e-06 2024-09-13T14:50:18.137 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.137 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.137 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:18.138 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121493+0000", 2024-09-13T14:50:18.138 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.191e-06 2024-09-13T14:50:18.138 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.138 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.138 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:18.138 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121511+0000", 2024-09-13T14:50:18.138 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7691999999999999e-05 2024-09-13T14:50:18.138 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.138 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.138 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:18.138 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121519+0000", 2024-09-13T14:50:18.138 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.2179999999999999e-06 2024-09-13T14:50:18.138 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.138 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.139 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:18.139 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121653+0000", 2024-09-13T14:50:18.139 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013379 2024-09-13T14:50:18.139 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.139 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.139 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:18.139 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121674+0000", 2024-09-13T14:50:18.139 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1566e-05 2024-09-13T14:50:18.139 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:18.139 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:18.139 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:18.139 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.139 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.139 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35607.0:127172 7.a e236/223 7:5a4cf385:::benchmark_data_smithi096_196313_object7041:head v 236'7870, mlcod=236'7868)", 2024-09-13T14:50:18.140 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:50:18.121541+0000", 2024-09-13T14:50:18.140 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0041010960000000003, 2024-09-13T14:50:18.140 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.004265675, 2024-09-13T14:50:18.140 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:18.140 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:18.140 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:50:18.140 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:18.140 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.140 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:18.140 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121541+0000", 2024-09-13T14:50:18.140 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:18.140 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.140 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.140 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:18.141 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121541+0000", 2024-09-13T14:50:18.141 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:18.141 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.141 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.141 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:18.141 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121541+0000", 2024-09-13T14:50:18.141 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.8000000000000002e-07 2024-09-13T14:50:18.141 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.141 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.141 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:18.142 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121545+0000", 2024-09-13T14:50:18.142 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4369999999999999e-06 2024-09-13T14:50:18.142 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.142 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.142 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:18.142 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121555+0000", 2024-09-13T14:50:18.142 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0051000000000001e-05 2024-09-13T14:50:18.142 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.142 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.142 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:18.143 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121560+0000", 2024-09-13T14:50:18.143 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.1510000000000003e-06 2024-09-13T14:50:18.143 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.143 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.143 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:18.143 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121838+0000", 2024-09-13T14:50:18.143 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000277836 2024-09-13T14:50:18.143 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.143 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.143 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:18.143 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121856+0000", 2024-09-13T14:50:18.143 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8074999999999999e-05 2024-09-13T14:50:18.143 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:18.144 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:18.144 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:18.144 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.144 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.144 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.35607.0:127174 7.5 e236/223 7:a8674a6e:::benchmark_data_smithi096_196313_object7043:head v 236'8021, mlcod=236'8019)", 2024-09-13T14:50:18.144 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:50:18.121578+0000", 2024-09-13T14:50:18.144 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0040644180000000002, 2024-09-13T14:50:18.144 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0042736659999999997, 2024-09-13T14:50:18.144 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:18.144 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:18.144 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:50:18.144 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:18.144 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.145 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:18.145 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121578+0000", 2024-09-13T14:50:18.145 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:18.145 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.145 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.145 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:18.145 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121578+0000", 2024-09-13T14:50:18.145 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:18.145 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.145 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.145 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:18.145 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121578+0000", 2024-09-13T14:50:18.145 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4900000000000001e-07 2024-09-13T14:50:18.146 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.146 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.146 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:18.146 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121582+0000", 2024-09-13T14:50:18.146 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.8809999999999998e-06 2024-09-13T14:50:18.146 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.146 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.146 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:18.146 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121588+0000", 2024-09-13T14:50:18.146 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3409999999999999e-06 2024-09-13T14:50:18.146 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.146 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.146 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:18.147 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121593+0000", 2024-09-13T14:50:18.147 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.386e-06 2024-09-13T14:50:18.147 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.147 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.147 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:18.147 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121655+0000", 2024-09-13T14:50:18.147 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1760000000000005e-05 2024-09-13T14:50:18.147 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.147 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.147 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:18.147 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121676+0000", 2024-09-13T14:50:18.147 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1922000000000001e-05 2024-09-13T14:50:18.147 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:18.147 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:18.147 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:18.148 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.148 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.148 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.35607.0:127149 7.e e236/223)", 2024-09-13T14:50:18.148 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:50:18.121614+0000", 2024-09-13T14:50:18.148 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.00402806, 2024-09-13T14:50:18.148 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0042801660000000002, 2024-09-13T14:50:18.148 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:18.148 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:18.148 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "queued for pg", 2024-09-13T14:50:18.148 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:18.148 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.148 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:18.148 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121614+0000", 2024-09-13T14:50:18.149 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:18.149 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.149 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.149 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:18.149 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121614+0000", 2024-09-13T14:50:18.149 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:18.149 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.149 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.149 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:18.149 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121615+0000", 2024-09-13T14:50:18.149 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4700000000000002e-07 2024-09-13T14:50:18.149 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.149 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.149 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.121617+0000", 2024-09-13T14:50:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4049999999999998e-06 2024-09-13T14:50:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.122693+0000", 2024-09-13T14:50:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0010759840000000001 2024-09-13T14:50:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.122702+0000", 2024-09-13T14:50:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3400000000000004e-06 2024-09-13T14:50:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:18.151 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:18.151 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.151 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.151 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35607.0:127149 7.e 7:71a5136b:::benchmark_data_smithi096_196313_object7018:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2024-09-13T14:50:18.151 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:50:18.086488+0000", 2024-09-13T14:50:18.151 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.039154605000000002, 2024-09-13T14:50:18.151 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.039451501, 2024-09-13T14:50:18.151 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:18.151 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:18.151 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:50:18.151 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:18.151 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35607", 2024-09-13T14:50:18.151 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:50:18.151 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 127149 2024-09-13T14:50:18.152 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.152 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:18.152 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.152 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:18.152 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.086488+0000", 2024-09-13T14:50:18.152 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:18.152 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.152 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.152 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:18.152 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.086488+0000", 2024-09-13T14:50:18.152 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:18.152 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.152 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.153 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:18.153 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.086488+0000", 2024-09-13T14:50:18.153 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9500000000000003e-07 2024-09-13T14:50:18.153 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.153 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.153 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:18.153 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.086491+0000", 2024-09-13T14:50:18.153 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.5239999999999999e-06 2024-09-13T14:50:18.153 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.153 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.153 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:18.153 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.086498+0000", 2024-09-13T14:50:18.153 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1679999999999998e-06 2024-09-13T14:50:18.153 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.153 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.154 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:18.154 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.086503+0000", 2024-09-13T14:50:18.154 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5509999999999999e-06 2024-09-13T14:50:18.154 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.154 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.154 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:18.154 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.094294+0000", 2024-09-13T14:50:18.154 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0077904849999999998 2024-09-13T14:50:18.154 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.154 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.154 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:18.154 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.094390+0000", 2024-09-13T14:50:18.154 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.6314e-05 2024-09-13T14:50:18.155 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.155 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.155 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:50:18.155 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.094483+0000", 2024-09-13T14:50:18.155 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3119000000000001e-05 2024-09-13T14:50:18.155 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.155 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.155 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:18.155 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.099912+0000", 2024-09-13T14:50:18.155 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.005428357 2024-09-13T14:50:18.155 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:18.155 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:18.155 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:18.155 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:50:18.101824+0000", 2024-09-13T14:50:18.156 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00191231 2024-09-13T14:50:18.156 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:18.156 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:18.156 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:18.156 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:18.156 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2024-09-13T14:50:18.156 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 6 2024-09-13T14:50:18.156 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:50:18.181 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:18.226 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:18.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:18.328 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:18.353 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:18.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:18.454 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:18.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.496 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:18.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:18.597 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:18.634 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:18.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:18.735 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:18.758 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:18.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:18.860 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:18.903 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:18.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:18.967 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:50:19.005 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:19.031 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:19.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:19.133 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:19.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.161 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:19.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:19.263 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:19.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.292 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:19.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:19.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:18 smithi096 ceph-mon[98901]: pgmap v688: 99 pgs: 99 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 902 op/s 2024-09-13T14:50:19.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:18 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1871196468' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:19.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:18 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:19.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:18 smithi096 ceph-mon[102928]: pgmap v688: 99 pgs: 99 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 902 op/s 2024-09-13T14:50:19.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:18 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1871196468' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:19.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:18 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:19.393 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:19.431 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:19.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:19.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:18 smithi204 ceph-mon[87071]: pgmap v688: 99 pgs: 99 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 902 op/s 2024-09-13T14:50:19.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:18 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1871196468' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:19.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:18 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:19.533 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:19.579 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:19.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:19.681 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:19.708 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:19.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:19.810 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:19.839 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:19.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:20.010 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:20.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.039 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:20.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:20.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:19 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:20.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:19 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:50:20.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:19 smithi096 ceph-mon[98901]: osdmap e237: 8 total, 8 up, 8 in 2024-09-13T14:50:20.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:19 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:20.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:19 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:50:20.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:19 smithi096 ceph-mon[102928]: osdmap e237: 8 total, 8 up, 8 in 2024-09-13T14:50:20.139 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:20.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.173 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:20.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:20.274 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:20.300 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:20.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:20.370 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:19 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:20.370 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:19 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:50:20.370 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:19 smithi204 ceph-mon[87071]: osdmap e237: 8 total, 8 up, 8 in 2024-09-13T14:50:20.370 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:50:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:50:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:20.402 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:20.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.442 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:20.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:20.445 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:50:20.543 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:20.570 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:20.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:20.672 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:20.683 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:20.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.699 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:20.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:20.800 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:20.858 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:20.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:20.960 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:20.990 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:20.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:21.051 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:20 smithi096 ceph-mon[98901]: pgmap v690: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 12 MiB/s wr, 838 op/s 2024-09-13T14:50:21.051 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:20 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:21.051 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:20 smithi096 ceph-mon[102928]: pgmap v690: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 12 MiB/s wr, 838 op/s 2024-09-13T14:50:21.052 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:20 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:21.092 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:21.135 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:21.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:21.237 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:21.277 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:21.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:21.379 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:21.403 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:21.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:21.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:20 smithi204 ceph-mon[87071]: pgmap v690: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 12 MiB/s wr, 838 op/s 2024-09-13T14:50:21.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:20 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:21.505 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:21.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.539 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:21.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:21.640 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:21.669 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:21.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:21.771 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:21.790 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:50:21.799 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:21.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:21.901 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:21.924 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:21.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:22.026 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:22.053 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:22.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:22.155 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:22.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.202 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:22.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:22.303 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:22.337 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:22.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:22.439 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:22.488 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:22.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:22.590 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:22.644 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:22.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:22.747 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:22.773 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:22.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:22.875 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:22.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.899 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:22.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:23.000 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:23.039 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:23.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:23.141 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:23.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.174 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:23.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:23.275 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:23.302 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:23.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:23.308 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:22 smithi096 ceph-mon[98901]: pgmap v691: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 4.0 MiB/s wr, 841 op/s 2024-09-13T14:50:23.308 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:22 smithi096 ceph-mon[102928]: pgmap v691: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 4.0 MiB/s wr, 841 op/s 2024-09-13T14:50:23.404 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:23.439 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:23.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:22 smithi204 ceph-mon[87071]: pgmap v691: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 4.0 MiB/s wr, 841 op/s 2024-09-13T14:50:23.540 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:23.577 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:23.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:23.678 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:23.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.709 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:23.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:23.809 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:23.842 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:23.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:23.944 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:23.982 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:23.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:24.084 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:24.120 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:24.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:24.222 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:24.235 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:24.247 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:24.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:24.349 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:24.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.388 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:24.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:24.489 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:24.513 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:24.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:24.615 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:24.655 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:24.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:24.757 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:24.797 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:24.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:24.899 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:24.934 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:24.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:25.036 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:25.073 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:25.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:25.145 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:50:25.146 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:50:25.176 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:25.202 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:50:25.202 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:50:25.211 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:25.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:25.242 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:50:25.242 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:50:25.273 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:50:25.273 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:50:25.311 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:50:25.311 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:50:25.312 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:25.351 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:25 smithi096 ceph-mon[98901]: pgmap v692: 99 pgs: 99 active+clean; 6.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 595 KiB/s wr, 853 op/s 2024-09-13T14:50:25.352 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:25 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2527965931' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:25.352 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:25 smithi096 ceph-mon[102928]: pgmap v692: 99 pgs: 99 active+clean; 6.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 595 KiB/s wr, 853 op/s 2024-09-13T14:50:25.352 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:25 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2527965931' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:25.352 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:50:25.353 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:50:25.353 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:25.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:25.393 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:50:25.393 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:50:25.427 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:50:25.427 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:50:25.455 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:25.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:25 smithi204 ceph-mon[87071]: pgmap v692: 99 pgs: 99 active+clean; 6.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 595 KiB/s wr, 853 op/s 2024-09-13T14:50:25.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:25 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2527965931' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:25.499 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:25.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:25.601 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:25.634 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:25.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:25.735 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:25.783 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:25.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:25.885 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:25.911 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:25.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:26.013 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:26.061 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:26.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:26.163 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:26.188 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:26.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:26.289 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:26.312 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:26.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:26.322 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:26.413 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:26.472 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:26.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:26.574 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:26.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.613 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:26.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:26.715 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:26.743 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:26.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:26.845 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:26.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.873 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:26.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:26.974 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:27.010 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:27.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:27.112 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:27.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.158 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:27.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:27.259 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:27.287 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:27.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:27.379 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:27 smithi096 ceph-mon[98901]: pgmap v693: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2024-09-13T14:50:27.379 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:27 smithi096 ceph-mon[102928]: pgmap v693: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2024-09-13T14:50:27.389 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:27.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.430 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:27.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:27.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:27 smithi204 ceph-mon[87071]: pgmap v693: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2024-09-13T14:50:27.531 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:27.565 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:27.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:27.667 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:27.693 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:27.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:27.752 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:50:27.752 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:50:27.794 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:27.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.835 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:27.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:27.936 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:27.972 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:27.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:28.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:50:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:50:27] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.51.0" 2024-09-13T14:50:28.074 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:28.116 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:28.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:28.218 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:28.248 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:28.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:28.326 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:50:28.350 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:28.416 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:28.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:28.518 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:28.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.549 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:28.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:28.650 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:28.710 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:28.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:28.812 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:28.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.871 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:28.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:28.972 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:28.998 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:28.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:29.100 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:29.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.125 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:29.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:29.226 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:29.251 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:29.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:29.256 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:50:29.257 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T14:50:29.354 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:29.381 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:29.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:29 smithi204 ceph-mon[87071]: pgmap v694: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-09-13T14:50:29.483 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:29.518 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:29.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:29.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:29 smithi096 ceph-mon[98901]: pgmap v694: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-09-13T14:50:29.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:29 smithi096 ceph-mon[102928]: pgmap v694: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-09-13T14:50:29.620 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:29.662 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:29.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:29.764 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:29.800 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:29.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:29.902 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:29.925 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:29.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:30.027 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:30.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.065 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:30.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:30.166 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:30.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.207 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:30.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:30.308 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:30.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.331 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:30.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:30.431 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:30.457 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:30.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:50:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:50:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:30.559 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:30.587 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:30.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:30.689 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:30.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.740 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:30.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:30.841 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:30.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.868 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:30.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:30.969 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:31.014 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:31.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:31.116 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:31.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.161 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:31.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:31.179 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:50:31.262 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:31.299 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:31.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:31.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:31 smithi096 ceph-mon[98901]: pgmap v695: 99 pgs: 99 active+clean; 6.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-09-13T14:50:31.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:31 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2768659117' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:31.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:31 smithi096 ceph-mon[102928]: pgmap v695: 99 pgs: 99 active+clean; 6.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-09-13T14:50:31.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:31 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2768659117' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:31.401 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:31.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.439 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:31.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:31.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:31 smithi204 ceph-mon[87071]: pgmap v695: 99 pgs: 99 active+clean; 6.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-09-13T14:50:31.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:31 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2768659117' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:31.540 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:31.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.578 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:31.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:31.634 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:50:31.679 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:31.707 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:31.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:31.810 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:31.826 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:31.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.882 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:31.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:31.983 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:32.007 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:32.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:32.109 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:32.145 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:32.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:32.247 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:32.276 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:32.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:32.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:32 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:50:32.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:32 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:50:32.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:32 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2768659117' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:32.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:32 smithi096 ceph-mon[98901]: osdmap e238: 8 total, 8 up, 8 in 2024-09-13T14:50:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:32 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:50:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:32 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:50:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:32 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2768659117' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:32 smithi096 ceph-mon[102928]: osdmap e238: 8 total, 8 up, 8 in 2024-09-13T14:50:32.378 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:32.402 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:32.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:32.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:32 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:50:32.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:32 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:50:32.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:32 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2768659117' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:32.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:32 smithi204 ceph-mon[87071]: osdmap e238: 8 total, 8 up, 8 in 2024-09-13T14:50:32.504 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:32.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.555 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:32.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:32.656 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:32.690 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:32.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:32.792 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:32.824 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:32.831 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:32.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:32.933 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:32.956 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:32.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:33.057 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:33.095 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:33.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:33.196 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:33.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.226 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:33.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:33.326 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:33.326 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:33.386 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:33.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:33.431 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:33 smithi096 ceph-mon[98901]: pgmap v697: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2024-09-13T14:50:33.431 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:33 smithi096 ceph-mon[102928]: pgmap v697: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2024-09-13T14:50:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:33 smithi204 ceph-mon[87071]: pgmap v697: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2024-09-13T14:50:33.488 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:33.531 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:33.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:33.633 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:33.658 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:33.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:33.760 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:33.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.791 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:33.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:33.892 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:33.923 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:33.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:34.026 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:34.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.054 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:34.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:34.155 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:34.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.195 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:34.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:34.296 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:34.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.325 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:34.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:34.426 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:34.461 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:34.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:34.514 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:50:34.563 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:34.613 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:34.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:34.715 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:34.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.740 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:34.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:34.841 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:34.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.869 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:34.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:34.970 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:35.011 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:35.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:35.112 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:35.146 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:35.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:35.248 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:35.291 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:35.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:35.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:35 smithi096 ceph-mon[98901]: pgmap v698: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-09-13T14:50:35.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:35 smithi096 ceph-mon[102928]: pgmap v698: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-09-13T14:50:35.393 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:35.420 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:35.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:35.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:35 smithi204 ceph-mon[87071]: pgmap v698: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-09-13T14:50:35.522 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:35.563 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:35.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:35.664 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:35.689 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:35.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:35.791 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:35.818 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:35.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:35.919 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:35.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.946 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:35.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:36.047 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:36.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.069 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:36.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:36.170 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:36.195 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:36.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:36.297 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:36.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.339 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:36.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:36.440 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:36.466 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:36.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:36.568 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:36.603 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:36.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:36.705 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:36.731 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:36.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:36.833 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:36.870 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:36.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:36.971 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:50:36.971 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:36.971 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:36 smithi096 ceph-mon[98901]: pgmap v699: 99 pgs: 99 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2024-09-13T14:50:36.973 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:36.973 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:50:36.973 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:36.973 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:36 smithi096 ceph-mon[102928]: pgmap v699: 99 pgs: 99 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2024-09-13T14:50:36.997 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:36.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:37.099 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:37.139 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:37.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:50:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:36 smithi204 ceph-mon[87071]: pgmap v699: 99 pgs: 99 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2024-09-13T14:50:37.241 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:37.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.269 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:37.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:37.370 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:37.395 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:37.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:37.497 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:37.521 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:37.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:37.623 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:37.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.660 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:37.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:37.761 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:37.788 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:37.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:37.852 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T14:50:37.858 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:50:37.858 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:50:37.858 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:50:37.858 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:50:37.858 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:50:37.858 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:50:37.858 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:50:37.858 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:50:37.858 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:50:37.859 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:50:37.859 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:50:37.859 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:50:37.859 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:50:37.859 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:50:37.859 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:50:37.859 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:50:37.889 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:37.933 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:37.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:38.035 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:38.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.078 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:38.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:38.118 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:37 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4017429789' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:38.119 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:37 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3272823674' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:50:38.119 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:37 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:50:38.119 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:37 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4017429789' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:38.119 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:37 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3272823674' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:50:38.119 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:37 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:50:38.119 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:50:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:50:37] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.51.0" 2024-09-13T14:50:38.179 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:37 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4017429789' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:37 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3272823674' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:50:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:37 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:50:38.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.222 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:38.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:38.323 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:38.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.370 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:38.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:38.471 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:38.514 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:38.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:38.616 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:38.649 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:38.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:38.751 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:38.778 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:38.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:38.869 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T14:50:38.880 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:38.903 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:38.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:39.005 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:39.029 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:39.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:39.131 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:39.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.179 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:39.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:39.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:38 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:50:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:38 smithi204 ceph-mon[87071]: osdmap e239: 8 total, 8 up, 8 in 2024-09-13T14:50:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:38 smithi204 ceph-mon[87071]: pgmap v701: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2024-09-13T14:50:39.281 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:39.321 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:39.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:39.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:38 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:50:39.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:38 smithi096 ceph-mon[98901]: osdmap e239: 8 total, 8 up, 8 in 2024-09-13T14:50:39.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:38 smithi096 ceph-mon[98901]: pgmap v701: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2024-09-13T14:50:39.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:38 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:50:39.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:38 smithi096 ceph-mon[102928]: osdmap e239: 8 total, 8 up, 8 in 2024-09-13T14:50:39.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:38 smithi096 ceph-mon[102928]: pgmap v701: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2024-09-13T14:50:39.423 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:39.449 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:39.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:39.551 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:39.578 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:39.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:39.680 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:39.710 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:39.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:39.811 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:39.839 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:39.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:39.941 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:39.975 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:39.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:40.077 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:40.113 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:40.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:40.215 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:40.244 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:40.272 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:40.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:40.376 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:40.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.421 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:40.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:50:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:50:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:40.522 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:40.557 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:40.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:40.659 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:40.688 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:40.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:40.789 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:40.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.823 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:40.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:40.924 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:40.963 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:40.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:41.065 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:41.091 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:41.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:41.193 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:41.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:40 smithi204 ceph-mon[87071]: pgmap v702: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2024-09-13T14:50:41.220 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:41.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:41.322 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:41.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:40 smithi096 ceph-mon[98901]: pgmap v702: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2024-09-13T14:50:41.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:40 smithi096 ceph-mon[102928]: pgmap v702: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2024-09-13T14:50:41.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.349 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:41.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:41.450 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:41.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.478 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:41.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:41.569 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:50:41.578 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:41.601 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:50:41.601 INFO:teuthology.orchestra.run.smithi096.stdout: "size": 20, 2024-09-13T14:50:41.601 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 600, 2024-09-13T14:50:41.601 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:50:41.601 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.601 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181921 6.14 6:291668c5:::benchmark_data_smithi096_167087_object64693:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:50:41.601 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:16.682370+0000", 2024-09-13T14:50:41.601 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 204.91323646399999, 2024-09-13T14:50:41.601 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.92264975199999999, 2024-09-13T14:50:41.601 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.601 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.602 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.602 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.602 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.602 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.602 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181921 2024-09-13T14:50:41.602 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.602 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.602 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.602 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.602 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682370+0000", 2024-09-13T14:50:41.602 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.602 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.602 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.603 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.603 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682370+0000", 2024-09-13T14:50:41.603 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.603 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.603 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.603 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.603 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682371+0000", 2024-09-13T14:50:41.603 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4750000000000001e-06 2024-09-13T14:50:41.603 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.603 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.603 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.603 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682377+0000", 2024-09-13T14:50:41.603 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7749999999999998e-06 2024-09-13T14:50:41.603 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.604 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.604 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.604 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682393+0000", 2024-09-13T14:50:41.604 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5710999999999999e-05 2024-09-13T14:50:41.604 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.604 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.604 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.604 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682401+0000", 2024-09-13T14:50:41.604 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9140000000000008e-06 2024-09-13T14:50:41.604 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.604 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.604 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.604 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682470+0000", 2024-09-13T14:50:41.604 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9227000000000007e-05 2024-09-13T14:50:41.605 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.605 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.605 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.605 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682479+0000", 2024-09-13T14:50:41.605 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9220000000000003e-06 2024-09-13T14:50:41.605 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.605 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.605 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.605 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.601256+0000", 2024-09-13T14:50:41.605 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.91877762500000004 2024-09-13T14:50:41.605 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.605 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.606 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.606 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.601372+0000", 2024-09-13T14:50:41.606 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011595 2024-09-13T14:50:41.606 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.606 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.606 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:50:41.606 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.601494+0000", 2024-09-13T14:50:41.606 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012179400000000001 2024-09-13T14:50:41.606 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.606 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.606 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.606 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.604870+0000", 2024-09-13T14:50:41.606 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0033754520000000001 2024-09-13T14:50:41.606 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.607 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.607 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.607 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.604936+0000", 2024-09-13T14:50:41.607 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5977999999999997e-05 2024-09-13T14:50:41.607 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.607 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.607 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.607 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.604946+0000", 2024-09-13T14:50:41.607 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0729e-05 2024-09-13T14:50:41.607 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.607 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.607 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.607 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.604983+0000", 2024-09-13T14:50:41.608 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6902e-05 2024-09-13T14:50:41.608 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.608 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.608 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.608 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.605020+0000", 2024-09-13T14:50:41.608 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6297999999999997e-05 2024-09-13T14:50:41.608 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.608 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.608 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.608 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.608 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.608 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181940 6.14 6:2de03b2d:::benchmark_data_smithi096_167087_object64712:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:50:41.608 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:16.696647+0000", 2024-09-13T14:50:41.609 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 204.898959668, 2024-09-13T14:50:41.609 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.91128050299999996, 2024-09-13T14:50:41.609 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.609 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.609 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.609 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.609 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.609 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.609 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181940 2024-09-13T14:50:41.609 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.609 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.610 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.610 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.610 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696647+0000", 2024-09-13T14:50:41.610 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.610 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.610 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.610 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.610 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696647+0000", 2024-09-13T14:50:41.610 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.610 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.610 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.610 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.610 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696648+0000", 2024-09-13T14:50:41.611 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.579e-06 2024-09-13T14:50:41.611 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.611 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.611 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.611 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696654+0000", 2024-09-13T14:50:41.611 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1349999999999997e-06 2024-09-13T14:50:41.611 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.611 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.611 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.611 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696667+0000", 2024-09-13T14:50:41.611 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2727e-05 2024-09-13T14:50:41.611 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.611 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.612 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.612 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696675+0000", 2024-09-13T14:50:41.612 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1899999999999995e-06 2024-09-13T14:50:41.612 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.612 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.612 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.612 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696911+0000", 2024-09-13T14:50:41.612 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00023602599999999999 2024-09-13T14:50:41.612 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.612 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.612 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.612 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696920+0000", 2024-09-13T14:50:41.612 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8300000000000002e-06 2024-09-13T14:50:41.612 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.613 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.613 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.613 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.603441+0000", 2024-09-13T14:50:41.613 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.90652066200000003 2024-09-13T14:50:41.613 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.613 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.613 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.613 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.603565+0000", 2024-09-13T14:50:41.613 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012396799999999999 2024-09-13T14:50:41.613 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.613 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.613 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:50:41.613 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.603668+0000", 2024-09-13T14:50:41.614 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000102938 2024-09-13T14:50:41.614 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.614 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.614 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.614 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.606432+0000", 2024-09-13T14:50:41.614 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027647000000000001 2024-09-13T14:50:41.614 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.614 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.614 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.614 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.606482+0000", 2024-09-13T14:50:41.614 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9243999999999998e-05 2024-09-13T14:50:41.614 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.614 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.614 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.615 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.607831+0000", 2024-09-13T14:50:41.615 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0013495219999999999 2024-09-13T14:50:41.615 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.615 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.615 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.615 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.607881+0000", 2024-09-13T14:50:41.615 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9648999999999998e-05 2024-09-13T14:50:41.615 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.615 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.616 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.617 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.607927+0000", 2024-09-13T14:50:41.617 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6332999999999998e-05 2024-09-13T14:50:41.617 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.617 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.617 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.617 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.617 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.617 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182060 6.14 6:2db5f18f:::benchmark_data_smithi096_167087_object64832:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:50:41.617 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:16.790348+0000", 2024-09-13T14:50:41.617 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 204.80525878, 2024-09-13T14:50:41.617 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.81781258199999995, 2024-09-13T14:50:41.617 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.617 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.618 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.618 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.618 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.618 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.618 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182060 2024-09-13T14:50:41.618 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.618 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.618 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.618 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.618 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790348+0000", 2024-09-13T14:50:41.618 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.618 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.618 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.618 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.619 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790348+0000", 2024-09-13T14:50:41.619 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.619 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.619 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.619 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.619 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790349+0000", 2024-09-13T14:50:41.619 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6309999999999999e-06 2024-09-13T14:50:41.619 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.619 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.619 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.619 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790360+0000", 2024-09-13T14:50:41.619 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1270999999999999e-05 2024-09-13T14:50:41.619 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.619 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.620 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.620 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790374+0000", 2024-09-13T14:50:41.620 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3995000000000001e-05 2024-09-13T14:50:41.620 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.620 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.620 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.620 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790383+0000", 2024-09-13T14:50:41.620 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.7520000000000002e-06 2024-09-13T14:50:41.620 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.620 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.620 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.620 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790459+0000", 2024-09-13T14:50:41.621 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.5407000000000005e-05 2024-09-13T14:50:41.621 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.621 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.621 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.621 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790467+0000", 2024-09-13T14:50:41.621 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.3089999999999996e-06 2024-09-13T14:50:41.621 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.621 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.621 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.621 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.605048+0000", 2024-09-13T14:50:41.621 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.81458113899999995 2024-09-13T14:50:41.621 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.621 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.622 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.622 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.605154+0000", 2024-09-13T14:50:41.622 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010594600000000001 2024-09-13T14:50:41.622 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.622 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.622 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:50:41.622 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.605265+0000", 2024-09-13T14:50:41.622 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011089700000000001 2024-09-13T14:50:41.622 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.622 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.622 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.622 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608038+0000", 2024-09-13T14:50:41.622 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027728029999999999 2024-09-13T14:50:41.622 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.623 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.623 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.623 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608082+0000", 2024-09-13T14:50:41.623 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4183e-05 2024-09-13T14:50:41.623 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.623 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.623 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.623 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608091+0000", 2024-09-13T14:50:41.623 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9770000000000006e-06 2024-09-13T14:50:41.623 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.623 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.623 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.623 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608118+0000", 2024-09-13T14:50:41.624 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7341000000000001e-05 2024-09-13T14:50:41.624 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.624 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.624 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.624 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608160+0000", 2024-09-13T14:50:41.624 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1931000000000002e-05 2024-09-13T14:50:41.624 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.624 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.624 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.624 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.624 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.624 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182076 6.14 6:2fdfb44e:::benchmark_data_smithi096_167087_object64848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:50:41.624 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:16.803957+0000", 2024-09-13T14:50:41.624 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 204.79164973100001, 2024-09-13T14:50:41.625 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.80631315599999998, 2024-09-13T14:50:41.625 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.625 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.625 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.625 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.625 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.625 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.625 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182076 2024-09-13T14:50:41.625 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.625 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.625 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.625 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.625 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803957+0000", 2024-09-13T14:50:41.626 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.626 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.626 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.626 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.626 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803957+0000", 2024-09-13T14:50:41.626 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.626 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.626 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.626 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.626 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803958+0000", 2024-09-13T14:50:41.626 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8989999999999999e-06 2024-09-13T14:50:41.626 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.626 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.627 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.627 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803964+0000", 2024-09-13T14:50:41.627 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6319999999999996e-06 2024-09-13T14:50:41.627 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.627 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.627 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.627 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803979+0000", 2024-09-13T14:50:41.627 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5045000000000001e-05 2024-09-13T14:50:41.627 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.627 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.627 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.627 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803987+0000", 2024-09-13T14:50:41.627 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3799999999999996e-06 2024-09-13T14:50:41.627 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.628 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.628 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.628 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.804041+0000", 2024-09-13T14:50:41.628 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4199000000000001e-05 2024-09-13T14:50:41.628 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.628 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.628 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.628 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.804048+0000", 2024-09-13T14:50:41.628 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3050000000000004e-06 2024-09-13T14:50:41.628 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.628 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.628 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.628 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.606528+0000", 2024-09-13T14:50:41.629 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.80247967200000003 2024-09-13T14:50:41.629 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.629 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.629 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.629 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.606642+0000", 2024-09-13T14:50:41.629 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011442699999999999 2024-09-13T14:50:41.629 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.629 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.629 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:50:41.629 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.606726+0000", 2024-09-13T14:50:41.629 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.4073000000000006e-05 2024-09-13T14:50:41.629 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.629 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.630 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.630 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608231+0000", 2024-09-13T14:50:41.630 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0015046090000000001 2024-09-13T14:50:41.630 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.630 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.630 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.630 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608248+0000", 2024-09-13T14:50:41.630 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7244e-05 2024-09-13T14:50:41.630 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.630 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.630 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.630 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.609959+0000", 2024-09-13T14:50:41.630 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001710563 2024-09-13T14:50:41.630 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.631 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.631 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.631 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.610178+0000", 2024-09-13T14:50:41.631 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00021929600000000001 2024-09-13T14:50:41.631 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.631 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.631 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.631 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.610270+0000", 2024-09-13T14:50:41.631 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.1811999999999994e-05 2024-09-13T14:50:41.631 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.631 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.631 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.632 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.632 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.632 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182107 6.14 6:2cba648f:::benchmark_data_smithi096_167087_object64879:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:50:41.632 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:16.833170+0000", 2024-09-13T14:50:41.632 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 204.76243586699999, 2024-09-13T14:50:41.632 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.78124621000000005, 2024-09-13T14:50:41.632 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.632 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.632 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.632 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.632 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.632 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.632 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182107 2024-09-13T14:50:41.633 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.633 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.633 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.633 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.633 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833170+0000", 2024-09-13T14:50:41.633 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.633 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.633 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.633 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.633 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833170+0000", 2024-09-13T14:50:41.633 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.633 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.633 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.633 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.634 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833172+0000", 2024-09-13T14:50:41.634 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6449999999999999e-06 2024-09-13T14:50:41.634 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.634 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.634 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.634 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833178+0000", 2024-09-13T14:50:41.634 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1770000000000003e-06 2024-09-13T14:50:41.634 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.634 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.634 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.634 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833192+0000", 2024-09-13T14:50:41.634 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4082e-05 2024-09-13T14:50:41.634 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.634 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.635 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.635 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833200+0000", 2024-09-13T14:50:41.635 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2520000000000002e-06 2024-09-13T14:50:41.635 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.635 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.635 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.635 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833256+0000", 2024-09-13T14:50:41.635 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6427e-05 2024-09-13T14:50:41.635 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.635 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.635 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.635 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833264+0000", 2024-09-13T14:50:41.636 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1629999999999996e-06 2024-09-13T14:50:41.636 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.636 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.636 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.636 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608283+0000", 2024-09-13T14:50:41.636 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.77501878800000001 2024-09-13T14:50:41.636 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.636 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.636 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.636 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608389+0000", 2024-09-13T14:50:41.636 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010569300000000001 2024-09-13T14:50:41.636 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.636 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.637 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:50:41.637 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608505+0000", 2024-09-13T14:50:41.637 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000116434 2024-09-13T14:50:41.637 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.637 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.637 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.637 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612122+0000", 2024-09-13T14:50:41.637 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0036171860000000001 2024-09-13T14:50:41.637 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.637 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.637 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.637 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.614303+0000", 2024-09-13T14:50:41.637 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0021807839999999998 2024-09-13T14:50:41.637 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.638 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.638 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.638 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.614323+0000", 2024-09-13T14:50:41.638 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9644e-05 2024-09-13T14:50:41.638 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.638 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.638 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.638 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.614369+0000", 2024-09-13T14:50:41.638 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6048999999999999e-05 2024-09-13T14:50:41.638 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.638 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.638 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.638 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.614417+0000", 2024-09-13T14:50:41.638 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7886000000000002e-05 2024-09-13T14:50:41.639 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.639 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.639 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.639 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.639 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.639 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181897 6.18 6:1a0fd876:::benchmark_data_smithi096_167087_object64669:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:50:41.639 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583616+0000", 2024-09-13T14:50:41.639 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 204.011990567, 2024-09-13T14:50:41.639 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.110115685, 2024-09-13T14:50:41.639 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.639 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.639 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.639 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.640 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.640 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.640 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181897 2024-09-13T14:50:41.640 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.640 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.640 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.640 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.640 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583616+0000", 2024-09-13T14:50:41.640 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.640 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.640 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.640 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.640 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583616+0000", 2024-09-13T14:50:41.641 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.641 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.641 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.641 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.641 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583617+0000", 2024-09-13T14:50:41.641 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.482e-06 2024-09-13T14:50:41.641 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.641 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.641 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.641 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583623+0000", 2024-09-13T14:50:41.641 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1190000000000002e-06 2024-09-13T14:50:41.641 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.641 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.641 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.642 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583637+0000", 2024-09-13T14:50:41.642 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3718e-05 2024-09-13T14:50:41.644 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.644 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.644 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.644 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583646+0000", 2024-09-13T14:50:41.644 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.602e-06 2024-09-13T14:50:41.644 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.644 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.645 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.645 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583943+0000", 2024-09-13T14:50:41.645 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00029740399999999997 2024-09-13T14:50:41.645 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.645 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.645 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.645 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583952+0000", 2024-09-13T14:50:41.645 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.4169999999999998e-06 2024-09-13T14:50:41.645 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.645 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.645 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.645 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.686789+0000", 2024-09-13T14:50:41.645 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1028368799999999 2024-09-13T14:50:41.645 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.646 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.646 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.646 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.686879+0000", 2024-09-13T14:50:41.646 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9676e-05 2024-09-13T14:50:41.646 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.646 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.646 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:50:41.646 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.687143+0000", 2024-09-13T14:50:41.646 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00026446800000000002 2024-09-13T14:50:41.646 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.646 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.646 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.646 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.692990+0000", 2024-09-13T14:50:41.646 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0058462310000000003 2024-09-13T14:50:41.647 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.647 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.647 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.647 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693001+0000", 2024-09-13T14:50:41.647 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1552999999999999e-05 2024-09-13T14:50:41.647 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.647 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.647 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.647 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693028+0000", 2024-09-13T14:50:41.647 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7104999999999999e-05 2024-09-13T14:50:41.647 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.647 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.647 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.648 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693064+0000", 2024-09-13T14:50:41.648 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.5503000000000002e-05 2024-09-13T14:50:41.648 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.648 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.648 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.648 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693489+0000", 2024-09-13T14:50:41.648 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00042502099999999998 2024-09-13T14:50:41.648 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.648 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.648 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.648 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693603+0000", 2024-09-13T14:50:41.648 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011386 2024-09-13T14:50:41.648 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.648 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.649 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.649 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693731+0000", 2024-09-13T14:50:41.649 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000128646 2024-09-13T14:50:41.649 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.649 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.649 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.649 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.649 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.649 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181961 6.18 6:1b4cf7bc:::benchmark_data_smithi096_167087_object64733:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:50:41.649 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583723+0000", 2024-09-13T14:50:41.649 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 204.01188335000001, 2024-09-13T14:50:41.649 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1144489200000001, 2024-09-13T14:50:41.649 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.650 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.650 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.650 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.650 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.650 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.650 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181961 2024-09-13T14:50:41.650 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.650 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.650 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.650 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.650 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583723+0000", 2024-09-13T14:50:41.650 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.650 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.651 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.651 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.651 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583723+0000", 2024-09-13T14:50:41.651 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.651 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.651 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.651 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.651 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583724+0000", 2024-09-13T14:50:41.651 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9100000000000004e-07 2024-09-13T14:50:41.651 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.651 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.651 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.651 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583727+0000", 2024-09-13T14:50:41.652 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.191e-06 2024-09-13T14:50:41.652 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.652 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.652 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.652 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583736+0000", 2024-09-13T14:50:41.652 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2480000000000002e-06 2024-09-13T14:50:41.652 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.652 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.652 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.652 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583742+0000", 2024-09-13T14:50:41.652 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3840000000000002e-06 2024-09-13T14:50:41.652 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.652 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.652 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.653 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583997+0000", 2024-09-13T14:50:41.653 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00025479700000000002 2024-09-13T14:50:41.653 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.653 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.653 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.653 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584006+0000", 2024-09-13T14:50:41.653 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9900000000000003e-06 2024-09-13T14:50:41.653 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.653 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.653 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.653 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.689501+0000", 2024-09-13T14:50:41.653 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.105495082 2024-09-13T14:50:41.653 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.654 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.654 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.654 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.689618+0000", 2024-09-13T14:50:41.654 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000116469 2024-09-13T14:50:41.654 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.654 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.654 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:50:41.654 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.689865+0000", 2024-09-13T14:50:41.654 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00024748899999999999 2024-09-13T14:50:41.654 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.654 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.654 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.654 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693075+0000", 2024-09-13T14:50:41.654 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0032099450000000001 2024-09-13T14:50:41.655 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.655 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.655 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.655 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693314+0000", 2024-09-13T14:50:41.655 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00023886600000000001 2024-09-13T14:50:41.655 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.655 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.655 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.655 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693417+0000", 2024-09-13T14:50:41.655 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000103482 2024-09-13T14:50:41.655 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.655 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.655 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.656 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693894+0000", 2024-09-13T14:50:41.656 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00047631099999999999 2024-09-13T14:50:41.656 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.656 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.656 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.656 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.697947+0000", 2024-09-13T14:50:41.656 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0040532220000000004 2024-09-13T14:50:41.656 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.656 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.656 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.656 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698010+0000", 2024-09-13T14:50:41.656 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3177999999999997e-05 2024-09-13T14:50:41.656 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.656 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.657 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.657 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698172+0000", 2024-09-13T14:50:41.657 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00016167500000000001 2024-09-13T14:50:41.657 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.657 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.657 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.657 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.657 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.657 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181983 6.18 6:1d5af90f:::benchmark_data_smithi096_167087_object64755:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:50:41.657 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583752+0000", 2024-09-13T14:50:41.657 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 204.011854368, 2024-09-13T14:50:41.657 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.114628411, 2024-09-13T14:50:41.657 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.658 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.658 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.658 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.658 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.658 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.658 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181983 2024-09-13T14:50:41.658 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.658 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.658 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.658 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.658 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583752+0000", 2024-09-13T14:50:41.658 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.658 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.658 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.659 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.659 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583752+0000", 2024-09-13T14:50:41.659 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.659 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.659 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.659 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.659 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583752+0000", 2024-09-13T14:50:41.659 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9099999999999999e-07 2024-09-13T14:50:41.659 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.659 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.659 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.659 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583754+0000", 2024-09-13T14:50:41.659 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7969999999999999e-06 2024-09-13T14:50:41.660 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.660 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.660 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.660 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583760+0000", 2024-09-13T14:50:41.660 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1600000000000003e-06 2024-09-13T14:50:41.660 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.660 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.660 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.660 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583765+0000", 2024-09-13T14:50:41.661 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0540000000000002e-06 2024-09-13T14:50:41.661 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.661 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.661 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.661 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584049+0000", 2024-09-13T14:50:41.661 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00028396299999999998 2024-09-13T14:50:41.661 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.661 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.661 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.661 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584057+0000", 2024-09-13T14:50:41.661 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.6920000000000002e-06 2024-09-13T14:50:41.661 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.661 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.661 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.662 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:41.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:41.663 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.691347+0000", 2024-09-13T14:50:41.663 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.107290417 2024-09-13T14:50:41.663 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.663 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.663 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.663 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.691416+0000", 2024-09-13T14:50:41.663 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9029000000000006e-05 2024-09-13T14:50:41.664 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.664 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.664 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:50:41.664 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.691630+0000", 2024-09-13T14:50:41.664 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00021387500000000001 2024-09-13T14:50:41.664 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.664 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.664 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.664 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693330+0000", 2024-09-13T14:50:41.664 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0016996730000000001 2024-09-13T14:50:41.664 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.664 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.665 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.665 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693794+0000", 2024-09-13T14:50:41.665 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00046411 2024-09-13T14:50:41.665 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.665 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.665 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.665 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693850+0000", 2024-09-13T14:50:41.665 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5733999999999999e-05 2024-09-13T14:50:41.665 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.665 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.665 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.665 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693924+0000", 2024-09-13T14:50:41.665 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3755000000000001e-05 2024-09-13T14:50:41.665 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.666 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.666 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.666 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698216+0000", 2024-09-13T14:50:41.666 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0042927490000000002 2024-09-13T14:50:41.666 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.666 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.666 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.666 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698273+0000", 2024-09-13T14:50:41.666 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6558000000000003e-05 2024-09-13T14:50:41.666 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.666 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.666 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.666 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698380+0000", 2024-09-13T14:50:41.667 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000107454 2024-09-13T14:50:41.667 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.667 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.667 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.667 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.667 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.667 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181992 6.18 6:1dd34154:::benchmark_data_smithi096_167087_object64764:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:50:41.667 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583771+0000", 2024-09-13T14:50:41.667 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 204.01183485000001, 2024-09-13T14:50:41.667 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1206707570000001, 2024-09-13T14:50:41.667 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.667 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.667 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.668 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.668 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.668 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.668 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181992 2024-09-13T14:50:41.668 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.668 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.668 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.668 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.668 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583771+0000", 2024-09-13T14:50:41.668 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.668 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.668 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.668 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.669 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583771+0000", 2024-09-13T14:50:41.669 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.669 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.669 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.669 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.669 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583772+0000", 2024-09-13T14:50:41.669 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6199999999999999e-07 2024-09-13T14:50:41.669 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.669 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.669 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.669 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583774+0000", 2024-09-13T14:50:41.669 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7179999999999999e-06 2024-09-13T14:50:41.669 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.670 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.670 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.670 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583779+0000", 2024-09-13T14:50:41.670 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9340000000000003e-06 2024-09-13T14:50:41.670 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.670 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.670 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.670 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583784+0000", 2024-09-13T14:50:41.670 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6569999999999996e-06 2024-09-13T14:50:41.670 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.670 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.670 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.670 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584091+0000", 2024-09-13T14:50:41.671 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00030664100000000002 2024-09-13T14:50:41.671 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.671 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.671 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.671 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584097+0000", 2024-09-13T14:50:41.671 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6640000000000001e-06 2024-09-13T14:50:41.671 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.671 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.671 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.671 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.694002+0000", 2024-09-13T14:50:41.671 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1099042059999999 2024-09-13T14:50:41.671 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.671 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.672 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.672 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.694103+0000", 2024-09-13T14:50:41.672 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000101208 2024-09-13T14:50:41.672 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.672 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.672 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:50:41.672 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.694412+0000", 2024-09-13T14:50:41.672 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00030881000000000002 2024-09-13T14:50:41.672 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.672 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.672 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.672 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698418+0000", 2024-09-13T14:50:41.672 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0040063989999999999 2024-09-13T14:50:41.673 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.673 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.673 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.673 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.703944+0000", 2024-09-13T14:50:41.673 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0055258440000000002 2024-09-13T14:50:41.673 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.673 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.673 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.673 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.703970+0000", 2024-09-13T14:50:41.673 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6594999999999999e-05 2024-09-13T14:50:41.673 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.673 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.673 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.673 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704017+0000", 2024-09-13T14:50:41.674 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6727999999999997e-05 2024-09-13T14:50:41.674 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.674 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.674 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.674 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704259+0000", 2024-09-13T14:50:41.674 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000241745 2024-09-13T14:50:41.674 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.674 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.674 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.674 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704318+0000", 2024-09-13T14:50:41.674 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9117000000000003e-05 2024-09-13T14:50:41.674 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.674 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.675 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.675 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704442+0000", 2024-09-13T14:50:41.675 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000124129 2024-09-13T14:50:41.675 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.675 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.675 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.675 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.675 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.675 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182029 6.18 6:1b014b20:::benchmark_data_smithi096_167087_object64801:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:50:41.675 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583790+0000", 2024-09-13T14:50:41.675 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 204.011816692, 2024-09-13T14:50:41.675 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1236003029999999, 2024-09-13T14:50:41.675 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.675 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.676 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.676 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.676 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.676 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.676 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182029 2024-09-13T14:50:41.676 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.676 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.676 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.676 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.676 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583790+0000", 2024-09-13T14:50:41.676 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.676 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.676 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.676 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.677 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583790+0000", 2024-09-13T14:50:41.677 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.677 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.677 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.677 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.677 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583790+0000", 2024-09-13T14:50:41.677 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.41e-07 2024-09-13T14:50:41.677 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.677 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.677 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.677 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583792+0000", 2024-09-13T14:50:41.677 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.734e-06 2024-09-13T14:50:41.677 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.678 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.678 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.678 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583798+0000", 2024-09-13T14:50:41.678 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8749999999999997e-06 2024-09-13T14:50:41.678 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.678 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.678 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.678 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583802+0000", 2024-09-13T14:50:41.678 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.685e-06 2024-09-13T14:50:41.678 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.678 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.678 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.678 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584124+0000", 2024-09-13T14:50:41.679 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00032207999999999998 2024-09-13T14:50:41.679 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.679 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.679 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.679 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584131+0000", 2024-09-13T14:50:41.679 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2439999999999998e-06 2024-09-13T14:50:41.679 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.679 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.679 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.679 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.695817+0000", 2024-09-13T14:50:41.679 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1116864829999999 2024-09-13T14:50:41.679 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.679 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.680 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.680 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.695938+0000", 2024-09-13T14:50:41.680 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000121177 2024-09-13T14:50:41.680 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.680 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.680 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:50:41.680 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.696188+0000", 2024-09-13T14:50:41.680 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00024953900000000001 2024-09-13T14:50:41.680 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.680 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.680 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.680 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698457+0000", 2024-09-13T14:50:41.680 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0022691069999999998 2024-09-13T14:50:41.681 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.681 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.681 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.681 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.703978+0000", 2024-09-13T14:50:41.681 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0055207859999999997 2024-09-13T14:50:41.681 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.681 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.681 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.681 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704061+0000", 2024-09-13T14:50:41.681 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.3146999999999995e-05 2024-09-13T14:50:41.681 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.681 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.681 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.682 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704145+0000", 2024-09-13T14:50:41.682 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.373e-05 2024-09-13T14:50:41.682 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.682 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.682 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.682 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707052+0000", 2024-09-13T14:50:41.682 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0029077640000000002 2024-09-13T14:50:41.682 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.682 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.682 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.682 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707257+0000", 2024-09-13T14:50:41.682 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00020468299999999999 2024-09-13T14:50:41.682 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.682 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.683 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.683 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707390+0000", 2024-09-13T14:50:41.683 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013292800000000001 2024-09-13T14:50:41.683 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.683 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.683 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.683 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.683 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.683 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182109 6.18 6:1e340a78:::benchmark_data_smithi096_167087_object64881:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:50:41.683 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583808+0000", 2024-09-13T14:50:41.683 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 204.01179866000001, 2024-09-13T14:50:41.683 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1239120069999999, 2024-09-13T14:50:41.683 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.683 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.684 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.684 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.684 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.684 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.684 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182109 2024-09-13T14:50:41.684 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.684 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.684 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.684 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.684 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583808+0000", 2024-09-13T14:50:41.684 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.684 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.684 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.685 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.685 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583808+0000", 2024-09-13T14:50:41.685 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.685 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.685 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.685 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.685 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583808+0000", 2024-09-13T14:50:41.685 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7099999999999997e-07 2024-09-13T14:50:41.685 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.685 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.685 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.685 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583810+0000", 2024-09-13T14:50:41.685 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.629e-06 2024-09-13T14:50:41.685 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.686 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.686 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.686 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583816+0000", 2024-09-13T14:50:41.686 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8930000000000002e-06 2024-09-13T14:50:41.686 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.686 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.686 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.686 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583820+0000", 2024-09-13T14:50:41.686 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7879999999999997e-06 2024-09-13T14:50:41.686 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.686 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.686 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.686 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584160+0000", 2024-09-13T14:50:41.687 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00034013899999999999 2024-09-13T14:50:41.687 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.687 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.687 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.687 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584168+0000", 2024-09-13T14:50:41.687 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.063e-06 2024-09-13T14:50:41.687 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.687 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.687 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.687 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698486+0000", 2024-09-13T14:50:41.687 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1143184820000001 2024-09-13T14:50:41.687 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.687 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.687 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.688 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698605+0000", 2024-09-13T14:50:41.688 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000119276 2024-09-13T14:50:41.688 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.688 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.688 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:50:41.688 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698944+0000", 2024-09-13T14:50:41.688 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0003389 2024-09-13T14:50:41.688 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.688 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.688 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.688 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.703983+0000", 2024-09-13T14:50:41.688 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0050390319999999997 2024-09-13T14:50:41.688 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.688 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.689 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.689 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704099+0000", 2024-09-13T14:50:41.689 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011535300000000001 2024-09-13T14:50:41.689 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.689 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.689 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.689 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704202+0000", 2024-09-13T14:50:41.689 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010301700000000001 2024-09-13T14:50:41.689 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.689 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.689 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.689 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707435+0000", 2024-09-13T14:50:41.689 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0032336909999999999 2024-09-13T14:50:41.690 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.690 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.690 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.690 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707572+0000", 2024-09-13T14:50:41.690 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000136923 2024-09-13T14:50:41.690 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.690 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.690 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.690 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707624+0000", 2024-09-13T14:50:41.690 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.1490000000000003e-05 2024-09-13T14:50:41.690 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.690 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.690 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.690 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707720+0000", 2024-09-13T14:50:41.691 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.5959999999999996e-05 2024-09-13T14:50:41.691 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.691 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.691 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.691 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.691 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.691 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182126 6.18 6:1d91b7c9:::benchmark_data_smithi096_167087_object64898:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:50:41.691 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583826+0000", 2024-09-13T14:50:41.691 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 204.01178042699999, 2024-09-13T14:50:41.691 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1240857499999999, 2024-09-13T14:50:41.691 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.691 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.691 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.692 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.692 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.692 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.692 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182126 2024-09-13T14:50:41.692 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.692 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.692 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.692 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.692 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583826+0000", 2024-09-13T14:50:41.692 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.692 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.692 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.692 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.693 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583826+0000", 2024-09-13T14:50:41.693 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.693 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.693 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.693 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.693 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583826+0000", 2024-09-13T14:50:41.693 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4900000000000001e-07 2024-09-13T14:50:41.693 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.693 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.693 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.693 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583828+0000", 2024-09-13T14:50:41.693 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6589999999999999e-06 2024-09-13T14:50:41.693 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.693 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.694 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.694 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583834+0000", 2024-09-13T14:50:41.694 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7799999999999997e-06 2024-09-13T14:50:41.694 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.694 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.694 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.694 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583839+0000", 2024-09-13T14:50:41.694 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.887e-06 2024-09-13T14:50:41.694 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.694 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.694 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.694 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584196+0000", 2024-09-13T14:50:41.694 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00035786699999999999 2024-09-13T14:50:41.695 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.695 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.695 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.695 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584204+0000", 2024-09-13T14:50:41.695 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2289999999999997e-06 2024-09-13T14:50:41.695 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.695 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.695 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.695 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.701272+0000", 2024-09-13T14:50:41.695 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1170683269999999 2024-09-13T14:50:41.695 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.695 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.695 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.696 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.701391+0000", 2024-09-13T14:50:41.696 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011928 2024-09-13T14:50:41.696 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.696 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.696 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:50:41.696 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.701768+0000", 2024-09-13T14:50:41.696 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00037707099999999998 2024-09-13T14:50:41.696 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.696 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.696 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.696 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707000+0000", 2024-09-13T14:50:41.696 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0052320379999999996 2024-09-13T14:50:41.696 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.696 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.697 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.697 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707013+0000", 2024-09-13T14:50:41.697 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2853999999999999e-05 2024-09-13T14:50:41.697 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.697 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.697 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.697 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707476+0000", 2024-09-13T14:50:41.697 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000462595 2024-09-13T14:50:41.697 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.697 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.697 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.697 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707518+0000", 2024-09-13T14:50:41.697 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2154000000000003e-05 2024-09-13T14:50:41.698 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.698 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.698 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.698 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707760+0000", 2024-09-13T14:50:41.698 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00024246099999999999 2024-09-13T14:50:41.698 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.698 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.698 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.698 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707802+0000", 2024-09-13T14:50:41.698 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1146999999999998e-05 2024-09-13T14:50:41.698 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.698 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.698 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.698 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707912+0000", 2024-09-13T14:50:41.699 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000110052 2024-09-13T14:50:41.699 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.699 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.699 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.699 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.699 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.699 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182128 6.18 6:1c1d9d13:::benchmark_data_smithi096_167087_object64900:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:50:41.699 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583844+0000", 2024-09-13T14:50:41.699 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 204.01176241300001, 2024-09-13T14:50:41.699 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1297894319999999, 2024-09-13T14:50:41.699 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.699 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.699 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.699 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.700 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.700 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.700 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182128 2024-09-13T14:50:41.700 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.700 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.700 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.700 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.700 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583844+0000", 2024-09-13T14:50:41.700 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.700 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.700 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.700 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.700 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583844+0000", 2024-09-13T14:50:41.701 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.701 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.701 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.701 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.701 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583844+0000", 2024-09-13T14:50:41.701 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6300000000000001e-07 2024-09-13T14:50:41.701 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.701 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.701 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.701 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583850+0000", 2024-09-13T14:50:41.701 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8209999999999998e-06 2024-09-13T14:50:41.701 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.701 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.701 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.702 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583856+0000", 2024-09-13T14:50:41.702 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8309999999999997e-06 2024-09-13T14:50:41.702 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.702 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.702 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.702 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583861+0000", 2024-09-13T14:50:41.702 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9969999999999998e-06 2024-09-13T14:50:41.702 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.702 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.702 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.702 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584231+0000", 2024-09-13T14:50:41.702 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00037027000000000002 2024-09-13T14:50:41.702 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.702 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.703 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.703 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584238+0000", 2024-09-13T14:50:41.703 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.562e-06 2024-09-13T14:50:41.703 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.703 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.703 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.703 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704485+0000", 2024-09-13T14:50:41.703 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.12024762 2024-09-13T14:50:41.703 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.703 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.703 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.703 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704619+0000", 2024-09-13T14:50:41.703 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013364200000000001 2024-09-13T14:50:41.704 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.704 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.704 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:50:41.704 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704986+0000", 2024-09-13T14:50:41.704 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00036733799999999999 2024-09-13T14:50:41.704 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.704 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.704 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.704 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713182+0000", 2024-09-13T14:50:41.704 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.008195239 2024-09-13T14:50:41.704 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.704 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.704 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.705 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713224+0000", 2024-09-13T14:50:41.705 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2001e-05 2024-09-13T14:50:41.705 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.705 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.705 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.705 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713287+0000", 2024-09-13T14:50:41.705 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3285000000000001e-05 2024-09-13T14:50:41.705 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.705 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.705 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.705 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713391+0000", 2024-09-13T14:50:41.705 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000104104 2024-09-13T14:50:41.705 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.706 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.707 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.707 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713434+0000", 2024-09-13T14:50:41.707 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3541000000000002e-05 2024-09-13T14:50:41.707 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.707 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.707 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.707 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713495+0000", 2024-09-13T14:50:41.707 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0489000000000003e-05 2024-09-13T14:50:41.707 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.707 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.707 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.707 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713633+0000", 2024-09-13T14:50:41.707 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000138329 2024-09-13T14:50:41.708 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.708 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.708 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.708 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.708 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.708 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182157 6.18 6:1a63e408:::benchmark_data_smithi096_167087_object64929:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:50:41.708 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583866+0000", 2024-09-13T14:50:41.708 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 204.011739871, 2024-09-13T14:50:41.708 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1353496999999999, 2024-09-13T14:50:41.708 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.708 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.708 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.708 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.709 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.709 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.709 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182157 2024-09-13T14:50:41.709 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.709 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.709 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.709 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.709 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583866+0000", 2024-09-13T14:50:41.709 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.709 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.709 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.710 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.710 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583866+0000", 2024-09-13T14:50:41.710 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.710 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.710 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.710 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.710 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583867+0000", 2024-09-13T14:50:41.710 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.41e-07 2024-09-13T14:50:41.710 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.710 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.710 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.710 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583868+0000", 2024-09-13T14:50:41.710 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5990000000000001e-06 2024-09-13T14:50:41.711 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.711 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.711 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.711 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583874+0000", 2024-09-13T14:50:41.711 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7579999999999998e-06 2024-09-13T14:50:41.711 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.711 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.711 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.711 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583890+0000", 2024-09-13T14:50:41.711 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6322999999999999e-05 2024-09-13T14:50:41.711 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.711 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.712 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.712 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584264+0000", 2024-09-13T14:50:41.712 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00037345899999999998 2024-09-13T14:50:41.712 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.712 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.712 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.712 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584271+0000", 2024-09-13T14:50:41.712 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6989999999999997e-06 2024-09-13T14:50:41.712 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.712 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.712 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.712 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707962+0000", 2024-09-13T14:50:41.712 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1236913660000001 2024-09-13T14:50:41.713 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.713 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.713 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.713 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.708079+0000", 2024-09-13T14:50:41.713 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000116842 2024-09-13T14:50:41.713 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.713 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.713 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:50:41.713 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.708445+0000", 2024-09-13T14:50:41.713 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00036596699999999997 2024-09-13T14:50:41.713 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.713 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.713 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.713 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713237+0000", 2024-09-13T14:50:41.714 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0047922049999999999 2024-09-13T14:50:41.714 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.714 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.714 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.714 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713680+0000", 2024-09-13T14:50:41.714 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000443163 2024-09-13T14:50:41.714 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.714 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.714 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.714 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713732+0000", 2024-09-13T14:50:41.714 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2092000000000002e-05 2024-09-13T14:50:41.714 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.715 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.715 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.715 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713822+0000", 2024-09-13T14:50:41.715 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9532999999999998e-05 2024-09-13T14:50:41.715 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.715 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.715 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.715 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.718875+0000", 2024-09-13T14:50:41.715 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0050529379999999999 2024-09-13T14:50:41.715 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.715 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.715 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.715 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719086+0000", 2024-09-13T14:50:41.715 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00021114800000000001 2024-09-13T14:50:41.716 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.716 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.716 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.716 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719216+0000", 2024-09-13T14:50:41.716 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013026699999999999 2024-09-13T14:50:41.716 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.716 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.716 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.716 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.716 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.716 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182208 6.18 6:1bb7076d:::benchmark_data_smithi096_167087_object64980:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:50:41.716 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583897+0000", 2024-09-13T14:50:41.716 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 204.01170912000001, 2024-09-13T14:50:41.717 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.135635011, 2024-09-13T14:50:41.717 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.717 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.717 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.717 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.717 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.717 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.717 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182208 2024-09-13T14:50:41.717 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.717 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.717 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.717 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.717 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583897+0000", 2024-09-13T14:50:41.717 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.718 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.718 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.718 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.718 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583897+0000", 2024-09-13T14:50:41.718 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.718 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.718 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.718 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.718 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583898+0000", 2024-09-13T14:50:41.718 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6300000000000001e-07 2024-09-13T14:50:41.718 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.718 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.718 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.718 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583899+0000", 2024-09-13T14:50:41.719 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.885e-06 2024-09-13T14:50:41.719 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.719 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.719 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.719 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583905+0000", 2024-09-13T14:50:41.719 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.003e-06 2024-09-13T14:50:41.719 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.719 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.719 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.719 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583910+0000", 2024-09-13T14:50:41.719 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8919999999999999e-06 2024-09-13T14:50:41.719 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.719 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.720 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.720 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584608+0000", 2024-09-13T14:50:41.720 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00069783300000000003 2024-09-13T14:50:41.720 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.720 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.720 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.720 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584616+0000", 2024-09-13T14:50:41.720 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.2209999999999997e-06 2024-09-13T14:50:41.720 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.720 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.720 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.720 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.710538+0000", 2024-09-13T14:50:41.720 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.125921197 2024-09-13T14:50:41.721 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.721 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.721 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.721 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.710659+0000", 2024-09-13T14:50:41.721 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000121261 2024-09-13T14:50:41.721 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.721 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.721 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:50:41.721 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.710998+0000", 2024-09-13T14:50:41.721 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00033875300000000001 2024-09-13T14:50:41.721 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.721 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.721 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.722 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713773+0000", 2024-09-13T14:50:41.722 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027755750000000002 2024-09-13T14:50:41.722 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.722 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.722 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.722 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.718788+0000", 2024-09-13T14:50:41.722 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0050151279999999998 2024-09-13T14:50:41.722 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.722 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.722 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.722 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.718829+0000", 2024-09-13T14:50:41.722 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0686000000000003e-05 2024-09-13T14:50:41.722 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.723 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.723 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.723 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719261+0000", 2024-09-13T14:50:41.723 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00043173400000000002 2024-09-13T14:50:41.723 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.723 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.723 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.723 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719336+0000", 2024-09-13T14:50:41.723 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.5703000000000003e-05 2024-09-13T14:50:41.723 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.723 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.723 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.723 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719425+0000", 2024-09-13T14:50:41.724 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.852e-05 2024-09-13T14:50:41.724 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.724 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.724 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.724 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719532+0000", 2024-09-13T14:50:41.724 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010725700000000001 2024-09-13T14:50:41.724 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.724 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.724 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.724 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.724 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.724 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182214 6.18 6:197a4439:::benchmark_data_smithi096_167087_object64986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:50:41.725 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.612524+0000", 2024-09-13T14:50:41.725 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 203.98308179399999, 2024-09-13T14:50:41.725 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.11521797, 2024-09-13T14:50:41.725 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.725 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.725 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.725 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.725 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.725 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.725 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182214 2024-09-13T14:50:41.725 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.725 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.725 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.725 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.726 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612524+0000", 2024-09-13T14:50:41.726 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.726 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.726 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.726 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.726 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612524+0000", 2024-09-13T14:50:41.726 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.726 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.726 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.726 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.726 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612526+0000", 2024-09-13T14:50:41.726 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.347e-06 2024-09-13T14:50:41.726 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.727 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.727 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.727 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612531+0000", 2024-09-13T14:50:41.727 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9799999999999998e-06 2024-09-13T14:50:41.727 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.727 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.727 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.727 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612542+0000", 2024-09-13T14:50:41.727 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0831e-05 2024-09-13T14:50:41.727 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.727 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.727 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.727 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612554+0000", 2024-09-13T14:50:41.728 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2283000000000001e-05 2024-09-13T14:50:41.728 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.728 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.728 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.728 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612613+0000", 2024-09-13T14:50:41.728 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8967e-05 2024-09-13T14:50:41.728 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.728 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.728 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.728 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612622+0000", 2024-09-13T14:50:41.728 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0399999999999998e-06 2024-09-13T14:50:41.728 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.729 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.729 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.729 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713854+0000", 2024-09-13T14:50:41.729 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.101231611 2024-09-13T14:50:41.729 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.729 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.729 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.729 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713963+0000", 2024-09-13T14:50:41.729 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010902 2024-09-13T14:50:41.729 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.729 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.729 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:50:41.729 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.714339+0000", 2024-09-13T14:50:41.729 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00037666499999999999 2024-09-13T14:50:41.730 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.730 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.730 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.730 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.718841+0000", 2024-09-13T14:50:41.730 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0045017240000000004 2024-09-13T14:50:41.730 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.730 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.730 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.730 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719298+0000", 2024-09-13T14:50:41.730 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00045695499999999999 2024-09-13T14:50:41.730 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.730 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.730 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.730 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.724722+0000", 2024-09-13T14:50:41.731 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0054238799999999998 2024-09-13T14:50:41.731 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.731 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.731 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.731 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.724759+0000", 2024-09-13T14:50:41.731 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6752999999999999e-05 2024-09-13T14:50:41.731 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.731 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.731 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.731 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727512+0000", 2024-09-13T14:50:41.731 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027530179999999999 2024-09-13T14:50:41.731 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.731 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.731 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.732 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727622+0000", 2024-09-13T14:50:41.732 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011055100000000001 2024-09-13T14:50:41.732 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.732 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.732 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.732 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727742+0000", 2024-09-13T14:50:41.732 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000120345 2024-09-13T14:50:41.732 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.732 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.732 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.732 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.732 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.732 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182224 6.18 6:1a82ea4f:::benchmark_data_smithi096_167087_object64996:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:50:41.733 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.621148+0000", 2024-09-13T14:50:41.733 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 203.974458301, 2024-09-13T14:50:41.733 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.115259811, 2024-09-13T14:50:41.733 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.733 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.733 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.733 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.733 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.733 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.733 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182224 2024-09-13T14:50:41.733 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.733 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.733 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.734 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.734 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621148+0000", 2024-09-13T14:50:41.734 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.734 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.734 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.734 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.734 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621148+0000", 2024-09-13T14:50:41.734 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.734 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.734 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.734 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.734 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621149+0000", 2024-09-13T14:50:41.734 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4729999999999999e-06 2024-09-13T14:50:41.734 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.735 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.735 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.735 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621154+0000", 2024-09-13T14:50:41.735 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0010000000000001e-06 2024-09-13T14:50:41.735 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.735 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.735 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.735 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621167+0000", 2024-09-13T14:50:41.735 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2558000000000001e-05 2024-09-13T14:50:41.735 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.735 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.735 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.735 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621174+0000", 2024-09-13T14:50:41.736 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.4390000000000003e-06 2024-09-13T14:50:41.736 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.736 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.736 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.736 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621688+0000", 2024-09-13T14:50:41.736 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00051370099999999998 2024-09-13T14:50:41.736 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.736 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.736 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.736 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621697+0000", 2024-09-13T14:50:41.736 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0529999999999996e-06 2024-09-13T14:50:41.736 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.737 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.737 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.737 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.716358+0000", 2024-09-13T14:50:41.737 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0946611879999999 2024-09-13T14:50:41.737 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.737 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.737 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.737 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.716475+0000", 2024-09-13T14:50:41.737 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000116692 2024-09-13T14:50:41.737 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.737 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.737 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:50:41.738 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.716866+0000", 2024-09-13T14:50:41.738 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00039121499999999999 2024-09-13T14:50:41.738 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.738 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.738 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.738 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.724682+0000", 2024-09-13T14:50:41.738 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.007815443 2024-09-13T14:50:41.738 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.738 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.738 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.738 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727435+0000", 2024-09-13T14:50:41.738 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027537030000000001 2024-09-13T14:50:41.738 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.739 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.739 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.739 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727474+0000", 2024-09-13T14:50:41.739 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9038000000000001e-05 2024-09-13T14:50:41.739 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.739 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.739 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.739 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727783+0000", 2024-09-13T14:50:41.739 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00030899999999999998 2024-09-13T14:50:41.739 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.739 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.739 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.739 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736174+0000", 2024-09-13T14:50:41.739 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0083904540000000003 2024-09-13T14:50:41.740 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.740 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.740 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.740 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736271+0000", 2024-09-13T14:50:41.740 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7455999999999997e-05 2024-09-13T14:50:41.740 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.740 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.740 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.740 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736408+0000", 2024-09-13T14:50:41.740 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013639699999999999 2024-09-13T14:50:41.740 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.740 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.740 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.741 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.741 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.741 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182244 6.18 6:1bc8bd61:::benchmark_data_smithi096_167087_object65016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:50:41.741 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.649532+0000", 2024-09-13T14:50:41.741 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 203.94607428099999, 2024-09-13T14:50:41.741 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0882464169999999, 2024-09-13T14:50:41.741 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.741 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.741 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.741 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.741 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.741 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.741 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182244 2024-09-13T14:50:41.742 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.742 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.742 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.742 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.742 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649532+0000", 2024-09-13T14:50:41.742 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.742 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.742 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.742 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.742 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649532+0000", 2024-09-13T14:50:41.742 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.742 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.742 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.742 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.743 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649534+0000", 2024-09-13T14:50:41.743 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.643e-06 2024-09-13T14:50:41.743 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.743 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.743 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.743 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649539+0000", 2024-09-13T14:50:41.743 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.835e-06 2024-09-13T14:50:41.743 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.743 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.743 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.743 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649556+0000", 2024-09-13T14:50:41.743 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6881e-05 2024-09-13T14:50:41.743 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.743 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.744 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.744 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649564+0000", 2024-09-13T14:50:41.744 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.8609999999999998e-06 2024-09-13T14:50:41.744 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.744 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.744 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.744 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649620+0000", 2024-09-13T14:50:41.744 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5701000000000001e-05 2024-09-13T14:50:41.744 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.744 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.744 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.744 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649629+0000", 2024-09-13T14:50:41.744 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.867e-06 2024-09-13T14:50:41.745 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.745 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.745 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.745 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719571+0000", 2024-09-13T14:50:41.745 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0699419960000001 2024-09-13T14:50:41.745 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.745 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.745 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.745 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719712+0000", 2024-09-13T14:50:41.745 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000140896 2024-09-13T14:50:41.745 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.745 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.745 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:50:41.745 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.720080+0000", 2024-09-13T14:50:41.746 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00036818400000000002 2024-09-13T14:50:41.746 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.746 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.746 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.746 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727822+0000", 2024-09-13T14:50:41.746 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0077419489999999997 2024-09-13T14:50:41.746 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.746 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.746 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.746 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736451+0000", 2024-09-13T14:50:41.746 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0086296340000000006 2024-09-13T14:50:41.746 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.746 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.747 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.747 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736465+0000", 2024-09-13T14:50:41.747 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3386e-05 2024-09-13T14:50:41.747 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.747 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.747 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.747 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737385+0000", 2024-09-13T14:50:41.747 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00092018600000000003 2024-09-13T14:50:41.747 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.747 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.747 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.747 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737579+0000", 2024-09-13T14:50:41.747 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00019378700000000001 2024-09-13T14:50:41.748 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.748 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.748 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.748 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737639+0000", 2024-09-13T14:50:41.748 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0445999999999997e-05 2024-09-13T14:50:41.748 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.748 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.748 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.748 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737778+0000", 2024-09-13T14:50:41.748 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013916500000000001 2024-09-13T14:50:41.748 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.748 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.748 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.749 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.749 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.749 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182259 6.18 6:188e1d88:::benchmark_data_smithi096_167087_object65031:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:50:41.749 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.675788+0000", 2024-09-13T14:50:41.749 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 203.919818513, 2024-09-13T14:50:41.749 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0622151870000001, 2024-09-13T14:50:41.749 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.749 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.749 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.749 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.749 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.749 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.749 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182259 2024-09-13T14:50:41.750 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.750 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.750 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.750 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.750 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675788+0000", 2024-09-13T14:50:41.750 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.750 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.750 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.750 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.750 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675788+0000", 2024-09-13T14:50:41.750 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.750 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.750 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.751 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.751 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675789+0000", 2024-09-13T14:50:41.751 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3960000000000001e-06 2024-09-13T14:50:41.751 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.751 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.751 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.751 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675795+0000", 2024-09-13T14:50:41.751 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7359999999999998e-06 2024-09-13T14:50:41.751 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.751 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.751 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.751 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675806+0000", 2024-09-13T14:50:41.751 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0953e-05 2024-09-13T14:50:41.752 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.752 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.752 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.752 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675813+0000", 2024-09-13T14:50:41.752 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7660000000000001e-06 2024-09-13T14:50:41.752 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.752 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.752 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.752 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675869+0000", 2024-09-13T14:50:41.752 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6610000000000002e-05 2024-09-13T14:50:41.752 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.752 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.752 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.753 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675878+0000", 2024-09-13T14:50:41.753 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6039999999999993e-06 2024-09-13T14:50:41.753 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.753 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.753 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.753 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.722283+0000", 2024-09-13T14:50:41.753 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.046405502 2024-09-13T14:50:41.753 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.753 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.753 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.753 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.722393+0000", 2024-09-13T14:50:41.753 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000109314 2024-09-13T14:50:41.753 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.753 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.754 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:50:41.754 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.722717+0000", 2024-09-13T14:50:41.754 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00032454599999999999 2024-09-13T14:50:41.754 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.754 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.754 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.754 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727862+0000", 2024-09-13T14:50:41.754 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.005144511 2024-09-13T14:50:41.754 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.754 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.754 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.754 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736481+0000", 2024-09-13T14:50:41.754 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0086192339999999999 2024-09-13T14:50:41.755 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.755 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.755 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.755 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737339+0000", 2024-09-13T14:50:41.755 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00085788100000000001 2024-09-13T14:50:41.755 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.755 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.755 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.755 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737433+0000", 2024-09-13T14:50:41.755 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3827999999999997e-05 2024-09-13T14:50:41.755 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.755 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.755 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.755 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737825+0000", 2024-09-13T14:50:41.756 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00039279099999999997 2024-09-13T14:50:41.756 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.756 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.756 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.756 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737878+0000", 2024-09-13T14:50:41.756 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2431e-05 2024-09-13T14:50:41.756 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.756 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.756 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.756 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.738003+0000", 2024-09-13T14:50:41.756 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000125084 2024-09-13T14:50:41.756 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.756 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.757 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.757 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.757 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.757 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182332 6.18 6:1cc35df1:::benchmark_data_smithi096_167087_object65104:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:50:41.757 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.849479+0000", 2024-09-13T14:50:41.757 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 203.74612690399999, 2024-09-13T14:50:41.757 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.90088822300000004, 2024-09-13T14:50:41.757 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:50:41.757 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:50:41.757 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:41.757 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:50:41.757 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:50:41.757 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:50:41.758 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182332 2024-09-13T14:50:41.758 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.758 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:50:41.758 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.758 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:50:41.758 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849479+0000", 2024-09-13T14:50:41.758 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.758 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.758 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.758 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:50:41.758 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849479+0000", 2024-09-13T14:50:41.758 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:50:41.758 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.758 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.759 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:50:41.759 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849481+0000", 2024-09-13T14:50:41.759 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1820000000000001e-06 2024-09-13T14:50:41.759 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.759 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.759 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:50:41.759 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849485+0000", 2024-09-13T14:50:41.759 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4660000000000001e-06 2024-09-13T14:50:41.759 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.759 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.759 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:50:41.759 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849495+0000", 2024-09-13T14:50:41.759 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7189999999999996e-06 2024-09-13T14:50:41.759 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.760 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.760 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:50:41.760 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849500+0000", 2024-09-13T14:50:41.760 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2650000000000004e-06 2024-09-13T14:50:41.760 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.760 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.760 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.760 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849548+0000", 2024-09-13T14:50:41.760 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7787999999999997e-05 2024-09-13T14:50:41.760 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.760 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.760 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:50:41.760 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849564+0000", 2024-09-13T14:50:41.761 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6016e-05 2024-09-13T14:50:41.761 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.761 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.761 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:50:41.761 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727890+0000", 2024-09-13T14:50:41.761 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.87832629399999995 2024-09-13T14:50:41.761 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.761 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.761 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:50:41.761 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727992+0000", 2024-09-13T14:50:41.761 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000101783 2024-09-13T14:50:41.761 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.761 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.762 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:50:41.762 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.728376+0000", 2024-09-13T14:50:41.762 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00038430400000000003 2024-09-13T14:50:41.762 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.762 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.762 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.762 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737484+0000", 2024-09-13T14:50:41.762 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.009108105 2024-09-13T14:50:41.762 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.762 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.762 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.762 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737529+0000", 2024-09-13T14:50:41.762 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4348999999999998e-05 2024-09-13T14:50:41.763 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.763 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.763 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.763 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.738046+0000", 2024-09-13T14:50:41.763 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00051743500000000001 2024-09-13T14:50:41.763 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.763 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.763 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:50:41.763 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.750093+0000", 2024-09-13T14:50:41.763 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.012047333 2024-09-13T14:50:41.763 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.763 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.763 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:41.764 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.750184+0000", 2024-09-13T14:50:41.764 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0740000000000002e-05 2024-09-13T14:50:41.764 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.764 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.764 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:50:41.764 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.750249+0000", 2024-09-13T14:50:41.764 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4417000000000003e-05 2024-09-13T14:50:41.764 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:50:41.764 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:50:41.764 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:50:41.764 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.750368+0000", 2024-09-13T14:50:41.764 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000119027 2024-09-13T14:50:41.764 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.765 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.765 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.765 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:50:41.765 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:50:41.765 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:50:41.765 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:41.792 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:41.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:41.893 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:41.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.940 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:41.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:42.041 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:42.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.078 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:42.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:42.179 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:42.216 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:42.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:42.318 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:42.341 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:42.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:42.443 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:42.483 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:42.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:42.585 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:42.622 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:42.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:42.724 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:42.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.768 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:42.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:42.869 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:42.893 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:42.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:42.995 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:43.037 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:43.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:43.139 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:43.163 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:43.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:43.182 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:42 smithi096 ceph-mon[98901]: pgmap v703: 99 pgs: 99 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-09-13T14:50:43.183 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:42 smithi096 ceph-mon[102928]: pgmap v703: 99 pgs: 99 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-09-13T14:50:43.197 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:43.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:42 smithi204 ceph-mon[87071]: pgmap v703: 99 pgs: 99 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-09-13T14:50:43.231 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:50:43.265 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:43.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.318 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:43.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:43.419 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:43.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.446 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:43.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:43.547 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:43.601 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:43.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:43.704 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:43.726 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:43.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:43.828 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:43.854 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:43.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:43.956 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:43.984 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:43.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:44.085 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:44.111 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:44.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:44.213 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:44.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.249 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:44.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:44.350 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:44.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.391 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:44.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:44.492 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:44.539 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:44.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:44.641 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:44.667 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:44.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:44.769 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:44.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.807 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:44.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:44.908 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:44.953 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:44 smithi096 ceph-mon[98901]: pgmap v704: 99 pgs: 99 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2024-09-13T14:50:44.953 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:44 smithi096 ceph-mon[102928]: pgmap v704: 99 pgs: 99 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2024-09-13T14:50:44.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.955 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:44.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:45.056 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:45.087 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:45.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:45.189 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:45.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:44 smithi204 ceph-mon[87071]: pgmap v704: 99 pgs: 99 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2024-09-13T14:50:45.223 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:45.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:45.283 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:45.286 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:45.326 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:45.350 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:45.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:45.452 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:45.507 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:45.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:45.609 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:45.648 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:45.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:45.752 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:45.789 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:45.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:45.891 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:45.912 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:50:45.917 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:45.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:45.945 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:45 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2045570913' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:50:45.945 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:45 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:50:45.945 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:45 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2045570913' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:50:45.946 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:45 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:50:46.019 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:46.056 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:46.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:46.160 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:46.190 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:46.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:46.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:45 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2045570913' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:50:46.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:45 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:50:46.291 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:46.329 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:46.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:46.431 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:46.497 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:46.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:46.599 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:46.628 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:46.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:46.730 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:46.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.773 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:46.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:46.874 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:46.898 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:46.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:46.976 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:46 smithi096 ceph-mon[98901]: pgmap v705: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2024-09-13T14:50:46.977 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:46 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:46.977 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:46 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:50:46.977 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:46 smithi096 ceph-mon[98901]: osdmap e240: 8 total, 8 up, 8 in 2024-09-13T14:50:46.977 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:46 smithi096 ceph-mon[102928]: pgmap v705: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2024-09-13T14:50:46.977 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:46 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:46.977 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:46 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:50:46.977 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:46 smithi096 ceph-mon[102928]: osdmap e240: 8 total, 8 up, 8 in 2024-09-13T14:50:47.000 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:47.027 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:47.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:47.129 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:47.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.156 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:47.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:47.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:46 smithi204 ceph-mon[87071]: pgmap v705: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2024-09-13T14:50:47.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:46 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:47.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:46 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:50:47.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:46 smithi204 ceph-mon[87071]: osdmap e240: 8 total, 8 up, 8 in 2024-09-13T14:50:47.257 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:47.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.285 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:47.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:47.387 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:47.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:47.517 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:47.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.556 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:47.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:47.657 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:47.704 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:47.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:47.806 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:47.832 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:47.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:47.934 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:47.957 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:47.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:48.059 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:48.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:50:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:50:47] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.51.0" 2024-09-13T14:50:48.084 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:48.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:48.186 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:48.214 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:48.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:48.315 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:48.344 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:48.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:48.447 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:48.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.474 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:48.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:48.575 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:48.615 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:48.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:48.717 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:48.752 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:48.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:48.854 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:48.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.929 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:48.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:48.949 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T14:50:49.030 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:49.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.055 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:49.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:49.157 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:49.181 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:49.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:49.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:48 smithi204 ceph-mon[87071]: pgmap v707: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-13T14:50:49.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:48 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1256651252' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:50:49.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:48 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:50:49.283 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:49.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.312 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:49.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:48 smithi096 ceph-mon[98901]: pgmap v707: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-13T14:50:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:48 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1256651252' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:50:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:48 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:50:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:48 smithi096 ceph-mon[102928]: pgmap v707: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-13T14:50:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:48 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1256651252' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:50:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:48 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:50:49.412 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:49.455 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:49.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:49.557 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:49.603 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:49.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:49.705 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:49.728 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:49.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:49.781 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:50:49.830 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:49.863 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:49.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:49.965 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:50.002 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:50.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:50.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:49 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:50:50.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:49 smithi096 ceph-mon[98901]: osdmap e241: 8 total, 8 up, 8 in 2024-09-13T14:50:50.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:49 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:50:50.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:49 smithi096 ceph-mon[102928]: osdmap e241: 8 total, 8 up, 8 in 2024-09-13T14:50:50.104 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:50.154 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:50.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:50.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:49 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:50:50.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:49 smithi204 ceph-mon[87071]: osdmap e241: 8 total, 8 up, 8 in 2024-09-13T14:50:50.256 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:50.296 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:50.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:50.398 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:50.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.424 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:50.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:50:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:50:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:50.525 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:50.559 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:50.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:50.661 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:50.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.701 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:50.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:50.802 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:50.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.829 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:50.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:50.930 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:50.959 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:50.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:51.061 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:51.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.083 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:51.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:51.184 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:51.206 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:51.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:51.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:50 smithi204 ceph-mon[87071]: pgmap v709: 99 pgs: 99 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-09-13T14:50:51.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:50 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:50:51.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:50 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:51.307 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:50 smithi096 ceph-mon[98901]: pgmap v709: 99 pgs: 99 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-09-13T14:50:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:50 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:50:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:50 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:50 smithi096 ceph-mon[102928]: pgmap v709: 99 pgs: 99 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-09-13T14:50:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:50 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:50:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:50 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:51.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.330 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:51.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:51.430 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:51.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.468 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:51.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:51.569 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:51.601 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:51.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:51.703 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:51.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.729 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:51.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:51.830 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:51.856 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:51.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:51.958 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:51.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.988 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:51.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:52.089 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:52.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.134 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:52.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:52.235 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:52.262 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:52.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:52.364 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:52.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.406 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:52.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:52.507 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:52.532 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:52.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:52.634 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:52.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.662 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:52.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:52.763 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:52.814 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:52.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:52.916 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:52.947 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:52.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:53.025 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:52 smithi096 ceph-mon[98901]: pgmap v710: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-09-13T14:50:53.025 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:52 smithi096 ceph-mon[102928]: pgmap v710: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-09-13T14:50:53.049 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:53.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.082 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:53.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:53.183 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:53.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:52 smithi204 ceph-mon[87071]: pgmap v710: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-09-13T14:50:53.222 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:53.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:53.324 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:53.356 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:53.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:53.458 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:53.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.494 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:53.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:53.564 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T14:50:53.595 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:53.661 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:53.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:53.763 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:53.790 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:53.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:53.892 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:53.914 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:53.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:54.016 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:54.042 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:54.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:54.144 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:54.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.199 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:54.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:54.278 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:50:54.278 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:50:54.300 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:54.311 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:50:54.312 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:50:54.337 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:54.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:54.348 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:50:54.348 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:50:54.383 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:50:54.383 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:50:54.422 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:50:54.422 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:50:54.439 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:54.449 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:50:54.450 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:50:54.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.484 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:54.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:54.492 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:50:54.492 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:50:54.524 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:50:54.524 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:50:54.585 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:54.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.626 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:54.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:54.727 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:54.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.752 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:54.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:54.807 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:50:54.807 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:50:54.854 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:54.892 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:54.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:54.994 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:55.034 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:55.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:55.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:54 smithi096 ceph-mon[98901]: pgmap v711: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 888 op/s 2024-09-13T14:50:55.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:54 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/153290397' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:55.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:54 smithi096 ceph-mon[102928]: pgmap v711: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 888 op/s 2024-09-13T14:50:55.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:54 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/153290397' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:55.136 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:55.162 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:55.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:55.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:54 smithi204 ceph-mon[87071]: pgmap v711: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 888 op/s 2024-09-13T14:50:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:54 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/153290397' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:55.264 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:55.291 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:55.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:55.392 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:55.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.436 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:55.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:55.537 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:55.561 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:55.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:55.663 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:55.689 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:55.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:55.791 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:55.806 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:55.816 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:55.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:55.918 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:55.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.942 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:55.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:56.043 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:56.082 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:56.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:56.184 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:56.223 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:56.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:56.324 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:56.349 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:56.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:56.451 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:56.487 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:56.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:56.582 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:50:56.588 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:56.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.678 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:56.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:56.739 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:56.779 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:56.806 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:56.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:56.908 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:56.963 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:56.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:57.065 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:57.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.089 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:57.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:57.096 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:56 smithi096 ceph-mon[98901]: pgmap v712: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2024-09-13T14:50:57.096 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:56 smithi096 ceph-mon[102928]: pgmap v712: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2024-09-13T14:50:57.190 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:57.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:56 smithi204 ceph-mon[87071]: pgmap v712: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2024-09-13T14:50:57.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.220 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:57.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:57.321 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:57.345 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:57.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:57.447 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:57.483 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:57.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:57.534 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:50:57.585 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:57.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.627 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:57.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:57.728 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:57.752 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:57.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:57.854 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:57.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.903 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:57.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:58.004 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:58.033 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:58.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:58.135 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:58.158 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:58.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:58.260 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:58.287 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:58.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:58.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:50:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:50:57] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.51.0" 2024-09-13T14:50:58.388 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:58.415 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:58.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:58.517 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:58.543 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:58.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:58.645 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:58.669 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:58.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:58.771 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:58.815 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:58.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:58.917 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:58.944 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:58.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:59.045 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:59.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.093 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:59.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:59.175 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:59.194 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:59.216 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:59.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:59.276 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:50:58 smithi096 ceph-mon[98901]: pgmap v713: 99 pgs: 99 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2024-09-13T14:50:59.277 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:50:58 smithi096 ceph-mon[102928]: pgmap v713: 99 pgs: 99 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2024-09-13T14:50:59.318 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:59.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.365 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:59.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:59.403 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:59.467 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:59.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:50:58 smithi204 ceph-mon[87071]: pgmap v713: 99 pgs: 99 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2024-09-13T14:50:59.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.548 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:59.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:59.649 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:59.672 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:59.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:59.773 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:59.853 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:50:59.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:59.955 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:59.978 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:50:59.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:00.080 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:00.104 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:00.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:00.206 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:00.229 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:00.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:00.331 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:00.357 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:00.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:00.458 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:00.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:51:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:51:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:00.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.505 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:00.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:00.607 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:00.634 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:00.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:00.735 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:00.769 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:00.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:00.871 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:00.917 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:00.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:00.990 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T14:51:00.994 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:51:01.019 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:01.047 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:01.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:01.149 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:01.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.183 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:01.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:01.284 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:01.308 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:01.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:01.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:00 smithi096 ceph-mon[98901]: pgmap v714: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T14:51:01.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:01 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3771826424' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:51:01.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:01 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:51:01.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:01 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4130675202' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:01.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:01 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:01.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:01 smithi096 ceph-mon[102928]: pgmap v714: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T14:51:01.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:01 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3771826424' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:51:01.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:01 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:51:01.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:01 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4130675202' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:01.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:01 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:01.409 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:01.436 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:01.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:00 smithi204 ceph-mon[87071]: pgmap v714: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T14:51:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:00 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3771826424' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:51:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:00 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:51:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:00 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4130675202' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:00 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:01.539 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:01.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.582 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:01.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:01.683 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:01.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.731 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:01.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:01.832 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:01.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.874 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:01.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:01.975 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:02.003 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:02.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:02.042 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T14:51:02.105 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:02.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.157 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:02.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:02.259 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:02.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.309 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:02.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:02.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:02 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:02.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:02 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:51:02.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:02 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:02.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:02 smithi096 ceph-mon[98901]: osdmap e242: 8 total, 8 up, 8 in 2024-09-13T14:51:02.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:02 smithi096 ceph-mon[98901]: osdmap e243: 8 total, 8 up, 8 in 2024-09-13T14:51:02.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:02 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:02.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:02 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:51:02.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:02 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:02.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:02 smithi096 ceph-mon[102928]: osdmap e242: 8 total, 8 up, 8 in 2024-09-13T14:51:02.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:02 smithi096 ceph-mon[102928]: osdmap e243: 8 total, 8 up, 8 in 2024-09-13T14:51:02.410 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:02.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.463 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:02.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:02.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:02 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:02 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:51:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:02 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:02 smithi204 ceph-mon[87071]: osdmap e242: 8 total, 8 up, 8 in 2024-09-13T14:51:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:02 smithi204 ceph-mon[87071]: osdmap e243: 8 total, 8 up, 8 in 2024-09-13T14:51:02.565 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:02.591 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:02.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:02.693 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:02.745 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:02.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:02.847 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:02.882 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:02.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:02.984 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:03.013 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:03.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:03.115 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:03.142 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:03.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:03.244 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:03.287 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:03.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:03 smithi096 ceph-mon[98901]: pgmap v716: 99 pgs: 2 peering, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2024-09-13T14:51:03.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:03 smithi096 ceph-mon[102928]: pgmap v716: 99 pgs: 2 peering, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2024-09-13T14:51:03.390 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:03.417 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:03.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:03.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:03 smithi204 ceph-mon[87071]: pgmap v716: 99 pgs: 2 peering, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2024-09-13T14:51:03.520 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:03.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.550 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:03.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:03.651 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:03.680 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:03.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:03.784 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:03.826 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:03.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:03.928 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:03.959 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:03.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:04.061 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:04.112 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:04.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:04.214 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:04.247 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:04.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:04.349 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:04.375 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:04.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:04.477 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:04.501 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:04.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:04.560 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:04.604 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:04.650 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:04.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:04.752 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:04.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.779 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:04.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:04.881 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:04.905 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:04.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:05.007 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:05.047 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:05.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:05.062 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:05 smithi096 ceph-mon[98901]: pgmap v718: 99 pgs: 2 peering, 97 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 71 B/s, 0 objects/s recovering 2024-09-13T14:51:05.063 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:05 smithi096 ceph-mon[102928]: pgmap v718: 99 pgs: 2 peering, 97 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 71 B/s, 0 objects/s recovering 2024-09-13T14:51:05.117 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:51:05.117 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:51:05.150 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:05.177 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:05.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:05.278 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:05.305 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:05.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:05.407 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:05.443 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:05.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:05 smithi204 ceph-mon[87071]: pgmap v718: 99 pgs: 2 peering, 97 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 71 B/s, 0 objects/s recovering 2024-09-13T14:51:05.546 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:05.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.584 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:05.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:05.685 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:05.741 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:05.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:05.842 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:05.871 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:05.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:05.973 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:06.018 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:06.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:06.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:51:06.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:51:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:06.121 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:06.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.168 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:06.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:06.173 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:51:06.269 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:06.329 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:06.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:06.431 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:06.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:51:06.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:06.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.484 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:06.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:06.585 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:06.615 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:06.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:06.717 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:06.742 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:06.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:06.844 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:06.872 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:06.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:06.974 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:07.024 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:07.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:07.029 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:07 smithi096 ceph-mon[98901]: pgmap v719: 99 pgs: 2 peering, 97 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 71 B/s, 0 objects/s recovering 2024-09-13T14:51:07.127 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:07.157 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:07.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:07 smithi204 ceph-mon[87071]: pgmap v719: 99 pgs: 2 peering, 97 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 71 B/s, 0 objects/s recovering 2024-09-13T14:51:07.259 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:07.297 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:07.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:07.302 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:07 smithi096 ceph-mon[102928]: pgmap v719: 99 pgs: 2 peering, 97 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 71 B/s, 0 objects/s recovering 2024-09-13T14:51:07.399 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:07.430 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:07.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:07.497 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:51:07.532 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:07.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.576 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:07.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:07.677 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:07.704 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:07.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:07.806 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:07.855 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:07.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:07.961 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:07.993 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:07.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:08.095 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:08.124 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:08.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:08.185 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:51:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:51:07] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.51.0" 2024-09-13T14:51:08.226 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:08.251 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:08.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:08.353 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:08.379 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:08.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:08.481 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:08.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.539 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:08.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:08.540 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:51:08.540 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:51:08.584 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:51:08.584 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:51:08.618 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:51:08.618 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:51:08.640 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:08.649 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:51:08.650 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:51:08.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.679 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:08.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:08.680 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:51:08.681 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:51:08.714 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:51:08.714 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:51:08.747 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:51:08.747 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:51:08.780 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:08.783 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:51:08.783 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:51:08.803 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:08.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:08.905 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:08.955 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:08.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:09.057 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:09.098 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:09.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:09.200 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:09.226 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:09.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:09.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:09 smithi096 ceph-mon[98901]: pgmap v720: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 71 B/s, 0 objects/s recovering 2024-09-13T14:51:09.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:09 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1767390374' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:09.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:09 smithi096 ceph-mon[102928]: pgmap v720: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 71 B/s, 0 objects/s recovering 2024-09-13T14:51:09.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:09 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1767390374' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:09.328 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:09.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.368 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:09.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:09 smithi204 ceph-mon[87071]: pgmap v720: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 71 B/s, 0 objects/s recovering 2024-09-13T14:51:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:09 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1767390374' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:09.469 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:09.492 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:09.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:09.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:09.634 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:09.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:09.736 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:09.764 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:09.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:09.865 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:09.895 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:09.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:09.997 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:10.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.039 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:10.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:10.140 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:10.160 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:10.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:10.262 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:10.282 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:10.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:10.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:10 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2872258517' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:51:10.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:10 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:51:10.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:10 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2872258517' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:51:10.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:10 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:51:10.384 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:10.405 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:10.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:10 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2872258517' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:51:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:10 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:51:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:51:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:51:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:10.507 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:10.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.529 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:10.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:10.630 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:10.651 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:10.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:10.753 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:10.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.776 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:10.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:10.781 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T14:51:10.877 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:10.900 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:10.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:10.964 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:11.002 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:11.024 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:11.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:11.126 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:11.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.150 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:11.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:11.251 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:11.273 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:11.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:11.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:11 smithi096 ceph-mon[102928]: pgmap v721: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 64 B/s, 0 objects/s recovering 2024-09-13T14:51:11.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:11 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:51:11.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:11 smithi096 ceph-mon[102928]: osdmap e244: 8 total, 8 up, 8 in 2024-09-13T14:51:11.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:11 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:11.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:11 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:11.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:11 smithi096 ceph-mon[98901]: pgmap v721: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 64 B/s, 0 objects/s recovering 2024-09-13T14:51:11.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:11 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:51:11.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:11 smithi096 ceph-mon[98901]: osdmap e244: 8 total, 8 up, 8 in 2024-09-13T14:51:11.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:11 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:11.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:11 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:11.376 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:11.397 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:11.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:11.427 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:11.439 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T14:51:11.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:11 smithi204 ceph-mon[87071]: pgmap v721: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 64 B/s, 0 objects/s recovering 2024-09-13T14:51:11.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:11 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:51:11.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:11 smithi204 ceph-mon[87071]: osdmap e244: 8 total, 8 up, 8 in 2024-09-13T14:51:11.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:11 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:11.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:11 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:11.500 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:11.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.527 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:11.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:11.628 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:11.653 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:11.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:11.755 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:11.780 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:11.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:11.882 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:11.903 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:11.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:12.005 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:12.031 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:12.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:12.133 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:12.156 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:12.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:12.258 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:12.285 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:12.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:12.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:12 smithi096 ceph-mon[98901]: osdmap e245: 8 total, 8 up, 8 in 2024-09-13T14:51:12.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:12 smithi096 ceph-mon[98901]: osdmap e246: 8 total, 8 up, 8 in 2024-09-13T14:51:12.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:12 smithi096 ceph-mon[102928]: osdmap e245: 8 total, 8 up, 8 in 2024-09-13T14:51:12.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:12 smithi096 ceph-mon[102928]: osdmap e246: 8 total, 8 up, 8 in 2024-09-13T14:51:12.387 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:12.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.445 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:12.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:12.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:12 smithi204 ceph-mon[87071]: osdmap e245: 8 total, 8 up, 8 in 2024-09-13T14:51:12.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:12 smithi204 ceph-mon[87071]: osdmap e246: 8 total, 8 up, 8 in 2024-09-13T14:51:12.546 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:12.570 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:12.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:12.621 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:51:12.672 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:12.694 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:12.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:12.796 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:12.841 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:12.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:12.943 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:12.968 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:12.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:13.070 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:13.105 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:13.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:13.207 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:13.230 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:13.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:13.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:13 smithi096 ceph-mon[98901]: pgmap v724: 99 pgs: 3 remapped+peering, 1 peering, 95 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-13T14:51:13.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:13 smithi096 ceph-mon[98901]: 7.7 starting backfill to osd.3 from (0'0,0'0] MAX to 243'10645 2024-09-13T14:51:13.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:13 smithi096 ceph-mon[98901]: 7.1 starting backfill to osd.3 from (0'0,0'0] MAX to 243'10488 2024-09-13T14:51:13.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:13 smithi096 ceph-mon[98901]: 7.6 starting backfill to osd.3 from (0'0,0'0] MAX to 243'10735 2024-09-13T14:51:13.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:13 smithi096 ceph-mon[98901]: 7.c starting backfill to osd.3 from (0'0,0'0] MAX to 243'10502 2024-09-13T14:51:13.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:13 smithi096 ceph-mon[102928]: pgmap v724: 99 pgs: 3 remapped+peering, 1 peering, 95 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-13T14:51:13.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:13 smithi096 ceph-mon[102928]: 7.7 starting backfill to osd.3 from (0'0,0'0] MAX to 243'10645 2024-09-13T14:51:13.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:13 smithi096 ceph-mon[102928]: 7.1 starting backfill to osd.3 from (0'0,0'0] MAX to 243'10488 2024-09-13T14:51:13.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:13 smithi096 ceph-mon[102928]: 7.6 starting backfill to osd.3 from (0'0,0'0] MAX to 243'10735 2024-09-13T14:51:13.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:13 smithi096 ceph-mon[102928]: 7.c starting backfill to osd.3 from (0'0,0'0] MAX to 243'10502 2024-09-13T14:51:13.332 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:13.379 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:13.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:13.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:13 smithi204 ceph-mon[87071]: pgmap v724: 99 pgs: 3 remapped+peering, 1 peering, 95 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-13T14:51:13.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:13 smithi204 ceph-mon[87071]: 7.7 starting backfill to osd.3 from (0'0,0'0] MAX to 243'10645 2024-09-13T14:51:13.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:13 smithi204 ceph-mon[87071]: 7.1 starting backfill to osd.3 from (0'0,0'0] MAX to 243'10488 2024-09-13T14:51:13.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:13 smithi204 ceph-mon[87071]: 7.6 starting backfill to osd.3 from (0'0,0'0] MAX to 243'10735 2024-09-13T14:51:13.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:13 smithi204 ceph-mon[87071]: 7.c starting backfill to osd.3 from (0'0,0'0] MAX to 243'10502 2024-09-13T14:51:13.481 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:13.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.559 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:13.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:13.660 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:13.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.685 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:13.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:13.787 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:13.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.831 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:13.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:13.932 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:13.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.974 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:13.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:14.075 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:14.104 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:14.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:14.206 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:14.233 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:14.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:14.335 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:14.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.384 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:14.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:14.486 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:14.512 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:14.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:14.603 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:14.612 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:14.613 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:14.636 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:14.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:14.646 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:51:14.647 INFO:teuthology.orchestra.run.smithi096.stdout: "size": 20, 2024-09-13T14:51:14.647 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 600, 2024-09-13T14:51:14.647 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:51:14.647 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.647 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114226 6.b 6:d7783488:::benchmark_data_smithi096_167087_object114225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2024-09-13T14:51:14.647 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.021815+0000", 2024-09-13T14:51:14.647 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.611090359, 2024-09-13T14:51:14.648 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.482906662, 2024-09-13T14:51:14.648 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.648 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.648 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "reached pg", 2024-09-13T14:51:14.648 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.648 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.648 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.648 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114226 2024-09-13T14:51:14.648 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.648 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.648 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.648 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.648 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.021815+0000", 2024-09-13T14:51:14.649 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.649 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.649 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.649 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.649 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.021815+0000", 2024-09-13T14:51:14.649 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.649 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.649 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.649 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.649 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.021815+0000", 2024-09-13T14:51:14.649 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3e-07 2024-09-13T14:51:14.649 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.649 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.650 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.650 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.021842+0000", 2024-09-13T14:51:14.650 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6421e-05 2024-09-13T14:51:14.650 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.650 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.650 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.650 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.021850+0000", 2024-09-13T14:51:14.650 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.4279999999999996e-06 2024-09-13T14:51:14.650 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.650 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.650 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.650 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.021856+0000", 2024-09-13T14:51:14.650 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1299999999999998e-06 2024-09-13T14:51:14.651 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.651 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.651 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.651 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.022161+0000", 2024-09-13T14:51:14.651 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00030501900000000001 2024-09-13T14:51:14.651 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.651 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.651 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.651 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.022321+0000", 2024-09-13T14:51:14.651 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00015967999999999999 2024-09-13T14:51:14.651 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.651 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.651 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:14.652 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.022540+0000", 2024-09-13T14:51:14.652 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000219192 2024-09-13T14:51:14.652 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.652 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.652 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.652 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.039491+0000", 2024-09-13T14:51:14.652 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.016950995 2024-09-13T14:51:14.652 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.652 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.652 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.652 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.039502+0000", 2024-09-13T14:51:14.652 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1219e-05 2024-09-13T14:51:14.652 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.653 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.653 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.653 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.024969+0000", 2024-09-13T14:51:14.653 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.98546682799999996 2024-09-13T14:51:14.656 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.656 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.656 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.656 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.024983+0000", 2024-09-13T14:51:14.656 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3720000000000001e-05 2024-09-13T14:51:14.656 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.656 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.656 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.656 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.492284+0000", 2024-09-13T14:51:14.656 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4673012459999999 2024-09-13T14:51:14.656 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.656 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.657 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for degraded object", 2024-09-13T14:51:14.657 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.495492+0000", 2024-09-13T14:51:14.657 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0032073510000000002 2024-09-13T14:51:14.657 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.657 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.657 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.657 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.504666+0000", 2024-09-13T14:51:14.657 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0091746610000000006 2024-09-13T14:51:14.657 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.657 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.657 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.657 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.504721+0000", 2024-09-13T14:51:14.657 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5142000000000002e-05 2024-09-13T14:51:14.658 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.658 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.658 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.658 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.658 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.658 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114233 6.b 6:dcab5b09:::benchmark_data_smithi096_167087_object114232:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2024-09-13T14:51:14.658 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.023153+0000", 2024-09-13T14:51:14.658 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.60975231800001, 2024-09-13T14:51:14.658 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.481772334, 2024-09-13T14:51:14.658 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.658 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.658 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.658 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.659 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.659 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.659 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114233 2024-09-13T14:51:14.659 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.659 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.659 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.659 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.659 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.023153+0000", 2024-09-13T14:51:14.659 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.659 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.659 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.659 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.659 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.023153+0000", 2024-09-13T14:51:14.659 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.660 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.660 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.660 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.660 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.023153+0000", 2024-09-13T14:51:14.660 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.46e-07 2024-09-13T14:51:14.660 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.660 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.660 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.660 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.023194+0000", 2024-09-13T14:51:14.660 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1041999999999998e-05 2024-09-13T14:51:14.660 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.660 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.661 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.661 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.023201+0000", 2024-09-13T14:51:14.661 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2289999999999997e-06 2024-09-13T14:51:14.661 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.661 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.661 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.661 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.023214+0000", 2024-09-13T14:51:14.661 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2412000000000001e-05 2024-09-13T14:51:14.661 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.661 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.661 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.662 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.039985+0000", 2024-09-13T14:51:14.662 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.016771569 2024-09-13T14:51:14.662 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.662 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.662 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.662 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.039994+0000", 2024-09-13T14:51:14.662 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.7679999999999997e-06 2024-09-13T14:51:14.662 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.662 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.662 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.662 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025231+0000", 2024-09-13T14:51:14.662 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.98523645599999998 2024-09-13T14:51:14.662 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.662 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.663 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.663 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025248+0000", 2024-09-13T14:51:14.663 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.747e-05 2024-09-13T14:51:14.663 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.663 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.663 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.663 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.495747+0000", 2024-09-13T14:51:14.663 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.470498774 2024-09-13T14:51:14.663 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.663 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.663 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.663 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.495842+0000", 2024-09-13T14:51:14.663 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.4565e-05 2024-09-13T14:51:14.664 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.664 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.664 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4,7", 2024-09-13T14:51:14.664 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.495948+0000", 2024-09-13T14:51:14.664 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010654699999999999 2024-09-13T14:51:14.664 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.664 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.664 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.664 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.500336+0000", 2024-09-13T14:51:14.664 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0043875679999999997 2024-09-13T14:51:14.664 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.664 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.664 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.665 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.500401+0000", 2024-09-13T14:51:14.665 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4968000000000005e-05 2024-09-13T14:51:14.665 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.665 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.665 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.665 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.504373+0000", 2024-09-13T14:51:14.665 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0039726960000000004 2024-09-13T14:51:14.665 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.665 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.665 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.665 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.504833+0000", 2024-09-13T14:51:14.666 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00045969000000000001 2024-09-13T14:51:14.666 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.666 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.666 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.666 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.504866+0000", 2024-09-13T14:51:14.666 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.3507000000000003e-05 2024-09-13T14:51:14.666 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.666 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.666 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.666 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.504925+0000", 2024-09-13T14:51:14.666 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8627e-05 2024-09-13T14:51:14.666 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.667 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.667 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.667 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.667 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.667 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114272 6.b 6:d08f64ca:::benchmark_data_smithi096_167087_object114271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-09-13T14:51:14.667 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.082094+0000", 2024-09-13T14:51:14.667 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.55081117200001, 2024-09-13T14:51:14.667 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4230454730000002, 2024-09-13T14:51:14.667 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.667 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.667 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.667 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.667 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.667 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.668 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114272 2024-09-13T14:51:14.668 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.668 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.668 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.668 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.668 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.082094+0000", 2024-09-13T14:51:14.668 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.668 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.669 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.669 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.669 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.082094+0000", 2024-09-13T14:51:14.669 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.669 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.669 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.669 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.669 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.082095+0000", 2024-09-13T14:51:14.669 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.274e-06 2024-09-13T14:51:14.669 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.669 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.669 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.669 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.082117+0000", 2024-09-13T14:51:14.670 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1418999999999999e-05 2024-09-13T14:51:14.670 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.670 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.670 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.670 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.082129+0000", 2024-09-13T14:51:14.670 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2707999999999999e-05 2024-09-13T14:51:14.670 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.670 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.670 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.670 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.082178+0000", 2024-09-13T14:51:14.670 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8356999999999998e-05 2024-09-13T14:51:14.670 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.670 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.670 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.671 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.082312+0000", 2024-09-13T14:51:14.671 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013410900000000001 2024-09-13T14:51:14.671 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.671 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.671 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.671 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.082322+0000", 2024-09-13T14:51:14.671 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0497e-05 2024-09-13T14:51:14.671 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.671 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.671 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.671 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025281+0000", 2024-09-13T14:51:14.671 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.94295873200000002 2024-09-13T14:51:14.671 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.671 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.672 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.672 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025289+0000", 2024-09-13T14:51:14.672 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.5469999999999999e-06 2024-09-13T14:51:14.672 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.672 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.672 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.672 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.498222+0000", 2024-09-13T14:51:14.672 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4729332900000001 2024-09-13T14:51:14.672 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.672 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.672 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.672 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.498361+0000", 2024-09-13T14:51:14.672 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000139011 2024-09-13T14:51:14.673 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.673 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.673 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4,7", 2024-09-13T14:51:14.673 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.498539+0000", 2024-09-13T14:51:14.673 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00017858500000000001 2024-09-13T14:51:14.673 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.673 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.673 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.673 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.500541+0000", 2024-09-13T14:51:14.673 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0020018089999999998 2024-09-13T14:51:14.673 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.673 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.673 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.673 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.504326+0000", 2024-09-13T14:51:14.674 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.003784918 2024-09-13T14:51:14.674 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.674 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.674 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.674 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.504759+0000", 2024-09-13T14:51:14.674 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00043290000000000001 2024-09-13T14:51:14.674 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.674 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.674 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.674 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.504980+0000", 2024-09-13T14:51:14.674 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00022085299999999999 2024-09-13T14:51:14.674 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.674 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.675 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.675 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.505036+0000", 2024-09-13T14:51:14.675 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5899999999999997e-05 2024-09-13T14:51:14.675 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.675 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.675 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.675 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.505139+0000", 2024-09-13T14:51:14.675 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000103564 2024-09-13T14:51:14.675 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.675 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.675 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.675 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.675 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.676 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114276 6.b 6:da575631:::benchmark_data_smithi096_167087_object114275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-09-13T14:51:14.676 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.086831+0000", 2024-09-13T14:51:14.676 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.54607457499998, 2024-09-13T14:51:14.676 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4226697059999998, 2024-09-13T14:51:14.676 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.676 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.676 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.676 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.676 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.676 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.676 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114276 2024-09-13T14:51:14.676 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.676 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.677 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.677 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.677 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.086831+0000", 2024-09-13T14:51:14.677 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.677 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.677 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.677 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.677 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.086831+0000", 2024-09-13T14:51:14.677 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.677 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.677 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.677 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.677 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.086832+0000", 2024-09-13T14:51:14.677 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5039999999999999e-06 2024-09-13T14:51:14.678 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.678 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.678 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.678 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.086853+0000", 2024-09-13T14:51:14.678 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0738e-05 2024-09-13T14:51:14.678 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.678 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.678 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.678 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.086865+0000", 2024-09-13T14:51:14.678 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2548000000000001e-05 2024-09-13T14:51:14.678 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.678 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.678 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.679 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.086873+0000", 2024-09-13T14:51:14.679 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.7649999999999993e-06 2024-09-13T14:51:14.679 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.679 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.679 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.679 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.086940+0000", 2024-09-13T14:51:14.679 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7008000000000006e-05 2024-09-13T14:51:14.679 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.679 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.679 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.679 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.086950+0000", 2024-09-13T14:51:14.679 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0396e-05 2024-09-13T14:51:14.679 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.680 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.680 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.680 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025313+0000", 2024-09-13T14:51:14.680 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.93836235800000001 2024-09-13T14:51:14.680 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.680 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.680 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.680 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025320+0000", 2024-09-13T14:51:14.680 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6800000000000004e-06 2024-09-13T14:51:14.680 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.680 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.680 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.680 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.500574+0000", 2024-09-13T14:51:14.681 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4752545889999999 2024-09-13T14:51:14.681 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.681 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.681 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.681 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.500664+0000", 2024-09-13T14:51:14.681 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0280999999999998e-05 2024-09-13T14:51:14.681 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.681 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.681 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4,7", 2024-09-13T14:51:14.681 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.500766+0000", 2024-09-13T14:51:14.681 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000101155 2024-09-13T14:51:14.681 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.681 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.682 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.682 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.504415+0000", 2024-09-13T14:51:14.682 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0036495719999999998 2024-09-13T14:51:14.682 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.682 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.682 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.682 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.504427+0000", 2024-09-13T14:51:14.682 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1629e-05 2024-09-13T14:51:14.682 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.682 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.682 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.682 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.508964+0000", 2024-09-13T14:51:14.682 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.004537017 2024-09-13T14:51:14.683 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.683 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.683 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.683 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.509135+0000", 2024-09-13T14:51:14.683 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00017143 2024-09-13T14:51:14.683 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.683 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.683 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.683 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.509199+0000", 2024-09-13T14:51:14.683 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3659999999999997e-05 2024-09-13T14:51:14.683 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.683 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.683 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.684 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.509500+0000", 2024-09-13T14:51:14.684 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00030137600000000001 2024-09-13T14:51:14.684 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.684 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.684 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.684 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.684 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.684 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114292 6.b 6:dbae161d:::benchmark_data_smithi096_167087_object114291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-09-13T14:51:14.684 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.105157+0000", 2024-09-13T14:51:14.684 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.52774772599997, 2024-09-13T14:51:14.684 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.404634159, 2024-09-13T14:51:14.684 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.684 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.685 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.685 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.685 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.685 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.685 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114292 2024-09-13T14:51:14.685 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.685 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.685 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.685 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.685 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.105157+0000", 2024-09-13T14:51:14.685 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.685 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.685 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.685 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.686 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.105157+0000", 2024-09-13T14:51:14.686 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.686 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.686 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.686 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.686 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.105159+0000", 2024-09-13T14:51:14.686 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2899999999999999e-06 2024-09-13T14:51:14.686 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.686 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.686 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.686 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.105178+0000", 2024-09-13T14:51:14.686 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9556e-05 2024-09-13T14:51:14.686 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.686 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.687 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.687 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.105188+0000", 2024-09-13T14:51:14.687 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3899999999999999e-06 2024-09-13T14:51:14.687 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.687 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.687 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.687 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.105193+0000", 2024-09-13T14:51:14.687 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5929999999999996e-06 2024-09-13T14:51:14.687 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.687 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.687 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.687 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.105857+0000", 2024-09-13T14:51:14.687 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000663884 2024-09-13T14:51:14.687 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.688 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.688 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.688 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.105867+0000", 2024-09-13T14:51:14.688 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0336999999999999e-05 2024-09-13T14:51:14.688 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.688 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.688 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.688 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025539+0000", 2024-09-13T14:51:14.688 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.91967133099999998 2024-09-13T14:51:14.688 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.688 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.688 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.688 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025559+0000", 2024-09-13T14:51:14.688 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9794000000000001e-05 2024-09-13T14:51:14.689 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.689 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.689 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.689 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.502433+0000", 2024-09-13T14:51:14.689 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4768748549999999 2024-09-13T14:51:14.689 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.689 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.689 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.689 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.502521+0000", 2024-09-13T14:51:14.689 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8059000000000005e-05 2024-09-13T14:51:14.689 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.689 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.689 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4,7", 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.502626+0000", 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000104416 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.504794+0000", 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.002168143 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.504804+0000", 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0012e-05 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.691 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.691 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.509083+0000", 2024-09-13T14:51:14.691 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0042789350000000002 2024-09-13T14:51:14.691 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.691 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.691 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.691 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.509565+0000", 2024-09-13T14:51:14.691 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000482082 2024-09-13T14:51:14.691 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.691 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.691 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.691 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.509618+0000", 2024-09-13T14:51:14.691 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3269000000000001e-05 2024-09-13T14:51:14.691 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.692 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.692 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.692 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.509792+0000", 2024-09-13T14:51:14.692 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000173213 2024-09-13T14:51:14.692 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.692 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.692 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.692 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.692 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.692 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114295 6.b 6:d00a084b:::benchmark_data_smithi096_167087_object114294:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-09-13T14:51:14.692 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.106519+0000", 2024-09-13T14:51:14.692 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.52638598700003, 2024-09-13T14:51:14.692 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.406915079, 2024-09-13T14:51:14.693 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.693 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.693 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.693 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.693 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.694 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.694 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114295 2024-09-13T14:51:14.694 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.694 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.694 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.694 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.694 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.106519+0000", 2024-09-13T14:51:14.694 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.694 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.694 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.694 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.695 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.106519+0000", 2024-09-13T14:51:14.695 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.695 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.695 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.695 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.695 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.106521+0000", 2024-09-13T14:51:14.695 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4389999999999999e-06 2024-09-13T14:51:14.695 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.695 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.695 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.695 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.106541+0000", 2024-09-13T14:51:14.695 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0871999999999999e-05 2024-09-13T14:51:14.696 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.696 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.696 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.696 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.106566+0000", 2024-09-13T14:51:14.696 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.5041999999999998e-05 2024-09-13T14:51:14.696 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.696 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.696 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.696 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.106573+0000", 2024-09-13T14:51:14.696 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.024e-06 2024-09-13T14:51:14.696 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.696 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.696 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.697 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.106630+0000", 2024-09-13T14:51:14.697 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6462000000000003e-05 2024-09-13T14:51:14.697 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.697 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.697 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.697 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.106638+0000", 2024-09-13T14:51:14.697 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.6760000000000007e-06 2024-09-13T14:51:14.697 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.697 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.697 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.697 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025592+0000", 2024-09-13T14:51:14.697 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.91895443300000001 2024-09-13T14:51:14.697 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.697 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.698 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.698 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025597+0000", 2024-09-13T14:51:14.698 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0289999999999996e-06 2024-09-13T14:51:14.698 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.698 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.698 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.698 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.505211+0000", 2024-09-13T14:51:14.698 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.47961371 2024-09-13T14:51:14.698 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.698 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.698 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.698 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.505421+0000", 2024-09-13T14:51:14.698 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000209726 2024-09-13T14:51:14.699 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.699 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.699 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4,7", 2024-09-13T14:51:14.699 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.505623+0000", 2024-09-13T14:51:14.699 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00020288500000000001 2024-09-13T14:51:14.699 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.699 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.699 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.699 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.509842+0000", 2024-09-13T14:51:14.699 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0042182649999999997 2024-09-13T14:51:14.699 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.699 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.699 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.699 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.509849+0000", 2024-09-13T14:51:14.700 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.0960000000000003e-06 2024-09-13T14:51:14.700 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.700 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.700 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.700 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.509880+0000", 2024-09-13T14:51:14.700 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.1667e-05 2024-09-13T14:51:14.700 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.700 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.700 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.700 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.513248+0000", 2024-09-13T14:51:14.700 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0033672110000000002 2024-09-13T14:51:14.700 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.700 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.701 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.701 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.513311+0000", 2024-09-13T14:51:14.701 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3318999999999995e-05 2024-09-13T14:51:14.701 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.701 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.701 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.701 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.513434+0000", 2024-09-13T14:51:14.701 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000123223 2024-09-13T14:51:14.701 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.701 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.701 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.701 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.701 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.702 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114296 6.b 6:d7a36de1:::benchmark_data_smithi096_167087_object114295:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-09-13T14:51:14.702 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.111517+0000", 2024-09-13T14:51:14.702 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.521387807, 2024-09-13T14:51:14.702 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4229328020000001, 2024-09-13T14:51:14.702 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.702 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.702 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.702 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.702 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.702 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.702 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114296 2024-09-13T14:51:14.702 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.702 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.703 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.703 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.703 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111517+0000", 2024-09-13T14:51:14.703 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.703 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.703 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.703 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.703 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111517+0000", 2024-09-13T14:51:14.703 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.703 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.703 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.704 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.704 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111519+0000", 2024-09-13T14:51:14.704 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.336e-06 2024-09-13T14:51:14.704 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.704 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.704 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.704 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111565+0000", 2024-09-13T14:51:14.704 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6545999999999998e-05 2024-09-13T14:51:14.704 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.704 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.704 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.704 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111582+0000", 2024-09-13T14:51:14.704 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6931000000000001e-05 2024-09-13T14:51:14.705 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.705 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.705 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.705 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111591+0000", 2024-09-13T14:51:14.705 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9700000000000005e-06 2024-09-13T14:51:14.705 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.705 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.705 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.705 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111664+0000", 2024-09-13T14:51:14.705 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2677999999999998e-05 2024-09-13T14:51:14.705 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.705 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.705 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.705 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111690+0000", 2024-09-13T14:51:14.706 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6386000000000001e-05 2024-09-13T14:51:14.706 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.706 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.706 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.706 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025679+0000", 2024-09-13T14:51:14.706 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.91398933900000001 2024-09-13T14:51:14.706 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.706 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.706 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.706 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025686+0000", 2024-09-13T14:51:14.706 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.0079999999999996e-06 2024-09-13T14:51:14.706 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.706 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.707 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.707 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.509994+0000", 2024-09-13T14:51:14.707 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.484307834 2024-09-13T14:51:14.707 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.707 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.707 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.707 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.510145+0000", 2024-09-13T14:51:14.707 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00015037699999999999 2024-09-13T14:51:14.707 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.707 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.707 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4,7", 2024-09-13T14:51:14.707 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.510310+0000", 2024-09-13T14:51:14.707 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000165532 2024-09-13T14:51:14.707 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.708 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.708 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.708 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.520545+0000", 2024-09-13T14:51:14.708 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.010234452999999999 2024-09-13T14:51:14.708 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.708 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.708 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.708 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.520615+0000", 2024-09-13T14:51:14.708 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.0173999999999996e-05 2024-09-13T14:51:14.708 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.708 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.708 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.708 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.520629+0000", 2024-09-13T14:51:14.709 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4336e-05 2024-09-13T14:51:14.709 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.709 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.709 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.709 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.534270+0000", 2024-09-13T14:51:14.709 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.013640464 2024-09-13T14:51:14.709 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.709 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.709 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.709 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.534338+0000", 2024-09-13T14:51:14.709 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8711e-05 2024-09-13T14:51:14.709 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.709 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.709 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.710 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.534450+0000", 2024-09-13T14:51:14.710 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000111727 2024-09-13T14:51:14.710 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.710 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.710 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.710 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.710 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.710 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114297 6.b 6:df613612:::benchmark_data_smithi096_167087_object114296:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-09-13T14:51:14.710 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.111617+0000", 2024-09-13T14:51:14.710 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.521288379, 2024-09-13T14:51:14.710 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4230570739999999, 2024-09-13T14:51:14.710 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.710 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.711 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.711 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.711 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.711 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.711 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114297 2024-09-13T14:51:14.711 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.711 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.711 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.711 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.711 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111617+0000", 2024-09-13T14:51:14.711 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.711 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.711 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.712 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.712 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111617+0000", 2024-09-13T14:51:14.712 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.712 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.712 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.712 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.712 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111617+0000", 2024-09-13T14:51:14.712 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7400000000000003e-07 2024-09-13T14:51:14.712 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.712 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.712 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.712 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111639+0000", 2024-09-13T14:51:14.712 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1121e-05 2024-09-13T14:51:14.712 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.713 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.713 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.713 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111647+0000", 2024-09-13T14:51:14.713 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.2660000000000001e-06 2024-09-13T14:51:14.713 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.713 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.713 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.713 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111653+0000", 2024-09-13T14:51:14.713 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4470000000000003e-06 2024-09-13T14:51:14.713 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.713 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.713 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.713 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111717+0000", 2024-09-13T14:51:14.714 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4084999999999994e-05 2024-09-13T14:51:14.714 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.714 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.714 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.714 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.111726+0000", 2024-09-13T14:51:14.714 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.7679999999999997e-06 2024-09-13T14:51:14.714 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.714 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.714 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.714 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025707+0000", 2024-09-13T14:51:14.714 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.91398108600000005 2024-09-13T14:51:14.714 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.714 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.714 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.714 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025714+0000", 2024-09-13T14:51:14.715 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4200000000000004e-06 2024-09-13T14:51:14.715 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.715 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.715 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.715 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.513593+0000", 2024-09-13T14:51:14.715 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.487878933 2024-09-13T14:51:14.715 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.715 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.715 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.715 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.513752+0000", 2024-09-13T14:51:14.715 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00015969500000000001 2024-09-13T14:51:14.715 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.715 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.715 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4,7", 2024-09-13T14:51:14.716 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.514123+0000", 2024-09-13T14:51:14.716 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00037098699999999998 2024-09-13T14:51:14.716 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.716 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.716 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.716 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.520635+0000", 2024-09-13T14:51:14.716 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0065122160000000004 2024-09-13T14:51:14.716 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.716 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.716 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.716 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.520683+0000", 2024-09-13T14:51:14.716 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7630000000000003e-05 2024-09-13T14:51:14.717 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.717 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.717 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.717 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.520743+0000", 2024-09-13T14:51:14.717 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0034999999999997e-05 2024-09-13T14:51:14.717 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.717 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.717 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.717 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.534508+0000", 2024-09-13T14:51:14.717 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.013764558 2024-09-13T14:51:14.717 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.717 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.717 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.717 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.534572+0000", 2024-09-13T14:51:14.718 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3975999999999999e-05 2024-09-13T14:51:14.718 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.718 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.718 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.718 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.534674+0000", 2024-09-13T14:51:14.718 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010217699999999999 2024-09-13T14:51:14.718 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.718 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.718 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.718 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.718 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.718 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114316 6.b 6:da0561f0:::benchmark_data_smithi096_167087_object114315:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-09-13T14:51:14.718 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.132414+0000", 2024-09-13T14:51:14.718 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.50049076400001, 2024-09-13T14:51:14.719 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4025338629999999, 2024-09-13T14:51:14.719 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.719 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.719 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.719 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.719 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.719 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.719 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114316 2024-09-13T14:51:14.719 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.719 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.719 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.719 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.719 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.132414+0000", 2024-09-13T14:51:14.720 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.720 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.720 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.720 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.720 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.132414+0000", 2024-09-13T14:51:14.720 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.720 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.720 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.720 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.721 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.132416+0000", 2024-09-13T14:51:14.721 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9489999999999999e-06 2024-09-13T14:51:14.721 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.721 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.721 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.721 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.132437+0000", 2024-09-13T14:51:14.721 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0738999999999999e-05 2024-09-13T14:51:14.721 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.721 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.721 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.721 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.132454+0000", 2024-09-13T14:51:14.721 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6538e-05 2024-09-13T14:51:14.721 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.722 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.722 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.722 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.132461+0000", 2024-09-13T14:51:14.722 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.8099999999999998e-06 2024-09-13T14:51:14.722 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.722 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.722 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.722 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.132528+0000", 2024-09-13T14:51:14.722 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6781999999999999e-05 2024-09-13T14:51:14.722 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.722 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.722 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.722 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.132542+0000", 2024-09-13T14:51:14.722 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3889e-05 2024-09-13T14:51:14.723 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.723 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.723 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.723 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025782+0000", 2024-09-13T14:51:14.723 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.89323986 2024-09-13T14:51:14.723 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.723 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.723 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.723 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025790+0000", 2024-09-13T14:51:14.723 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.6839999999999996e-06 2024-09-13T14:51:14.723 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.723 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.723 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.724 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.517357+0000", 2024-09-13T14:51:14.724 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4915678510000001 2024-09-13T14:51:14.724 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.724 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.724 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.724 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.517463+0000", 2024-09-13T14:51:14.724 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010545299999999999 2024-09-13T14:51:14.724 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.724 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.724 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4,7", 2024-09-13T14:51:14.724 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.517566+0000", 2024-09-13T14:51:14.724 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000103601 2024-09-13T14:51:14.724 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.725 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.725 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.725 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.520791+0000", 2024-09-13T14:51:14.725 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0032241330000000001 2024-09-13T14:51:14.725 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.725 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.725 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.725 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.520814+0000", 2024-09-13T14:51:14.725 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3764000000000002e-05 2024-09-13T14:51:14.725 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.725 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.725 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.725 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.520845+0000", 2024-09-13T14:51:14.725 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.0165e-05 2024-09-13T14:51:14.726 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.726 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.726 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.726 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.534787+0000", 2024-09-13T14:51:14.726 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.013942273999999999 2024-09-13T14:51:14.726 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.726 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.726 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.726 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.534848+0000", 2024-09-13T14:51:14.726 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1663999999999998e-05 2024-09-13T14:51:14.726 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.726 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.726 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.727 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.534948+0000", 2024-09-13T14:51:14.727 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.9706999999999999e-05 2024-09-13T14:51:14.727 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.727 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.727 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.727 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.727 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.727 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114341 6.b 6:d0bbbeba:::benchmark_data_smithi096_167087_object114340:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-09-13T14:51:14.727 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.157998+0000", 2024-09-13T14:51:14.727 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.47490736899999, 2024-09-13T14:51:14.727 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3771527940000001, 2024-09-13T14:51:14.727 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.727 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.728 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.728 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.728 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.728 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.728 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114341 2024-09-13T14:51:14.728 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.728 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.728 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.728 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.728 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.157998+0000", 2024-09-13T14:51:14.728 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.728 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.728 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.729 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.729 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.157998+0000", 2024-09-13T14:51:14.729 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.729 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.729 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.729 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.729 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.157999+0000", 2024-09-13T14:51:14.729 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2309999999999999e-06 2024-09-13T14:51:14.729 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.729 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.729 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.729 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.158025+0000", 2024-09-13T14:51:14.729 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6353999999999999e-05 2024-09-13T14:51:14.730 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.730 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.730 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.730 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.158037+0000", 2024-09-13T14:51:14.730 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.184e-05 2024-09-13T14:51:14.730 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.730 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.730 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.730 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.158044+0000", 2024-09-13T14:51:14.730 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5679999999999996e-06 2024-09-13T14:51:14.730 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.730 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.730 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.731 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.158114+0000", 2024-09-13T14:51:14.731 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.0277999999999994e-05 2024-09-13T14:51:14.731 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.731 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.731 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.731 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.158122+0000", 2024-09-13T14:51:14.731 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.6279999999999996e-06 2024-09-13T14:51:14.731 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.731 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.731 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.731 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025833+0000", 2024-09-13T14:51:14.731 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.86771119900000004 2024-09-13T14:51:14.731 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.731 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.731 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.732 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025842+0000", 2024-09-13T14:51:14.732 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.4609999999999997e-06 2024-09-13T14:51:14.732 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.732 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.732 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.732 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.520891+0000", 2024-09-13T14:51:14.732 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.495048878 2024-09-13T14:51:14.732 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.732 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.732 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.732 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.521061+0000", 2024-09-13T14:51:14.732 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00017027399999999999 2024-09-13T14:51:14.732 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.733 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.733 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4,7", 2024-09-13T14:51:14.733 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.521233+0000", 2024-09-13T14:51:14.733 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00017169799999999999 2024-09-13T14:51:14.733 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.733 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.733 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.733 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.527804+0000", 2024-09-13T14:51:14.733 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0065710129999999997 2024-09-13T14:51:14.733 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.733 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.733 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.733 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.527852+0000", 2024-09-13T14:51:14.734 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7818999999999997e-05 2024-09-13T14:51:14.734 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.734 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.734 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.734 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.534690+0000", 2024-09-13T14:51:14.734 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0068382959999999998 2024-09-13T14:51:14.734 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.734 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.734 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.734 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.534999+0000", 2024-09-13T14:51:14.734 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00030852199999999998 2024-09-13T14:51:14.734 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.734 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.734 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.735 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.535053+0000", 2024-09-13T14:51:14.735 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4066e-05 2024-09-13T14:51:14.735 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.735 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.735 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.735 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.535151+0000", 2024-09-13T14:51:14.735 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7669000000000003e-05 2024-09-13T14:51:14.735 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.735 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.735 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.735 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.735 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.735 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114343 6.b 6:dfbcc871:::benchmark_data_smithi096_167087_object114342:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-09-13T14:51:14.736 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.162844+0000", 2024-09-13T14:51:14.736 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.470061117, 2024-09-13T14:51:14.736 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3727367429999999, 2024-09-13T14:51:14.736 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.736 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.736 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.736 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.736 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.736 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.736 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114343 2024-09-13T14:51:14.736 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.736 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.737 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.738 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.738 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.162844+0000", 2024-09-13T14:51:14.738 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.738 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.738 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.738 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.738 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.162844+0000", 2024-09-13T14:51:14.738 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.738 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.738 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.738 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.738 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.162845+0000", 2024-09-13T14:51:14.739 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.285e-06 2024-09-13T14:51:14.739 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.739 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.739 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.739 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.162865+0000", 2024-09-13T14:51:14.739 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9715e-05 2024-09-13T14:51:14.739 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.739 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.739 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.739 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.162880+0000", 2024-09-13T14:51:14.739 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5136e-05 2024-09-13T14:51:14.739 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.739 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.740 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.740 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.162887+0000", 2024-09-13T14:51:14.740 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.951e-06 2024-09-13T14:51:14.740 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.740 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.740 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.740 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.162948+0000", 2024-09-13T14:51:14.740 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1257e-05 2024-09-13T14:51:14.740 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.740 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.740 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.740 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.162958+0000", 2024-09-13T14:51:14.740 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.8679999999999994e-06 2024-09-13T14:51:14.741 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.741 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.741 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.741 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025874+0000", 2024-09-13T14:51:14.741 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.86291543699999995 2024-09-13T14:51:14.741 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.741 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.741 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.741 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025882+0000", 2024-09-13T14:51:14.741 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.2859999999999999e-06 2024-09-13T14:51:14.741 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.741 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.741 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.742 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.524245+0000", 2024-09-13T14:51:14.742 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.498362636 2024-09-13T14:51:14.742 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.742 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.742 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.742 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.524408+0000", 2024-09-13T14:51:14.742 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00016374400000000001 2024-09-13T14:51:14.742 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.742 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.742 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4,7", 2024-09-13T14:51:14.742 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.524597+0000", 2024-09-13T14:51:14.742 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00018886799999999999 2024-09-13T14:51:14.742 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.743 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.743 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.743 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.534710+0000", 2024-09-13T14:51:14.743 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.010112393000000001 2024-09-13T14:51:14.743 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.743 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.743 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.743 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.535202+0000", 2024-09-13T14:51:14.743 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00049257900000000004 2024-09-13T14:51:14.743 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.743 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.743 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.743 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.535282+0000", 2024-09-13T14:51:14.743 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0076999999999999e-05 2024-09-13T14:51:14.744 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.744 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.744 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.744 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.535447+0000", 2024-09-13T14:51:14.744 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00016527600000000001 2024-09-13T14:51:14.744 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.744 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.744 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.744 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.535490+0000", 2024-09-13T14:51:14.744 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2962e-05 2024-09-13T14:51:14.744 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.744 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.744 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.745 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.535581+0000", 2024-09-13T14:51:14.745 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0272999999999994e-05 2024-09-13T14:51:14.745 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.745 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.745 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.745 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.745 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.745 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114347 6.b 6:dd8b765a:::benchmark_data_smithi096_167087_object114346:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-09-13T14:51:14.745 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.166902+0000", 2024-09-13T14:51:14.745 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.46600311999998, 2024-09-13T14:51:14.745 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3769748769999999, 2024-09-13T14:51:14.745 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.745 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.745 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.746 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.746 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.746 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.746 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114347 2024-09-13T14:51:14.746 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.746 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.746 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.746 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.746 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.166902+0000", 2024-09-13T14:51:14.746 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.746 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.746 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.746 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.747 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.166902+0000", 2024-09-13T14:51:14.747 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.747 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.747 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.747 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.747 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.166903+0000", 2024-09-13T14:51:14.747 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.133e-06 2024-09-13T14:51:14.747 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.747 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.747 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.747 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.166922+0000", 2024-09-13T14:51:14.747 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9196999999999999e-05 2024-09-13T14:51:14.747 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.748 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.748 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.748 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.166933+0000", 2024-09-13T14:51:14.748 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1060000000000001e-05 2024-09-13T14:51:14.748 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.748 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.748 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.748 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.166940+0000", 2024-09-13T14:51:14.748 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4400000000000002e-06 2024-09-13T14:51:14.748 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.748 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.748 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.748 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.167001+0000", 2024-09-13T14:51:14.749 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1332000000000002e-05 2024-09-13T14:51:14.749 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.749 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.749 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.749 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.167011+0000", 2024-09-13T14:51:14.749 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.9899999999999992e-06 2024-09-13T14:51:14.749 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.749 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.749 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.749 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025943+0000", 2024-09-13T14:51:14.749 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.85893211800000002 2024-09-13T14:51:14.749 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.750 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.750 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.750 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025951+0000", 2024-09-13T14:51:14.750 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0790000000000001e-06 2024-09-13T14:51:14.750 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.750 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.750 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.750 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.527887+0000", 2024-09-13T14:51:14.750 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.501936095 2024-09-13T14:51:14.750 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.750 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.750 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.750 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.528017+0000", 2024-09-13T14:51:14.751 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000129958 2024-09-13T14:51:14.751 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.751 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.751 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4,7", 2024-09-13T14:51:14.751 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.528165+0000", 2024-09-13T14:51:14.751 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00014729200000000001 2024-09-13T14:51:14.751 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.751 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.751 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.751 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.534719+0000", 2024-09-13T14:51:14.751 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0065543340000000002 2024-09-13T14:51:14.751 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.751 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.751 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.752 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.535343+0000", 2024-09-13T14:51:14.752 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00062412500000000001 2024-09-13T14:51:14.752 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.752 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.752 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.752 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.535618+0000", 2024-09-13T14:51:14.752 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00027537500000000001 2024-09-13T14:51:14.752 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.752 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.752 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.752 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.543702+0000", 2024-09-13T14:51:14.752 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0080836610000000007 2024-09-13T14:51:14.753 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.753 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.753 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.753 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.543764+0000", 2024-09-13T14:51:14.753 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2310000000000005e-05 2024-09-13T14:51:14.753 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.753 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.753 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.753 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.543877+0000", 2024-09-13T14:51:14.753 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000112378 2024-09-13T14:51:14.753 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.753 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.753 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.754 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.754 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.754 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114353 6.b 6:da1a8463:::benchmark_data_smithi096_167087_object114352:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-09-13T14:51:14.754 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.171910+0000", 2024-09-13T14:51:14.754 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.460995336, 2024-09-13T14:51:14.754 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3722562979999999, 2024-09-13T14:51:14.754 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.754 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.754 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.754 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.754 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.754 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.754 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114353 2024-09-13T14:51:14.755 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.755 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.755 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.755 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.755 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.171910+0000", 2024-09-13T14:51:14.755 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.755 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.755 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.755 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.755 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.171910+0000", 2024-09-13T14:51:14.755 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.755 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.755 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.756 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.756 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.171911+0000", 2024-09-13T14:51:14.756 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0219999999999999e-06 2024-09-13T14:51:14.756 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.756 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.756 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.756 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.171930+0000", 2024-09-13T14:51:14.756 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9386e-05 2024-09-13T14:51:14.756 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.756 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.756 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.756 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.171943+0000", 2024-09-13T14:51:14.756 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2398999999999999e-05 2024-09-13T14:51:14.757 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.757 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.757 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.757 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.171949+0000", 2024-09-13T14:51:14.757 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8199999999999999e-06 2024-09-13T14:51:14.757 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.757 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.757 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.757 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.172013+0000", 2024-09-13T14:51:14.757 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4011999999999996e-05 2024-09-13T14:51:14.757 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.757 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.757 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.757 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.172023+0000", 2024-09-13T14:51:14.758 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.4110000000000002e-06 2024-09-13T14:51:14.758 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.758 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.758 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.758 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025978+0000", 2024-09-13T14:51:14.758 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.85395492299999998 2024-09-13T14:51:14.758 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.758 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.758 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.758 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.025985+0000", 2024-09-13T14:51:14.758 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.4050000000000003e-06 2024-09-13T14:51:14.758 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.758 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.758 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.759 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.531261+0000", 2024-09-13T14:51:14.759 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5052759500000001 2024-09-13T14:51:14.759 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.759 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.759 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.759 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.531389+0000", 2024-09-13T14:51:14.759 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012830200000000001 2024-09-13T14:51:14.759 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.759 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.759 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4,7", 2024-09-13T14:51:14.759 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.531519+0000", 2024-09-13T14:51:14.759 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000129653 2024-09-13T14:51:14.759 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.759 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.760 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.760 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.535359+0000", 2024-09-13T14:51:14.760 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0038397819999999999 2024-09-13T14:51:14.760 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.760 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.760 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.760 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.535412+0000", 2024-09-13T14:51:14.760 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3282000000000002e-05 2024-09-13T14:51:14.760 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.760 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.760 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.760 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.535729+0000", 2024-09-13T14:51:14.760 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00031699000000000001 2024-09-13T14:51:14.761 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.761 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.761 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.761 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.544021+0000", 2024-09-13T14:51:14.761 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0082923349999999996 2024-09-13T14:51:14.761 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.761 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.761 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.761 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.544074+0000", 2024-09-13T14:51:14.761 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2998999999999999e-05 2024-09-13T14:51:14.761 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.761 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.761 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.761 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.544166+0000", 2024-09-13T14:51:14.762 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.1626999999999995e-05 2024-09-13T14:51:14.762 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.762 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.762 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.762 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.762 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.762 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114356 6.b 6:d4f5d2bc:::benchmark_data_smithi096_167087_object114355:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-09-13T14:51:14.762 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.175704+0000", 2024-09-13T14:51:14.762 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.45720065, 2024-09-13T14:51:14.762 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3797433030000001, 2024-09-13T14:51:14.762 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.762 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.762 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.762 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114356 2024-09-13T14:51:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.175704+0000", 2024-09-13T14:51:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.175704+0000", 2024-09-13T14:51:14.763 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.764 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.764 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.764 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.764 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.175706+0000", 2024-09-13T14:51:14.764 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.291e-06 2024-09-13T14:51:14.764 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.764 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.764 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.764 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.175724+0000", 2024-09-13T14:51:14.764 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7850999999999999e-05 2024-09-13T14:51:14.764 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.764 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.764 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.765 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.175734+0000", 2024-09-13T14:51:14.765 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0647e-05 2024-09-13T14:51:14.765 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.765 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.765 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.765 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.175740+0000", 2024-09-13T14:51:14.765 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3000000000000001e-06 2024-09-13T14:51:14.765 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.765 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.765 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.765 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.175805+0000", 2024-09-13T14:51:14.765 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5493000000000005e-05 2024-09-13T14:51:14.765 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.766 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.766 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.766 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.175816+0000", 2024-09-13T14:51:14.766 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1192000000000001e-05 2024-09-13T14:51:14.766 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.766 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.766 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.766 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.026010+0000", 2024-09-13T14:51:14.766 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.85019409499999998 2024-09-13T14:51:14.766 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.766 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.766 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.766 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.026018+0000", 2024-09-13T14:51:14.767 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.6179999999999997e-06 2024-09-13T14:51:14.767 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.767 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.767 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.767 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.535880+0000", 2024-09-13T14:51:14.767 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.509862195 2024-09-13T14:51:14.767 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.767 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.767 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.767 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.536124+0000", 2024-09-13T14:51:14.767 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000243527 2024-09-13T14:51:14.767 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.767 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.768 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4,7", 2024-09-13T14:51:14.768 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.536304+0000", 2024-09-13T14:51:14.768 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00017988600000000001 2024-09-13T14:51:14.768 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.768 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.768 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.768 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.543652+0000", 2024-09-13T14:51:14.768 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0073484700000000002 2024-09-13T14:51:14.768 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.768 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.768 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.768 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.543936+0000", 2024-09-13T14:51:14.768 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00028392599999999999 2024-09-13T14:51:14.769 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.769 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.769 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.769 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.543983+0000", 2024-09-13T14:51:14.769 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6628000000000001e-05 2024-09-13T14:51:14.769 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.769 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.769 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.769 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.555291+0000", 2024-09-13T14:51:14.769 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011308682 2024-09-13T14:51:14.769 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.769 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.769 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.769 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.555346+0000", 2024-09-13T14:51:14.770 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4298000000000001e-05 2024-09-13T14:51:14.770 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.770 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.770 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.770 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.555448+0000", 2024-09-13T14:51:14.770 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000102204 2024-09-13T14:51:14.770 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.770 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.770 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.770 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.770 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.770 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114362 6.b 6:d0969e39:::benchmark_data_smithi096_167087_object114361:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-09-13T14:51:14.770 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.187779+0000", 2024-09-13T14:51:14.771 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.44512593799999, 2024-09-13T14:51:14.771 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3678495320000001, 2024-09-13T14:51:14.771 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.771 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.771 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.771 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.771 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.771 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.771 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114362 2024-09-13T14:51:14.771 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.771 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.771 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.771 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.772 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.187779+0000", 2024-09-13T14:51:14.772 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.772 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.772 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.772 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.772 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.187779+0000", 2024-09-13T14:51:14.772 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.772 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.772 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.772 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.772 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.187780+0000", 2024-09-13T14:51:14.772 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2619999999999999e-06 2024-09-13T14:51:14.772 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.773 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.773 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.773 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.187798+0000", 2024-09-13T14:51:14.773 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7385e-05 2024-09-13T14:51:14.773 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.773 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.773 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.773 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.187809+0000", 2024-09-13T14:51:14.773 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1026000000000001e-05 2024-09-13T14:51:14.773 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.773 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.773 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.773 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.187814+0000", 2024-09-13T14:51:14.774 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2159999999999997e-06 2024-09-13T14:51:14.774 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.774 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.774 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.774 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.187871+0000", 2024-09-13T14:51:14.774 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6511000000000002e-05 2024-09-13T14:51:14.774 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.774 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.774 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.774 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.187878+0000", 2024-09-13T14:51:14.774 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.537e-06 2024-09-13T14:51:14.774 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.774 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.775 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.775 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.026044+0000", 2024-09-13T14:51:14.775 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.83816608599999998 2024-09-13T14:51:14.775 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.775 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.775 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.775 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.026052+0000", 2024-09-13T14:51:14.775 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.4660000000000002e-06 2024-09-13T14:51:14.775 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.775 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.775 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.775 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.540298+0000", 2024-09-13T14:51:14.775 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.514245944 2024-09-13T14:51:14.776 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.776 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.776 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.776 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.540417+0000", 2024-09-13T14:51:14.776 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000119222 2024-09-13T14:51:14.776 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.776 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.776 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4,7", 2024-09-13T14:51:14.776 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.540564+0000", 2024-09-13T14:51:14.776 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00014722499999999999 2024-09-13T14:51:14.776 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.776 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.776 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.777 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.544177+0000", 2024-09-13T14:51:14.777 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.003613354 2024-09-13T14:51:14.777 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.777 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.777 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.777 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.555142+0000", 2024-09-13T14:51:14.777 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.01096427 2024-09-13T14:51:14.777 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.777 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.777 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.777 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.555218+0000", 2024-09-13T14:51:14.777 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.6515999999999997e-05 2024-09-13T14:51:14.777 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.777 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.778 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.778 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.555486+0000", 2024-09-13T14:51:14.778 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00026828799999999999 2024-09-13T14:51:14.778 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.778 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.778 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.778 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.555535+0000", 2024-09-13T14:51:14.778 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8523999999999998e-05 2024-09-13T14:51:14.778 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.778 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.778 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.778 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.555629+0000", 2024-09-13T14:51:14.778 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3700000000000001e-05 2024-09-13T14:51:14.778 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.779 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.779 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.779 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.779 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.779 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:114364 6.b 6:d85d7d2f:::benchmark_data_smithi096_167087_object114363:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-09-13T14:51:14.779 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:45:33.194910+0000", 2024-09-13T14:51:14.779 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 341.43799541800001, 2024-09-13T14:51:14.779 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3687936779999998, 2024-09-13T14:51:14.779 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.779 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.779 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.779 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.779 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.779 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.780 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 114364 2024-09-13T14:51:14.780 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.780 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.780 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.780 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.780 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.194910+0000", 2024-09-13T14:51:14.780 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.780 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.780 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.780 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.780 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.194910+0000", 2024-09-13T14:51:14.780 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.780 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.781 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.781 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.781 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.194911+0000", 2024-09-13T14:51:14.781 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.198e-06 2024-09-13T14:51:14.781 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.781 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.781 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.781 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.194932+0000", 2024-09-13T14:51:14.781 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1155e-05 2024-09-13T14:51:14.781 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.781 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.781 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.781 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.194945+0000", 2024-09-13T14:51:14.781 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3213999999999999e-05 2024-09-13T14:51:14.782 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.782 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.782 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.782 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.194952+0000", 2024-09-13T14:51:14.782 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2990000000000002e-06 2024-09-13T14:51:14.782 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.782 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.782 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.782 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.195011+0000", 2024-09-13T14:51:14.782 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9781999999999998e-05 2024-09-13T14:51:14.782 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.782 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.782 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.783 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:33.195019+0000", 2024-09-13T14:51:14.783 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1599999999999998e-06 2024-09-13T14:51:14.783 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.783 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.783 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.783 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.026076+0000", 2024-09-13T14:51:14.783 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.83105675400000001 2024-09-13T14:51:14.783 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.783 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.783 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.783 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:34.026083+0000", 2024-09-13T14:51:14.783 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.126e-06 2024-09-13T14:51:14.783 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.783 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.784 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.784 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.544216+0000", 2024-09-13T14:51:14.784 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.518132963 2024-09-13T14:51:14.784 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.784 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.784 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.784 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.544362+0000", 2024-09-13T14:51:14.784 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00014607300000000001 2024-09-13T14:51:14.784 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.784 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.784 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4,7", 2024-09-13T14:51:14.784 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.544544+0000", 2024-09-13T14:51:14.785 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000181669 2024-09-13T14:51:14.785 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.785 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.785 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.785 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.555181+0000", 2024-09-13T14:51:14.785 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.010637300000000001 2024-09-13T14:51:14.785 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.785 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.785 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.785 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.555254+0000", 2024-09-13T14:51:14.785 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2902000000000001e-05 2024-09-13T14:51:14.785 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.785 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.786 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.786 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.555666+0000", 2024-09-13T14:51:14.786 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00041145800000000002 2024-09-13T14:51:14.786 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.786 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.786 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.786 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.563544+0000", 2024-09-13T14:51:14.786 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0078778270000000004 2024-09-13T14:51:14.786 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.786 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.786 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.786 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.563599+0000", 2024-09-13T14:51:14.786 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5492000000000003e-05 2024-09-13T14:51:14.786 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.787 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.787 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.787 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:45:35.563703+0000", 2024-09-13T14:51:14.787 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000104306 2024-09-13T14:51:14.787 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.787 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.787 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.787 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.787 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.787 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:136885 6.f 6:ff68987f:::benchmark_data_smithi096_167087_object19657:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:51:14.787 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:02.100541+0000", 2024-09-13T14:51:14.787 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 312.53236361099999, 2024-09-13T14:51:14.787 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.142641856, 2024-09-13T14:51:14.787 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.788 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.788 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "reached pg", 2024-09-13T14:51:14.788 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.788 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.788 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.788 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 136885 2024-09-13T14:51:14.788 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.788 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.788 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.788 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.788 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100541+0000", 2024-09-13T14:51:14.788 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.788 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.789 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.789 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.789 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100541+0000", 2024-09-13T14:51:14.789 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.789 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.789 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.789 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.789 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100543+0000", 2024-09-13T14:51:14.789 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.161e-06 2024-09-13T14:51:14.789 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.789 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.789 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.789 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100545+0000", 2024-09-13T14:51:14.789 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.8250000000000001e-06 2024-09-13T14:51:14.790 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.790 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.790 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.790 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100559+0000", 2024-09-13T14:51:14.790 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3419e-05 2024-09-13T14:51:14.790 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.790 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.790 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.790 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100564+0000", 2024-09-13T14:51:14.790 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5600000000000001e-06 2024-09-13T14:51:14.790 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.790 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.790 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.791 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.120992+0000", 2024-09-13T14:51:14.791 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.020427595999999999 2024-09-13T14:51:14.791 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.791 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.791 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.791 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.121000+0000", 2024-09-13T14:51:14.791 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1070000000000005e-06 2024-09-13T14:51:14.791 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.791 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.791 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.791 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:03.087612+0000", 2024-09-13T14:51:14.791 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.96661190100000005 2024-09-13T14:51:14.791 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.791 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.792 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.792 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:03.087623+0000", 2024-09-13T14:51:14.792 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1260000000000001e-05 2024-09-13T14:51:14.792 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.792 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.792 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.792 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.240296+0000", 2024-09-13T14:51:14.792 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.152672205 2024-09-13T14:51:14.792 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.792 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.792 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.792 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.243183+0000", 2024-09-13T14:51:14.792 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.002887822 2024-09-13T14:51:14.793 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.793 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.793 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.793 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.793 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.793 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:136890 6.f 6:f85813ec:::benchmark_data_smithi096_167087_object19662:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:51:14.793 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:02.100575+0000", 2024-09-13T14:51:14.793 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 312.53233042300002, 2024-09-13T14:51:14.793 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1470130040000002, 2024-09-13T14:51:14.793 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.793 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.793 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "reached pg", 2024-09-13T14:51:14.793 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.793 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.794 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.794 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 136890 2024-09-13T14:51:14.794 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.794 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.794 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.794 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.794 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100575+0000", 2024-09-13T14:51:14.794 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.794 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.794 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.794 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.794 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100575+0000", 2024-09-13T14:51:14.794 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.795 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.795 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.795 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.795 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100575+0000", 2024-09-13T14:51:14.795 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5600000000000001e-07 2024-09-13T14:51:14.795 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.795 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.795 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.795 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100577+0000", 2024-09-13T14:51:14.795 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.2019999999999998e-06 2024-09-13T14:51:14.795 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.795 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.795 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.795 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100583+0000", 2024-09-13T14:51:14.796 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5829999999999997e-06 2024-09-13T14:51:14.796 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.796 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.796 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.796 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100587+0000", 2024-09-13T14:51:14.796 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3959999999999999e-06 2024-09-13T14:51:14.796 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.796 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.796 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.796 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.121022+0000", 2024-09-13T14:51:14.796 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.020434324 2024-09-13T14:51:14.796 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.796 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.797 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.797 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.121028+0000", 2024-09-13T14:51:14.797 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0449999999999997e-06 2024-09-13T14:51:14.797 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.797 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.797 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.797 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:03.087646+0000", 2024-09-13T14:51:14.797 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.96661854199999997 2024-09-13T14:51:14.797 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.797 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.797 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.797 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:03.087655+0000", 2024-09-13T14:51:14.797 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2240000000000001e-06 2024-09-13T14:51:14.797 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.798 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.798 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.798 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.247507+0000", 2024-09-13T14:51:14.798 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.15985191 2024-09-13T14:51:14.798 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.798 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.798 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.798 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.247588+0000", 2024-09-13T14:51:14.798 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0321999999999994e-05 2024-09-13T14:51:14.798 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.798 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.798 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.798 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.799 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.799 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:136896 6.f 6:fee81a4c:::benchmark_data_smithi096_167087_object19668:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:51:14.799 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:02.100593+0000", 2024-09-13T14:51:14.799 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 312.53231219000003, 2024-09-13T14:51:14.799 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1849127830000001, 2024-09-13T14:51:14.799 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.799 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.799 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.799 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.799 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.799 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.799 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 136896 2024-09-13T14:51:14.799 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.799 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.800 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.800 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.800 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100593+0000", 2024-09-13T14:51:14.800 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.800 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.800 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.800 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.800 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100593+0000", 2024-09-13T14:51:14.800 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.800 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.800 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.800 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.800 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100593+0000", 2024-09-13T14:51:14.801 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.9900000000000002e-07 2024-09-13T14:51:14.801 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.801 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.801 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.801 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100595+0000", 2024-09-13T14:51:14.801 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7710000000000001e-06 2024-09-13T14:51:14.801 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.801 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.801 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.801 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100600+0000", 2024-09-13T14:51:14.801 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7530000000000001e-06 2024-09-13T14:51:14.801 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.801 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.802 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.802 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.100604+0000", 2024-09-13T14:51:14.802 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.8619999999999996e-06 2024-09-13T14:51:14.802 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.802 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.802 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.802 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.121048+0000", 2024-09-13T14:51:14.802 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.020444331999999999 2024-09-13T14:51:14.802 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.802 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.802 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.802 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.121054+0000", 2024-09-13T14:51:14.802 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1800000000000001e-06 2024-09-13T14:51:14.802 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.803 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.803 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.803 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:03.087683+0000", 2024-09-13T14:51:14.803 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.96662914799999999 2024-09-13T14:51:14.803 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.803 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.803 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.803 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:03.087691+0000", 2024-09-13T14:51:14.803 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3320000000000003e-06 2024-09-13T14:51:14.803 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.803 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.803 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.803 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.253518+0000", 2024-09-13T14:51:14.804 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1658274159999999 2024-09-13T14:51:14.804 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.804 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.804 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.804 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.254400+0000", 2024-09-13T14:51:14.804 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00088200099999999999 2024-09-13T14:51:14.804 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.804 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.804 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,2,4", 2024-09-13T14:51:14.804 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.254888+0000", 2024-09-13T14:51:14.804 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00048815099999999998 2024-09-13T14:51:14.804 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.804 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.804 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.805 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.269657+0000", 2024-09-13T14:51:14.805 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.014768701 2024-09-13T14:51:14.805 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.805 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.805 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.805 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.282545+0000", 2024-09-13T14:51:14.805 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.012887846 2024-09-13T14:51:14.805 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.805 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.805 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.805 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.284054+0000", 2024-09-13T14:51:14.805 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0015090839999999999 2024-09-13T14:51:14.805 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.285335+0000", 2024-09-13T14:51:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001281253 2024-09-13T14:51:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.285394+0000", 2024-09-13T14:51:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8804999999999997e-05 2024-09-13T14:51:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.285506+0000", 2024-09-13T14:51:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011184900000000001 2024-09-13T14:51:14.807 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.807 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.807 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.807 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.807 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.807 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:174413 6.16 6:6d2249f6:::benchmark_data_smithi096_167087_object57185:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-09-13T14:51:14.807 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:07.111178+0000", 2024-09-13T14:51:14.807 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 247.52172700700001, 2024-09-13T14:51:14.807 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1115644760000001, 2024-09-13T14:51:14.807 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:51:14.807 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:51:14.807 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:14.807 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:51:14.807 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:51:14.808 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:14.808 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 174413 2024-09-13T14:51:14.808 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.808 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:51:14.808 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.808 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:51:14.808 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:07.111178+0000", 2024-09-13T14:51:14.808 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.808 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.808 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.808 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:51:14.809 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:14.809 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:07.111178+0000", 2024-09-13T14:51:14.810 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:51:14.810 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.810 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.810 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:51:14.810 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:07.111178+0000", 2024-09-13T14:51:14.810 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.2399999999999999e-07 2024-09-13T14:51:14.810 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.810 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.810 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:51:14.810 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:07.111180+0000", 2024-09-13T14:51:14.810 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.73e-06 2024-09-13T14:51:14.810 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.810 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.811 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:51:14.811 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:07.111186+0000", 2024-09-13T14:51:14.811 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7989999999999999e-06 2024-09-13T14:51:14.811 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.811 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.811 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:51:14.811 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:07.111191+0000", 2024-09-13T14:51:14.811 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7029999999999997e-06 2024-09-13T14:51:14.811 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.811 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.811 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.811 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:07.111384+0000", 2024-09-13T14:51:14.811 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00019355800000000001 2024-09-13T14:51:14.812 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.812 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.812 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:51:14.812 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:07.111391+0000", 2024-09-13T14:51:14.812 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6370000000000002e-06 2024-09-13T14:51:14.812 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.812 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.812 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:51:14.812 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:08.201099+0000", 2024-09-13T14:51:14.812 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.089708439 2024-09-13T14:51:14.812 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.812 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.813 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:51:14.813 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:08.201224+0000", 2024-09-13T14:51:14.813 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012487300000000001 2024-09-13T14:51:14.813 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.813 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.813 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,4,5,7", 2024-09-13T14:51:14.813 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:08.201383+0000", 2024-09-13T14:51:14.813 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00015854000000000001 2024-09-13T14:51:14.813 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.813 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.813 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:51:14.813 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:08.202690+0000", 2024-09-13T14:51:14.813 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001307709 2024-09-13T14:51:14.813 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.814 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.814 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.814 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:08.203354+0000", 2024-09-13T14:51:14.814 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00066349099999999999 2024-09-13T14:51:14.814 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.814 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.814 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.814 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:08.207501+0000", 2024-09-13T14:51:14.814 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0041471360000000001 2024-09-13T14:51:14.814 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.814 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.814 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.814 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:08.217275+0000", 2024-09-13T14:51:14.815 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.009774474 2024-09-13T14:51:14.815 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.815 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.815 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:14.815 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:08.222460+0000", 2024-09-13T14:51:14.815 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0051848040000000003 2024-09-13T14:51:14.815 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.815 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.815 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:51:14.815 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:08.222526+0000", 2024-09-13T14:51:14.815 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5883000000000006e-05 2024-09-13T14:51:14.815 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:51:14.815 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:51:14.815 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:51:14.816 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:08.222743+0000", 2024-09-13T14:51:14.816 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000216376 2024-09-13T14:51:14.816 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.816 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.816 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.816 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:51:14.816 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:51:14.816 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:51:14.854 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:14.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:14.955 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:14.985 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:14.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:15.086 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:15.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.112 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:15.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:15.213 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:15.254 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:15.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:15.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:15 smithi096 ceph-mon[98901]: pgmap v726: 99 pgs: 1 active+remapped+backfill_wait, 3 remapped+peering, 1 peering, 94 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 4392/212298 objects misplaced (2.069%) 2024-09-13T14:51:15.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:15 smithi096 ceph-mon[102928]: pgmap v726: 99 pgs: 1 active+remapped+backfill_wait, 3 remapped+peering, 1 peering, 94 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 4392/212298 objects misplaced (2.069%) 2024-09-13T14:51:15.356 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:15.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.392 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:15.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:15.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:15 smithi204 ceph-mon[87071]: pgmap v726: 99 pgs: 1 active+remapped+backfill_wait, 3 remapped+peering, 1 peering, 94 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 4392/212298 objects misplaced (2.069%) 2024-09-13T14:51:15.493 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:15.519 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:15.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:15.621 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:15.663 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:15.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:15.765 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:15.806 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:15.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:15.908 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:15.934 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:15.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:16.036 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:16.073 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:16.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:16.175 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:16.200 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:16.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:16.302 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:16.327 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:16.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:16.429 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:16.451 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:16.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:16.552 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:16.583 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:16.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:16.685 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:16.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.727 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:16.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:16.828 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:16.860 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:16.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:16.961 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:16.982 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:16.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:17.084 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:17.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.120 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:17.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:17.221 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:17.246 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:17.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:17.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:17 smithi096 ceph-mon[98901]: pgmap v727: 99 pgs: 1 active+remapped+backfill_wait, 3 remapped+peering, 95 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 4392/211521 objects misplaced (2.076%); 0 B/s, 1 objects/s recovering 2024-09-13T14:51:17.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:17 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:17.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:17 smithi096 ceph-mon[102928]: pgmap v727: 99 pgs: 1 active+remapped+backfill_wait, 3 remapped+peering, 95 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 4392/211521 objects misplaced (2.076%); 0 B/s, 1 objects/s recovering 2024-09-13T14:51:17.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:17 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:17.347 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:17.374 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:17.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:17.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:17 smithi204 ceph-mon[87071]: pgmap v727: 99 pgs: 1 active+remapped+backfill_wait, 3 remapped+peering, 95 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 4392/211521 objects misplaced (2.076%); 0 B/s, 1 objects/s recovering 2024-09-13T14:51:17.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:17 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:17.476 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:17.516 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:17.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:17.618 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:17.643 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:17.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:17.745 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:17.772 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:17.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:17.874 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:17.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.921 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:17.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:18.022 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:18.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.051 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:18.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:18.152 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:18.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.195 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:18.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:18.296 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:18.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:51:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:51:17] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.51.0" 2024-09-13T14:51:18.324 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:18.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:18.426 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:18.452 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:18.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:18.554 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:18.579 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:18.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:18.680 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:18.719 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:18.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:18.821 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:18.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.864 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:18.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:18.965 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:19.015 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:19.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:19.117 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:19.163 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:19.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:19.184 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:19 smithi096 ceph-mon[98901]: pgmap v728: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 16484/207003 objects misplaced (7.963%); 4.5 MiB/s, 74 objects/s recovering 2024-09-13T14:51:19.184 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:19 smithi096 ceph-mon[102928]: pgmap v728: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 16484/207003 objects misplaced (7.963%); 4.5 MiB/s, 74 objects/s recovering 2024-09-13T14:51:19.265 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:19.296 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:19.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:19.398 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:19.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.445 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:19.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:19.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:19 smithi204 ceph-mon[87071]: pgmap v728: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 16484/207003 objects misplaced (7.963%); 4.5 MiB/s, 74 objects/s recovering 2024-09-13T14:51:19.546 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:19.571 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:19.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:19.673 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:19.700 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:19.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:19.802 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:19.829 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:19.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:19.931 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:19.958 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:19.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:20.060 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:20.084 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:20.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:20.186 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:20.225 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:20.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:20.327 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:20.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.380 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:20.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:51:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:51:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:20.482 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:20.515 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:20.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:20.617 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:20.683 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:20.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:20.785 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:20.831 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:20.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:20.934 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:20.978 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:20.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:21.064 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:51:21.080 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:21.099 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:51:21.112 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:21.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:21.214 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:21.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.264 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:21.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:21.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:21 smithi096 ceph-mon[98901]: pgmap v729: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 16239/202170 objects misplaced (8.032%); 4.0 MiB/s, 65 objects/s recovering 2024-09-13T14:51:21.357 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:21 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3475365446' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:51:21.357 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:21 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:51:21.357 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:21.358 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:21 smithi096 ceph-mon[102928]: pgmap v729: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 16239/202170 objects misplaced (8.032%); 4.0 MiB/s, 65 objects/s recovering 2024-09-13T14:51:21.358 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:21 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3475365446' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:51:21.358 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:21 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:51:21.358 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:21.366 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:21.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.426 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:21.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:21.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:21 smithi204 ceph-mon[87071]: pgmap v729: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 16239/202170 objects misplaced (8.032%); 4.0 MiB/s, 65 objects/s recovering 2024-09-13T14:51:21.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:21 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3475365446' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:51:21.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:21 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:51:21.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:21.509 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:21.527 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:21.571 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:21.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:21.673 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:21.712 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:21.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:21.814 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:21.842 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:21.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:21.944 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:21.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.975 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:21.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:22.076 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:22.108 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:22.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:22.210 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:22.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:22 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:22.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:22 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:51:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:22 smithi204 ceph-mon[87071]: osdmap e247: 8 total, 8 up, 8 in 2024-09-13T14:51:22.270 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:22 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:22.270 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:22 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:51:22.270 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:22 smithi096 ceph-mon[98901]: osdmap e247: 8 total, 8 up, 8 in 2024-09-13T14:51:22.270 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:22 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:22.270 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:22 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:51:22.270 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:22 smithi096 ceph-mon[102928]: osdmap e247: 8 total, 8 up, 8 in 2024-09-13T14:51:22.271 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:22.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:22.373 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:22.416 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:22.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:22.518 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:22.548 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:22.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:22.650 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:22.671 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:22.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:22.772 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:22.802 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:22.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:22.904 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:22.944 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:22.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:23.048 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:23.087 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:23.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:23.190 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:23.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.221 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:23.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:23.322 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:23.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.360 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:23.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:23.401 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:23 smithi096 ceph-mon[98901]: pgmap v731: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 14958/197265 objects misplaced (7.583%); 7.6 MiB/s, 123 objects/s recovering 2024-09-13T14:51:23.401 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:23 smithi096 ceph-mon[102928]: pgmap v731: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 14958/197265 objects misplaced (7.583%); 7.6 MiB/s, 123 objects/s recovering 2024-09-13T14:51:23.462 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:23.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:23 smithi204 ceph-mon[87071]: pgmap v731: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 14958/197265 objects misplaced (7.583%); 7.6 MiB/s, 123 objects/s recovering 2024-09-13T14:51:23.486 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:23.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:23.588 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:23.641 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:23.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:23.743 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:23.768 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:23.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:23.870 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:23.904 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:51:23.904 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:51:23.904 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:51:23.904 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:51:23.904 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:51:23.904 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:51:23.905 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:51:23.905 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:51:23.907 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:23.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:23.914 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:51:23.915 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:51:23.955 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:51:23.955 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:51:23.993 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:51:23.993 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:51:24.010 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:24.031 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:51:24.031 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:51:24.035 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:24.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:24.138 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:24.162 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:24.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:24.197 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:24 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2708655405' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:24.265 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:24.278 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:24 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2708655405' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:24.279 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:24 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2708655405' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:24.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.300 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:24.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:24.402 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:24.435 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:24.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:24.466 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:51:24.466 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:51:24.537 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:24.566 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:24.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:24.668 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:24.710 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:24.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:24.812 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:24.840 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:24.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:24.941 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:24.977 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:24.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:25.079 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:25.103 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:25.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:25.112 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T14:51:25.138 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:51:25.183 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:51:25.205 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:25.252 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:25.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:25 smithi096 ceph-mon[98901]: pgmap v732: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 14719/190638 objects misplaced (7.721%); 7.5 MiB/s, 121 objects/s recovering 2024-09-13T14:51:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:25 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1900745885' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:51:25.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:25 smithi096 ceph-mon[102928]: pgmap v732: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 14719/190638 objects misplaced (7.721%); 7.5 MiB/s, 121 objects/s recovering 2024-09-13T14:51:25.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:25 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1900745885' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:51:25.354 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:25.398 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:25.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:25 smithi204 ceph-mon[87071]: pgmap v732: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 14719/190638 objects misplaced (7.721%); 7.5 MiB/s, 121 objects/s recovering 2024-09-13T14:51:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:25 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1900745885' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:51:25.500 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:25.518 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:25.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.538 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:25.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:25.639 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:25.681 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:25.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:25.783 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:25.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.815 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:25.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:25.916 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:25.946 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:25.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:26.047 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:26.076 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:26.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:26.178 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:26.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.206 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:26.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:26.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:26 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1900745885' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:51:26.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:26 smithi204 ceph-mon[87071]: osdmap e248: 8 total, 8 up, 8 in 2024-09-13T14:51:26.307 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:26.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:26 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1900745885' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:51:26.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:26 smithi096 ceph-mon[98901]: osdmap e248: 8 total, 8 up, 8 in 2024-09-13T14:51:26.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:26 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1900745885' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:51:26.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:26 smithi096 ceph-mon[102928]: osdmap e248: 8 total, 8 up, 8 in 2024-09-13T14:51:26.338 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:26.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:26.441 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:26.476 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:26.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:26.519 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T14:51:26.579 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:26.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.605 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:26.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:26.627 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:26.705 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:26.734 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:26.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:26.791 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:51:26.836 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:26.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.918 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:26.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:27.019 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:27.048 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:27.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:27.150 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:27.176 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:27.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:27.278 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:27.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.327 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:27.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:27.425 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:27 smithi096 ceph-mon[98901]: pgmap v734: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 14719/189318 objects misplaced (7.775%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T14:51:27.427 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:27 smithi096 ceph-mon[102928]: pgmap v734: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 14719/189318 objects misplaced (7.775%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T14:51:27.428 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:27.453 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:27.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:27.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:27 smithi204 ceph-mon[87071]: pgmap v734: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 14719/189318 objects misplaced (7.775%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T14:51:27.556 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:27.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.602 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:27.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:27.679 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:51:27.679 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:51:27.703 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:27.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.751 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:27.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:27.852 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:27.880 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:27.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:27.982 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:28.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.008 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:28.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:28.023 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:51:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:51:27] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.51.0" 2024-09-13T14:51:28.109 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:28.150 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:28.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:28.252 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:28.278 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:28.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:28.303 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:28.380 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:28.406 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:28.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:28.508 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:28.558 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:28.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:28.660 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:28.700 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:28.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:28.802 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:28.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.841 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:28.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:28.852 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:51:28.942 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:28.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.999 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:28.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:29.100 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:29.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.135 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:29.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:29.187 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:29.236 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:29.286 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:29.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:29.388 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:29.408 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:29 smithi096 ceph-mon[98901]: pgmap v735: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 13571/184353 objects misplaced (7.361%); 9.1 MiB/s, 146 objects/s recovering 2024-09-13T14:51:29.408 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:29 smithi096 ceph-mon[102928]: pgmap v735: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 13571/184353 objects misplaced (7.361%); 9.1 MiB/s, 146 objects/s recovering 2024-09-13T14:51:29.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.416 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:29.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:29.448 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:29 smithi204 ceph-mon[87071]: pgmap v735: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 13571/184353 objects misplaced (7.361%); 9.1 MiB/s, 146 objects/s recovering 2024-09-13T14:51:29.517 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:29.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.589 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:29.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:29.690 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:29.718 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:29.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:29.820 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:29.848 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:29.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:29.951 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:29.980 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:29.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:30.082 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:30.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.112 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:30.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:30.213 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:30.240 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:30.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:30.342 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:30.378 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:30.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:51:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:51:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:30.479 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:30.509 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:30.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:30.611 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:30.661 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:30.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:30.763 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:30.796 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:30.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:30.898 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:30.945 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:30.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:30.994 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:51:30.994 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:51:31.048 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:31.088 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:31.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:31.190 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:31.229 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:31.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:31.331 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:31.354 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:31.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:31.456 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:31.460 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:31 smithi096 ceph-mon[98901]: pgmap v736: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 13349/179457 objects misplaced (7.439%); 8.3 MiB/s, 132 objects/s recovering 2024-09-13T14:51:31.460 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:31 smithi096 ceph-mon[102928]: pgmap v736: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 13349/179457 objects misplaced (7.439%); 8.3 MiB/s, 132 objects/s recovering 2024-09-13T14:51:31.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:31 smithi204 ceph-mon[87071]: pgmap v736: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 13349/179457 objects misplaced (7.439%); 8.3 MiB/s, 132 objects/s recovering 2024-09-13T14:51:31.482 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:31.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:31.584 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:31.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.610 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:31.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:31.649 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:51:31.711 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:31.727 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:31.758 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:31.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:31.860 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:31.916 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:31.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:32.001 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:32.018 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:32.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.046 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:32.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:32.147 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:32.148 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:51:32.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.195 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:32.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:32.296 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:32.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:32 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2295964141' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:32 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2295964141' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:32.327 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:32.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:32.429 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:32.445 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:32 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2295964141' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:32.455 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:32.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:32.557 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:32.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.597 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:32.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:32.698 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:32.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.732 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:32.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:32.833 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:32.858 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:32.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:32.959 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:32.983 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:32.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:33.085 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:33.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.135 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:33.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:33.236 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:33.291 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:33.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:33.370 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:33 smithi096 ceph-mon[98901]: pgmap v737: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 12281/174432 objects misplaced (7.041%); 6.5 MiB/s, 104 objects/s recovering 2024-09-13T14:51:33.370 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:33 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:51:33.371 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:33 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:51:33.371 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:33 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2295964141' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:33.371 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:33 smithi096 ceph-mon[98901]: osdmap e249: 8 total, 8 up, 8 in 2024-09-13T14:51:33.371 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:33.371 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:33.371 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:33 smithi096 ceph-mon[102928]: pgmap v737: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 12281/174432 objects misplaced (7.041%); 6.5 MiB/s, 104 objects/s recovering 2024-09-13T14:51:33.371 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:33 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:51:33.371 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:33 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:51:33.371 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:33 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2295964141' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:33.371 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:33 smithi096 ceph-mon[102928]: osdmap e249: 8 total, 8 up, 8 in 2024-09-13T14:51:33.372 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:33.372 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:33.393 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:33.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.421 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:33.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:33 smithi204 ceph-mon[87071]: pgmap v737: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 12281/174432 objects misplaced (7.041%); 6.5 MiB/s, 104 objects/s recovering 2024-09-13T14:51:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:33 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:51:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:33 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:51:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:33 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2295964141' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:33 smithi204 ceph-mon[87071]: osdmap e249: 8 total, 8 up, 8 in 2024-09-13T14:51:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:33.522 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:33.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.548 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:33.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:33.649 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:33.711 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:33.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:33.813 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:33.838 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:33.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:33.940 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:33.967 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:33.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:34.069 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:34.094 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:34.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:34.196 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:34.226 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:34.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:34.244 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:34 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:51:34.244 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:34 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:51:34.244 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:34 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:34.245 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:34 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:51:34.245 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:34 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:51:34.245 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:34 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:34.328 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:34.367 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:34.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:34.426 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:34 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:51:34.426 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:34 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:51:34.426 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:34 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:34.469 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:34.507 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:34.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:34.609 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:34.640 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:34.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:34.705 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:51:34.705 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:51:34.743 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:34.799 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:34.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:34.901 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:34.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.944 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:34.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:35.045 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:35.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.072 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:35.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:35.173 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:35.196 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:35.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:35.244 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:35 smithi096 ceph-mon[98901]: pgmap v739: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 12043/168318 objects misplaced (7.155%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:51:35.244 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:35 smithi096 ceph-mon[102928]: pgmap v739: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 12043/168318 objects misplaced (7.155%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:51:35.299 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:35.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.326 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:35.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:35.427 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:35.452 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:35.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:35.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:35 smithi204 ceph-mon[87071]: pgmap v739: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 12043/168318 objects misplaced (7.155%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:51:35.553 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:35.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.578 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:35.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:35.679 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:35.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.705 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:35.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:35.722 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:51:35.772 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:51:35.772 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:51:35.806 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:35.819 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:51:35.819 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:51:35.834 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:35.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:35.871 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:51:35.871 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:51:35.908 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:51:35.908 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:51:35.936 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:35.948 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:51:35.948 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:51:35.988 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:51:35.989 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:51:35.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.991 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:35.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:36.022 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:51:36.023 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:51:36.051 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:51:36.052 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:51:36.093 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:36.094 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:36.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.135 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:36.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:36.236 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:36.246 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:51:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.264 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:36.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:36.365 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:36.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.398 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:36.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:36 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3229600960' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:36 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1358460361' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:51:36.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:36 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:51:36.499 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:36.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.543 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:36.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:36.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:36 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3229600960' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:36.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:36.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:36.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:36 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1358460361' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:51:36.575 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:36 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:51:36.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:36 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3229600960' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:36.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:36.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:36.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:36 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1358460361' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:51:36.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:36 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:51:36.644 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:36.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.671 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:36.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:36.772 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:36.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.800 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:36.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:36.847 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T14:51:36.901 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:36.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.929 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:36.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:37.031 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:37.064 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:37.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:37.166 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:37.191 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:37.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:37.293 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:37.333 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:37.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:37.435 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:37.442 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:37 smithi204 ceph-mon[87071]: pgmap v740: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 12043/166917 objects misplaced (7.215%); 6.5 MiB/s, 104 objects/s recovering 2024-09-13T14:51:37.442 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:37 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:51:37.442 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:37 smithi204 ceph-mon[87071]: osdmap e250: 8 total, 8 up, 8 in 2024-09-13T14:51:37.465 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:37.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:37.567 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:37.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:37 smithi096 ceph-mon[98901]: pgmap v740: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 12043/166917 objects misplaced (7.215%); 6.5 MiB/s, 104 objects/s recovering 2024-09-13T14:51:37.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:37 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:51:37.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:37 smithi096 ceph-mon[98901]: osdmap e250: 8 total, 8 up, 8 in 2024-09-13T14:51:37.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:37 smithi096 ceph-mon[102928]: pgmap v740: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 12043/166917 objects misplaced (7.215%); 6.5 MiB/s, 104 objects/s recovering 2024-09-13T14:51:37.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:37 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:51:37.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:37 smithi096 ceph-mon[102928]: osdmap e250: 8 total, 8 up, 8 in 2024-09-13T14:51:37.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.596 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:37.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:37.697 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:37.738 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:37.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:37.841 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:37.876 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:37.876 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:51:37.877 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:51:37.877 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T14:51:37.877 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T14:51:37.877 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:51:37.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:37.978 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:38.005 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:38.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:38.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:51:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:51:37] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.51.0" 2024-09-13T14:51:38.107 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:38.133 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:38.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:38.235 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:38.276 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:38.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:38.378 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:38.405 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:38.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:38.507 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:38.543 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:38.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:38.639 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:38.648 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:38.655 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T14:51:38.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.708 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:38.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:38.810 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:38.858 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:38.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:38.951 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:51:38.960 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:39.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.025 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:39.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:39.126 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:39.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.156 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:39.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:39.257 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:39.293 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:39.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.294 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:39.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:39.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:39 smithi096 ceph-mon[98901]: pgmap v742: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 11032/163011 objects misplaced (6.768%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:51:39.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:39 smithi096 ceph-mon[102928]: pgmap v742: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 11032/163011 objects misplaced (6.768%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:51:39.395 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:39.424 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:39.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:39.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:39 smithi204 ceph-mon[87071]: pgmap v742: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 11032/163011 objects misplaced (6.768%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:51:39.525 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:39.550 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:39.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:39.652 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:39.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.682 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:39.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:39.783 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:39.802 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:51:39.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.832 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:39.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:39.933 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:39.958 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:39.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:40.060 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:40.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.084 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:40.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:40.123 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:40.184 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:40.214 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 14:51:39 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=cleanup t=2024-09-13T14:51:39.85841896Z level=info msg="Completed cleanup jobs" duration=11.406826ms 2024-09-13T14:51:40.244 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:40.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:40.347 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:40.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.402 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:40.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:51:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:51:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:40.503 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:40.558 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:40.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:40.660 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:40.691 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:40.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:40.793 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:40.822 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:40.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:40.924 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:40.951 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:40.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:41.053 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:41.095 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:41.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:41.197 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:41.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.236 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:41.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:41.294 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:41 smithi204 ceph-mon[87071]: pgmap v743: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 10803/157359 objects misplaced (6.865%); 3.8 MiB/s, 61 objects/s recovering 2024-09-13T14:51:41.337 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:41.373 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:41.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:41.422 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:51:41.422 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T14:51:41.423 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T14:51:41.423 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:51:41.423 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.423 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:136898 6.a 6:52dddb7a:::benchmark_data_smithi096_167087_object19670:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-09-13T14:51:41.423 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:02.071966+0000", 2024-09-13T14:51:41.423 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 339.34571275100001, 2024-09-13T14:51:41.423 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1809877279999998, 2024-09-13T14:51:41.423 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.423 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:51:41.423 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.423 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.423 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.424 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 136898 2024-09-13T14:51:41.424 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.424 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.424 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.424 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.424 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.071966+0000", 2024-09-13T14:51:41.424 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.424 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.424 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.424 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.424 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.071966+0000", 2024-09-13T14:51:41.424 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.424 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.425 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.425 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.425 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.071964+0000", 2024-09-13T14:51:41.425 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:51:41.425 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.425 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.425 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.425 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.071971+0000", 2024-09-13T14:51:41.425 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8979999999999999e-06 2024-09-13T14:51:41.425 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.425 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.425 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.425 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.071983+0000", 2024-09-13T14:51:41.426 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2252999999999999e-05 2024-09-13T14:51:41.426 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.426 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.426 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.426 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.071988+0000", 2024-09-13T14:51:41.426 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4219999999999999e-06 2024-09-13T14:51:41.426 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.426 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.426 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.426 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.072036+0000", 2024-09-13T14:51:41.426 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7686999999999999e-05 2024-09-13T14:51:41.426 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.426 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.426 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.427 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.072148+0000", 2024-09-13T14:51:41.427 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000112098 2024-09-13T14:51:41.427 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.427 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.427 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:51:41.427 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.072269+0000", 2024-09-13T14:51:41.427 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000121001 2024-09-13T14:51:41.427 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.427 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.427 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.427 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.075388+0000", 2024-09-13T14:51:41.427 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003119204 2024-09-13T14:51:41.428 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.428 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.428 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.428 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.084975+0000", 2024-09-13T14:51:41.428 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.009586598 2024-09-13T14:51:41.428 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.428 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.428 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.428 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.084985+0000", 2024-09-13T14:51:41.428 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0001e-05 2024-09-13T14:51:41.428 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.428 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.428 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.428 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.080975+0000", 2024-09-13T14:51:41.429 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99599027500000004 2024-09-13T14:51:41.429 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.429 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.429 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.429 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.080985+0000", 2024-09-13T14:51:41.429 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0304e-05 2024-09-13T14:51:41.429 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.429 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.429 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.429 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.252742+0000", 2024-09-13T14:51:41.429 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1717568709999999 2024-09-13T14:51:41.429 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.429 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.430 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.430 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.252953+0000", 2024-09-13T14:51:41.430 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00021098499999999999 2024-09-13T14:51:41.430 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.430 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.430 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.430 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.430 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.430 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:136914 6.a 6:55b09444:::benchmark_data_smithi096_167087_object19686:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:51:41.430 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:02.127662+0000", 2024-09-13T14:51:41.430 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 339.29001608999999, 2024-09-13T14:51:41.430 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.160209542, 2024-09-13T14:51:41.430 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.430 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.431 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.431 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.431 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.431 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 136914 2024-09-13T14:51:41.431 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.431 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.431 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.431 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.431 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127662+0000", 2024-09-13T14:51:41.431 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.431 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.431 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.431 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.431 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127662+0000", 2024-09-13T14:51:41.432 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.432 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.432 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.432 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.432 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127661+0000", 2024-09-13T14:51:41.432 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:41.432 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.433 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.433 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.433 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127666+0000", 2024-09-13T14:51:41.433 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5140000000000001e-06 2024-09-13T14:51:41.433 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.433 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.433 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.433 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127675+0000", 2024-09-13T14:51:41.433 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.2429999999999994e-06 2024-09-13T14:51:41.433 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.433 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.433 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.433 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127681+0000", 2024-09-13T14:51:41.434 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6999999999999996e-06 2024-09-13T14:51:41.434 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.434 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.434 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.434 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127925+0000", 2024-09-13T14:51:41.434 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000243482 2024-09-13T14:51:41.434 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.434 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.434 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.434 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127940+0000", 2024-09-13T14:51:41.434 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5267000000000001e-05 2024-09-13T14:51:41.434 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.434 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.434 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.435 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.081012+0000", 2024-09-13T14:51:41.435 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95307223399999996 2024-09-13T14:51:41.435 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.435 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.435 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.435 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.081019+0000", 2024-09-13T14:51:41.435 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8750000000000002e-06 2024-09-13T14:51:41.435 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.435 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.435 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.435 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.252980+0000", 2024-09-13T14:51:41.435 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1719605639999999 2024-09-13T14:51:41.435 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.435 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.436 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.436 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.253078+0000", 2024-09-13T14:51:41.436 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.8524999999999996e-05 2024-09-13T14:51:41.436 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.436 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.436 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,1,2", 2024-09-13T14:51:41.436 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.253670+0000", 2024-09-13T14:51:41.436 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00059221000000000002 2024-09-13T14:51:41.436 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.436 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.436 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.436 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.265595+0000", 2024-09-13T14:51:41.436 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011924817000000001 2024-09-13T14:51:41.437 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.437 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.437 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.437 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.281055+0000", 2024-09-13T14:51:41.437 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.015459878 2024-09-13T14:51:41.437 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.437 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.437 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.437 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.287402+0000", 2024-09-13T14:51:41.437 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0063468079999999998 2024-09-13T14:51:41.437 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.437 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.438 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.438 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.287687+0000", 2024-09-13T14:51:41.438 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00028478299999999999 2024-09-13T14:51:41.438 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.438 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.438 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.438 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.287766+0000", 2024-09-13T14:51:41.438 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9006999999999998e-05 2024-09-13T14:51:41.438 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.439 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.439 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.439 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.287872+0000", 2024-09-13T14:51:41.439 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000106199 2024-09-13T14:51:41.439 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.439 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.439 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.439 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.439 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.439 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:136924 6.a 6:5a66c8f3:::benchmark_data_smithi096_167087_object19696:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:51:41.439 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:02.144715+0000", 2024-09-13T14:51:41.439 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 339.27296322799998, 2024-09-13T14:51:41.440 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1435050819999999, 2024-09-13T14:51:41.440 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.440 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.440 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.440 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.440 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.440 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 136924 2024-09-13T14:51:41.440 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.440 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.440 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.440 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.440 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144715+0000", 2024-09-13T14:51:41.440 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.441 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.441 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.441 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.441 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144715+0000", 2024-09-13T14:51:41.441 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.441 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.441 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.441 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.441 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144714+0000", 2024-09-13T14:51:41.441 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:41.441 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.441 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.441 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.441 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144720+0000", 2024-09-13T14:51:41.442 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2960000000000004e-06 2024-09-13T14:51:41.442 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.442 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.442 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.442 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144734+0000", 2024-09-13T14:51:41.442 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4085e-05 2024-09-13T14:51:41.442 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.442 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.442 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.442 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144741+0000", 2024-09-13T14:51:41.442 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3749999999999999e-06 2024-09-13T14:51:41.442 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.442 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.443 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.443 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144801+0000", 2024-09-13T14:51:41.443 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0922999999999999e-05 2024-09-13T14:51:41.443 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.443 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.443 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.443 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144812+0000", 2024-09-13T14:51:41.443 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1028e-05 2024-09-13T14:51:41.443 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.443 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.443 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.443 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.081057+0000", 2024-09-13T14:51:41.443 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93624500700000002 2024-09-13T14:51:41.443 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.444 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.444 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.444 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.081064+0000", 2024-09-13T14:51:41.444 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0049999999999998e-06 2024-09-13T14:51:41.444 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.444 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.444 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.444 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.262409+0000", 2024-09-13T14:51:41.444 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1813444209999999 2024-09-13T14:51:41.444 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.444 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.444 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.444 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.262527+0000", 2024-09-13T14:51:41.444 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011780199999999999 2024-09-13T14:51:41.445 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.445 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.445 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,1,2", 2024-09-13T14:51:41.445 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.262666+0000", 2024-09-13T14:51:41.445 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013970200000000001 2024-09-13T14:51:41.445 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.445 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.445 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.445 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.265606+0000", 2024-09-13T14:51:41.445 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002939119 2024-09-13T14:51:41.445 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.445 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.445 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.445 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.281078+0000", 2024-09-13T14:51:41.446 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.015472365 2024-09-13T14:51:41.446 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.446 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.446 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.446 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.287429+0000", 2024-09-13T14:51:41.446 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0063514660000000001 2024-09-13T14:51:41.446 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.446 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.446 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.446 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.287914+0000", 2024-09-13T14:51:41.446 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00048462899999999998 2024-09-13T14:51:41.446 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.446 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.446 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.447 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.287969+0000", 2024-09-13T14:51:41.447 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4987000000000001e-05 2024-09-13T14:51:41.447 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.447 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.447 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.447 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.288220+0000", 2024-09-13T14:51:41.447 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00025116899999999998 2024-09-13T14:51:41.447 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.447 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.447 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.447 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.447 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.447 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156602 6.7 6:e165c28c:::benchmark_data_smithi096_167087_object39374:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:51:41.447 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.020029+0000", 2024-09-13T14:51:41.448 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 309.39764892199997, 2024-09-13T14:51:41.448 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0175504319999999, 2024-09-13T14:51:41.448 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.448 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:51:41.448 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.448 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.448 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.448 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156602 2024-09-13T14:51:41.448 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.448 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.448 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.448 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.448 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020029+0000", 2024-09-13T14:51:41.448 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.449 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.449 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.449 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.449 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020029+0000", 2024-09-13T14:51:41.449 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.449 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.449 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.449 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.449 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020026+0000", 2024-09-13T14:51:41.449 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999962 2024-09-13T14:51:41.449 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.449 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.449 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.449 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020046+0000", 2024-09-13T14:51:41.450 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0434000000000001e-05 2024-09-13T14:51:41.450 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.450 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.450 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.450 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020088+0000", 2024-09-13T14:51:41.450 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1777999999999999e-05 2024-09-13T14:51:41.450 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.450 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.450 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.450 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020102+0000", 2024-09-13T14:51:41.450 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3747999999999999e-05 2024-09-13T14:51:41.450 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.450 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.450 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.451 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020232+0000", 2024-09-13T14:51:41.451 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013036999999999999 2024-09-13T14:51:41.451 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.451 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.451 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op must wait for map", 2024-09-13T14:51:41.451 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020240+0000", 2024-09-13T14:51:41.451 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6389999999999992e-06 2024-09-13T14:51:41.451 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.451 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.451 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.451 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.184666+0000", 2024-09-13T14:51:41.451 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16442584399999999 2024-09-13T14:51:41.451 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.451 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.453 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.453 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.184672+0000", 2024-09-13T14:51:41.453 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9229999999999999e-06 2024-09-13T14:51:41.453 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.453 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.453 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.453 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.037500+0000", 2024-09-13T14:51:41.453 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.85282839700000002 2024-09-13T14:51:41.453 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.453 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.453 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.453 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.037580+0000", 2024-09-13T14:51:41.454 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9878999999999999e-05 2024-09-13T14:51:41.454 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.454 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.454 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.454 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.454 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.454 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156607 6.17 6:e84859bf:::benchmark_data_smithi096_167087_object39379:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:51:41.454 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.020123+0000", 2024-09-13T14:51:41.454 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 309.39755510100002, 2024-09-13T14:51:41.454 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0606309060000001, 2024-09-13T14:51:41.454 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.454 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:51:41.454 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.455 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.455 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.455 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156607 2024-09-13T14:51:41.455 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.455 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.455 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.455 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.455 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020123+0000", 2024-09-13T14:51:41.455 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.455 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.455 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.455 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.455 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020123+0000", 2024-09-13T14:51:41.455 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.456 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.456 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.456 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.456 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020123+0000", 2024-09-13T14:51:41.456 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:51:41.456 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.456 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.456 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.456 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020126+0000", 2024-09-13T14:51:41.456 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.309e-06 2024-09-13T14:51:41.456 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.456 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.456 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.457 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020135+0000", 2024-09-13T14:51:41.457 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.9039999999999998e-06 2024-09-13T14:51:41.457 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.457 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.457 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.457 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020139+0000", 2024-09-13T14:51:41.457 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7299999999999999e-06 2024-09-13T14:51:41.457 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.457 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.457 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.457 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.184549+0000", 2024-09-13T14:51:41.457 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16441050099999999 2024-09-13T14:51:41.457 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.457 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.458 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.458 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.184557+0000", 2024-09-13T14:51:41.458 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.4880000000000001e-06 2024-09-13T14:51:41.458 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.458 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.458 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.458 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.080662+0000", 2024-09-13T14:51:41.458 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.89610567900000004 2024-09-13T14:51:41.458 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.458 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.458 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.458 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.080754+0000", 2024-09-13T14:51:41.458 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1776999999999999e-05 2024-09-13T14:51:41.458 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.459 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.459 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.459 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.459 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.459 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156608 6.17 6:e83c8e69:::benchmark_data_smithi096_167087_object39380:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:51:41.459 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.020143+0000", 2024-09-13T14:51:41.459 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 309.39753544299998, 2024-09-13T14:51:41.459 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0607072280000001, 2024-09-13T14:51:41.459 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.459 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:51:41.459 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.459 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.459 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.460 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156608 2024-09-13T14:51:41.460 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.460 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.460 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.460 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.460 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020143+0000", 2024-09-13T14:51:41.460 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.460 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.460 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.460 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.460 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020143+0000", 2024-09-13T14:51:41.460 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.460 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.460 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.461 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.461 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020143+0000", 2024-09-13T14:51:41.461 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:51:41.461 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.461 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.461 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.461 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020145+0000", 2024-09-13T14:51:41.461 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4490000000000002e-06 2024-09-13T14:51:41.461 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.461 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.461 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.461 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020153+0000", 2024-09-13T14:51:41.461 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.571e-06 2024-09-13T14:51:41.461 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.462 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.462 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.462 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.020156+0000", 2024-09-13T14:51:41.462 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7790000000000002e-06 2024-09-13T14:51:41.462 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.462 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.462 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.462 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.184573+0000", 2024-09-13T14:51:41.462 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.164417129 2024-09-13T14:51:41.462 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.462 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.462 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.462 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.184578+0000", 2024-09-13T14:51:41.463 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2699999999999998e-06 2024-09-13T14:51:41.463 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.463 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.463 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.463 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.080778+0000", 2024-09-13T14:51:41.463 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.89619979699999996 2024-09-13T14:51:41.463 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.463 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.463 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.463 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.080850+0000", 2024-09-13T14:51:41.463 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2558000000000006e-05 2024-09-13T14:51:41.463 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.463 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.463 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.464 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.464 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.464 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156620 6.7 6:e7ee908f:::benchmark_data_smithi096_167087_object39392:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:51:41.464 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.064983+0000", 2024-09-13T14:51:41.464 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 309.35269562000002, 2024-09-13T14:51:41.464 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.98910626599999996, 2024-09-13T14:51:41.464 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.464 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.464 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.464 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.464 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.464 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156620 2024-09-13T14:51:41.464 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.464 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.465 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.465 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.465 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.064983+0000", 2024-09-13T14:51:41.465 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.465 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.465 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.465 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.465 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.064983+0000", 2024-09-13T14:51:41.465 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.465 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.465 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.465 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.465 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.064980+0000", 2024-09-13T14:51:41.465 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999971 2024-09-13T14:51:41.466 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.466 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.466 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.466 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.064995+0000", 2024-09-13T14:51:41.466 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5729000000000001e-05 2024-09-13T14:51:41.466 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.466 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.466 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.466 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.065025+0000", 2024-09-13T14:51:41.466 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9637000000000001e-05 2024-09-13T14:51:41.466 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.466 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.466 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.466 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.065037+0000", 2024-09-13T14:51:41.467 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1744e-05 2024-09-13T14:51:41.467 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.467 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.467 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.467 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.184918+0000", 2024-09-13T14:51:41.467 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.119881526 2024-09-13T14:51:41.467 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.467 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.467 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.467 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.184935+0000", 2024-09-13T14:51:41.467 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6475000000000001e-05 2024-09-13T14:51:41.467 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.467 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.468 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.468 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.037601+0000", 2024-09-13T14:51:41.468 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.85266643900000005 2024-09-13T14:51:41.468 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.468 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.468 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.468 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.037698+0000", 2024-09-13T14:51:41.468 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6688000000000007e-05 2024-09-13T14:51:41.468 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.468 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.468 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:51:41.468 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.038109+0000", 2024-09-13T14:51:41.468 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00041130399999999998 2024-09-13T14:51:41.469 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.469 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.469 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.469 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.042750+0000", 2024-09-13T14:51:41.469 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0046412880000000004 2024-09-13T14:51:41.469 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.469 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.469 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.469 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.050102+0000", 2024-09-13T14:51:41.469 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0073513490000000001 2024-09-13T14:51:41.469 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.469 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.469 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.469 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.053991+0000", 2024-09-13T14:51:41.470 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0038888820000000002 2024-09-13T14:51:41.470 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.470 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.470 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.470 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.054047+0000", 2024-09-13T14:51:41.470 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6336999999999999e-05 2024-09-13T14:51:41.470 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.470 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.470 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.470 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.054089+0000", 2024-09-13T14:51:41.470 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1959000000000003e-05 2024-09-13T14:51:41.470 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.470 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.470 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.471 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.471 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.471 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:206971 6.19 6:98fbcdeb:::benchmark_data_smithi096_167087_object89743:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:51:41.471 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:42.920361+0000", 2024-09-13T14:51:41.471 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 238.49731711000001, 2024-09-13T14:51:41.471 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0268845019999999, 2024-09-13T14:51:41.471 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.471 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.471 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.471 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.471 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.471 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 206971 2024-09-13T14:51:41.471 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.472 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.472 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.472 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.472 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.920361+0000", 2024-09-13T14:51:41.472 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.472 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.472 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.472 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.472 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.920361+0000", 2024-09-13T14:51:41.472 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.472 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.472 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.472 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.472 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.920360+0000", 2024-09-13T14:51:41.473 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:41.473 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.473 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.473 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.473 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.920366+0000", 2024-09-13T14:51:41.473 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4559999999999997e-06 2024-09-13T14:51:41.473 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.473 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.473 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.473 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.920377+0000", 2024-09-13T14:51:41.473 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0814e-05 2024-09-13T14:51:41.473 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.473 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.473 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.474 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.920382+0000", 2024-09-13T14:51:41.474 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.293e-06 2024-09-13T14:51:41.474 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.474 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.474 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.474 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.921347+0000", 2024-09-13T14:51:41.474 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00096445000000000005 2024-09-13T14:51:41.474 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.474 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.474 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.474 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.921355+0000", 2024-09-13T14:51:41.474 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9500000000000001e-06 2024-09-13T14:51:41.474 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.474 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.475 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.475 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.928567+0000", 2024-09-13T14:51:41.475 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.007212894 2024-09-13T14:51:41.475 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.475 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.475 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.475 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.928699+0000", 2024-09-13T14:51:41.475 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000131481 2024-09-13T14:51:41.475 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.475 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.475 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:41.475 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.928992+0000", 2024-09-13T14:51:41.475 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00029271199999999999 2024-09-13T14:51:41.475 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.476 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.476 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.476 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.932209+0000", 2024-09-13T14:51:41.476 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0032176840000000002 2024-09-13T14:51:41.476 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.476 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.476 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.476 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947158+0000", 2024-09-13T14:51:41.476 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.014948215000000001 2024-09-13T14:51:41.476 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.476 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.476 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.476 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947185+0000", 2024-09-13T14:51:41.476 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7708999999999998e-05 2024-09-13T14:51:41.477 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.477 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.477 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.477 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947211+0000", 2024-09-13T14:51:41.477 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5511999999999999e-05 2024-09-13T14:51:41.477 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.477 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.477 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.477 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947246+0000", 2024-09-13T14:51:41.477 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4913999999999998e-05 2024-09-13T14:51:41.477 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.477 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.477 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.477 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.478 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.478 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:206974 6.19 6:9e07544c:::benchmark_data_smithi096_167087_object89746:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:51:41.478 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:42.922389+0000", 2024-09-13T14:51:41.478 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 238.495289531, 2024-09-13T14:51:41.478 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.025043792, 2024-09-13T14:51:41.478 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.478 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.478 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.478 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.478 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.478 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 206974 2024-09-13T14:51:41.478 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.478 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.478 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.479 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.479 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922389+0000", 2024-09-13T14:51:41.479 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.479 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.479 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.479 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.479 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922389+0000", 2024-09-13T14:51:41.479 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.479 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.479 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.479 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.479 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922388+0000", 2024-09-13T14:51:41.479 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:51:41.479 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.480 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.480 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.480 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922394+0000", 2024-09-13T14:51:41.480 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0379999999999996e-06 2024-09-13T14:51:41.480 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.480 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.480 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.480 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922409+0000", 2024-09-13T14:51:41.480 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5366000000000002e-05 2024-09-13T14:51:41.480 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.480 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.480 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.480 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922416+0000", 2024-09-13T14:51:41.481 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8379999999999996e-06 2024-09-13T14:51:41.481 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.481 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.481 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.481 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922464+0000", 2024-09-13T14:51:41.481 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8154e-05 2024-09-13T14:51:41.481 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.481 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.481 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.481 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922471+0000", 2024-09-13T14:51:41.481 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5019999999999998e-06 2024-09-13T14:51:41.481 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.481 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.481 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.482 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.930084+0000", 2024-09-13T14:51:41.482 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0076137460000001 2024-09-13T14:51:41.482 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.482 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.482 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.482 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.930187+0000", 2024-09-13T14:51:41.482 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000102728 2024-09-13T14:51:41.482 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.482 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.482 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:41.482 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.930373+0000", 2024-09-13T14:51:41.482 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018575899999999999 2024-09-13T14:51:41.482 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.483 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.483 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.483 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.934440+0000", 2024-09-13T14:51:41.483 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.004067083 2024-09-13T14:51:41.483 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.483 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.483 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.483 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947295+0000", 2024-09-13T14:51:41.483 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012855594 2024-09-13T14:51:41.483 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.483 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.483 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.483 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947357+0000", 2024-09-13T14:51:41.483 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1308000000000003e-05 2024-09-13T14:51:41.484 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.484 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.484 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.484 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947398+0000", 2024-09-13T14:51:41.484 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1702000000000002e-05 2024-09-13T14:51:41.484 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.484 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.484 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.484 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947433+0000", 2024-09-13T14:51:41.484 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4088000000000003e-05 2024-09-13T14:51:41.484 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.484 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.484 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.484 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.485 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.485 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:206975 6.19 6:9c2f714a:::benchmark_data_smithi096_167087_object89747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:51:41.485 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:42.922638+0000", 2024-09-13T14:51:41.485 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 238.49504063800001, 2024-09-13T14:51:41.485 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0248549309999999, 2024-09-13T14:51:41.485 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.485 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.485 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.485 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.485 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.485 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 206975 2024-09-13T14:51:41.485 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.485 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.486 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.486 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.486 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922638+0000", 2024-09-13T14:51:41.486 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.486 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.486 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.486 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.486 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922638+0000", 2024-09-13T14:51:41.486 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.486 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.486 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.486 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.486 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922637+0000", 2024-09-13T14:51:41.486 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:51:41.487 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.487 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.487 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.487 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922642+0000", 2024-09-13T14:51:41.487 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.724e-06 2024-09-13T14:51:41.487 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.487 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.487 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.487 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922654+0000", 2024-09-13T14:51:41.487 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.216e-05 2024-09-13T14:51:41.487 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.487 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.487 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.487 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922659+0000", 2024-09-13T14:51:41.488 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6540000000000004e-06 2024-09-13T14:51:41.488 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.488 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.488 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.488 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922693+0000", 2024-09-13T14:51:41.488 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4094000000000003e-05 2024-09-13T14:51:41.488 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.488 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.488 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.488 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.922699+0000", 2024-09-13T14:51:41.488 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5199999999999997e-06 2024-09-13T14:51:41.488 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.488 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.488 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.489 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.931161+0000", 2024-09-13T14:51:41.489 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.008462492 2024-09-13T14:51:41.489 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.489 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.489 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.489 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.931260+0000", 2024-09-13T14:51:41.489 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.8867000000000001e-05 2024-09-13T14:51:41.489 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.489 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.489 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:41.489 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.931446+0000", 2024-09-13T14:51:41.489 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000186064 2024-09-13T14:51:41.489 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.490 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.490 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.490 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.936635+0000", 2024-09-13T14:51:41.490 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0051886370000000003 2024-09-13T14:51:41.490 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.490 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.490 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.490 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947323+0000", 2024-09-13T14:51:41.490 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010688211 2024-09-13T14:51:41.490 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.490 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.490 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.490 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947451+0000", 2024-09-13T14:51:41.490 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000127546 2024-09-13T14:51:41.491 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.491 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.491 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.491 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947474+0000", 2024-09-13T14:51:41.491 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.3329999999999999e-05 2024-09-13T14:51:41.491 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.491 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.491 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.491 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947493+0000", 2024-09-13T14:51:41.491 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8423e-05 2024-09-13T14:51:41.491 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.491 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.491 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.491 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.492 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.492 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:207000 6.19 6:9c5fbd07:::benchmark_data_smithi096_167087_object89772:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:51:41.492 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:42.941745+0000", 2024-09-13T14:51:41.492 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 238.475933535, 2024-09-13T14:51:41.492 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0059952110000001, 2024-09-13T14:51:41.492 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.492 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.492 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.492 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.492 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.492 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 207000 2024-09-13T14:51:41.492 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.492 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.493 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.493 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.493 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.941745+0000", 2024-09-13T14:51:41.493 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.493 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.493 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.493 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.493 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.941745+0000", 2024-09-13T14:51:41.493 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.493 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.493 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.493 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.494 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.941743+0000", 2024-09-13T14:51:41.494 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:41.494 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.494 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.494 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.494 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.941749+0000", 2024-09-13T14:51:41.494 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5459999999999999e-06 2024-09-13T14:51:41.494 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.494 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.494 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.494 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.941759+0000", 2024-09-13T14:51:41.494 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0524e-05 2024-09-13T14:51:41.494 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.495 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.495 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.495 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.941764+0000", 2024-09-13T14:51:41.495 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.617e-06 2024-09-13T14:51:41.495 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.495 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.495 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.495 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.941806+0000", 2024-09-13T14:51:41.495 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2091000000000001e-05 2024-09-13T14:51:41.495 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.495 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.495 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.495 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.941814+0000", 2024-09-13T14:51:41.496 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7989999999999994e-06 2024-09-13T14:51:41.496 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.496 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.496 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.496 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.932263+0000", 2024-09-13T14:51:41.496 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99044897499999995 2024-09-13T14:51:41.496 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.496 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.496 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.496 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.932334+0000", 2024-09-13T14:51:41.496 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1278000000000004e-05 2024-09-13T14:51:41.496 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.496 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.497 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:41.497 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.932520+0000", 2024-09-13T14:51:41.497 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018609400000000001 2024-09-13T14:51:41.497 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.497 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.497 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.497 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.936642+0000", 2024-09-13T14:51:41.497 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.004121652 2024-09-13T14:51:41.497 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.497 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.497 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.497 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947635+0000", 2024-09-13T14:51:41.497 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010993035 2024-09-13T14:51:41.497 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.498 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.498 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.498 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947661+0000", 2024-09-13T14:51:41.498 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5496000000000001e-05 2024-09-13T14:51:41.498 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.498 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.498 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.498 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947703+0000", 2024-09-13T14:51:41.498 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2898999999999997e-05 2024-09-13T14:51:41.498 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.498 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.498 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.498 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947740+0000", 2024-09-13T14:51:41.499 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6539999999999999e-05 2024-09-13T14:51:41.499 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.499 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.499 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.499 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.499 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.499 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:207019 6.19 6:985acf83:::benchmark_data_smithi096_167087_object89791:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:51:41.499 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:42.956380+0000", 2024-09-13T14:51:41.499 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 238.46129842100001, 2024-09-13T14:51:41.499 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99151018700000004, 2024-09-13T14:51:41.499 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.499 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.499 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.500 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.500 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.500 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 207019 2024-09-13T14:51:41.500 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.500 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.500 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.500 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.500 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.956380+0000", 2024-09-13T14:51:41.500 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.500 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.500 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.500 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.500 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.956380+0000", 2024-09-13T14:51:41.501 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.501 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.501 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.501 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.501 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.956379+0000", 2024-09-13T14:51:41.501 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:51:41.501 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.501 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.501 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.501 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.956384+0000", 2024-09-13T14:51:41.501 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7670000000000003e-06 2024-09-13T14:51:41.501 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.501 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.501 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.502 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.956394+0000", 2024-09-13T14:51:41.502 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0064000000000001e-05 2024-09-13T14:51:41.502 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.502 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.502 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.502 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.956398+0000", 2024-09-13T14:51:41.502 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0740000000000003e-06 2024-09-13T14:51:41.502 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.502 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.502 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.502 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.956433+0000", 2024-09-13T14:51:41.502 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5534000000000002e-05 2024-09-13T14:51:41.502 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.502 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.503 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.503 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.956440+0000", 2024-09-13T14:51:41.503 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4690000000000002e-06 2024-09-13T14:51:41.503 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.503 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.503 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.503 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.933346+0000", 2024-09-13T14:51:41.503 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97690568 2024-09-13T14:51:41.503 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.503 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.503 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.503 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.933447+0000", 2024-09-13T14:51:41.503 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000101519 2024-09-13T14:51:41.503 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.504 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.504 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:41.504 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.933633+0000", 2024-09-13T14:51:41.504 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018639700000000001 2024-09-13T14:51:41.504 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.504 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.504 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.504 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.937805+0000", 2024-09-13T14:51:41.504 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0041715490000000001 2024-09-13T14:51:41.504 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.504 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.504 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.504 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947781+0000", 2024-09-13T14:51:41.504 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0099756640000000004 2024-09-13T14:51:41.505 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.505 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.505 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.505 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947817+0000", 2024-09-13T14:51:41.505 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6273000000000003e-05 2024-09-13T14:51:41.505 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.505 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.505 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.505 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947857+0000", 2024-09-13T14:51:41.505 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.964e-05 2024-09-13T14:51:41.505 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.505 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.505 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.505 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947890+0000", 2024-09-13T14:51:41.506 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3476000000000003e-05 2024-09-13T14:51:41.506 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.506 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.506 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.506 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.506 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.506 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:207035 6.19 6:9ac97f42:::benchmark_data_smithi096_167087_object89807:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:51:41.506 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:42.968711+0000", 2024-09-13T14:51:41.506 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 238.44896767700001, 2024-09-13T14:51:41.506 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97932851099999996, 2024-09-13T14:51:41.506 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.506 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.506 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.506 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.507 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.507 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 207035 2024-09-13T14:51:41.507 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.507 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.507 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.507 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.507 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.968711+0000", 2024-09-13T14:51:41.507 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.507 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.507 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.507 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.507 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.968711+0000", 2024-09-13T14:51:41.507 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.507 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.508 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.508 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.508 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.968709+0000", 2024-09-13T14:51:41.508 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:51:41.508 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.508 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.508 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.508 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.968715+0000", 2024-09-13T14:51:41.508 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5079999999999996e-06 2024-09-13T14:51:41.508 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.508 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.508 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.508 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.968730+0000", 2024-09-13T14:51:41.509 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4935e-05 2024-09-13T14:51:41.512 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:41.513 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.513 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.514 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.514 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.968736+0000", 2024-09-13T14:51:41.514 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8039999999999998e-06 2024-09-13T14:51:41.514 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.514 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.514 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.514 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.968778+0000", 2024-09-13T14:51:41.514 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2276e-05 2024-09-13T14:51:41.514 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.514 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.514 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.514 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.968786+0000", 2024-09-13T14:51:41.514 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5510000000000002e-06 2024-09-13T14:51:41.515 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.515 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.515 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.515 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.934473+0000", 2024-09-13T14:51:41.515 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.96568769899999995 2024-09-13T14:51:41.515 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.515 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.515 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.515 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.934548+0000", 2024-09-13T14:51:41.515 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5269000000000001e-05 2024-09-13T14:51:41.515 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.515 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.515 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:41.516 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.934726+0000", 2024-09-13T14:51:41.516 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000177629 2024-09-13T14:51:41.516 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.516 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.516 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.516 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.937826+0000", 2024-09-13T14:51:41.516 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0031003599999999999 2024-09-13T14:51:41.516 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.516 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.516 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.516 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947921+0000", 2024-09-13T14:51:41.516 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010094242999999999 2024-09-13T14:51:41.516 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.517 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.517 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.517 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.947966+0000", 2024-09-13T14:51:41.517 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5090999999999999e-05 2024-09-13T14:51:41.517 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.517 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.517 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.517 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948007+0000", 2024-09-13T14:51:41.517 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1128000000000003e-05 2024-09-13T14:51:41.517 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.517 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.517 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.517 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948039+0000", 2024-09-13T14:51:41.517 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2172000000000003e-05 2024-09-13T14:51:41.518 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.518 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.518 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.518 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.518 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.518 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:207046 6.19 6:9cb25e95:::benchmark_data_smithi096_167087_object89818:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:51:41.518 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:42.979728+0000", 2024-09-13T14:51:41.518 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 238.43795056600001, 2024-09-13T14:51:41.518 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.96848380599999995, 2024-09-13T14:51:41.518 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.518 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.518 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.518 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.519 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.519 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 207046 2024-09-13T14:51:41.519 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.519 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.519 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.519 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.519 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.979728+0000", 2024-09-13T14:51:41.519 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.519 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.519 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.519 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.519 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.979728+0000", 2024-09-13T14:51:41.519 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.519 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.520 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.520 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.520 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.979727+0000", 2024-09-13T14:51:41.520 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:51:41.520 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.520 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.520 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.520 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.979731+0000", 2024-09-13T14:51:41.520 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6179999999999999e-06 2024-09-13T14:51:41.520 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.520 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.520 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.520 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.979740+0000", 2024-09-13T14:51:41.521 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1230000000000006e-06 2024-09-13T14:51:41.521 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.521 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.521 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.521 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.979743+0000", 2024-09-13T14:51:41.521 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7419999999999999e-06 2024-09-13T14:51:41.522 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.522 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.522 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.522 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.979777+0000", 2024-09-13T14:51:41.522 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3756e-05 2024-09-13T14:51:41.522 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.522 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.522 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.522 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.979783+0000", 2024-09-13T14:51:41.522 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4430000000000002e-06 2024-09-13T14:51:41.522 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.522 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.522 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.522 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.935568+0000", 2024-09-13T14:51:41.523 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95578573300000003 2024-09-13T14:51:41.523 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.523 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.523 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.523 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.935676+0000", 2024-09-13T14:51:41.523 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010741100000000001 2024-09-13T14:51:41.523 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.523 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.523 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:41.523 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.935861+0000", 2024-09-13T14:51:41.523 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018524599999999999 2024-09-13T14:51:41.523 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.523 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.524 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.524 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.940935+0000", 2024-09-13T14:51:41.524 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0050738720000000001 2024-09-13T14:51:41.524 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.524 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.524 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.524 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948104+0000", 2024-09-13T14:51:41.524 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0071689240000000001 2024-09-13T14:51:41.524 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.524 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.524 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.524 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948141+0000", 2024-09-13T14:51:41.524 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6739000000000002e-05 2024-09-13T14:51:41.524 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.525 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.525 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.525 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948182+0000", 2024-09-13T14:51:41.525 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1254000000000002e-05 2024-09-13T14:51:41.525 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.525 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.525 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.525 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948212+0000", 2024-09-13T14:51:41.525 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9689e-05 2024-09-13T14:51:41.525 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.525 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.525 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.525 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.526 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.526 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:207052 6.19 6:99d67287:::benchmark_data_smithi096_167087_object89824:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:51:41.526 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:42.981279+0000", 2024-09-13T14:51:41.526 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 238.43639922, 2024-09-13T14:51:41.526 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.96710304400000002, 2024-09-13T14:51:41.526 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.526 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.526 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.526 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.526 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.526 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 207052 2024-09-13T14:51:41.526 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.526 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.526 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.527 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.527 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.981279+0000", 2024-09-13T14:51:41.527 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.527 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.527 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.527 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.527 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.981279+0000", 2024-09-13T14:51:41.527 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.527 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.527 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.527 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.527 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.981278+0000", 2024-09-13T14:51:41.527 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:51:41.527 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.527 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.528 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.528 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.981284+0000", 2024-09-13T14:51:41.528 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3519999999999997e-06 2024-09-13T14:51:41.528 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.528 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.528 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.528 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.981295+0000", 2024-09-13T14:51:41.528 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1839e-05 2024-09-13T14:51:41.528 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.528 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.528 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.528 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.981302+0000", 2024-09-13T14:51:41.528 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2859999999999996e-06 2024-09-13T14:51:41.529 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.529 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.529 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.529 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.981337+0000", 2024-09-13T14:51:41.529 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5559999999999998e-05 2024-09-13T14:51:41.529 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.529 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.529 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.529 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:42.981343+0000", 2024-09-13T14:51:41.529 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2230000000000004e-06 2024-09-13T14:51:41.529 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.529 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.529 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.529 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.936668+0000", 2024-09-13T14:51:41.530 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95532426000000004 2024-09-13T14:51:41.530 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.530 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.530 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.530 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.936771+0000", 2024-09-13T14:51:41.530 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000103692 2024-09-13T14:51:41.530 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.530 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.530 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:41.530 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.936960+0000", 2024-09-13T14:51:41.530 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018829400000000001 2024-09-13T14:51:41.530 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.530 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.530 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.531 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.943644+0000", 2024-09-13T14:51:41.531 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.006684433 2024-09-13T14:51:41.531 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.531 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.531 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.531 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948246+0000", 2024-09-13T14:51:41.531 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0046023110000000004 2024-09-13T14:51:41.531 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.531 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.531 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.531 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948300+0000", 2024-09-13T14:51:41.531 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3596e-05 2024-09-13T14:51:41.531 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.532 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.532 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.532 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948346+0000", 2024-09-13T14:51:41.532 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6199000000000003e-05 2024-09-13T14:51:41.532 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.532 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.532 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.532 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948382+0000", 2024-09-13T14:51:41.532 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5896000000000003e-05 2024-09-13T14:51:41.532 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.532 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.532 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.532 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.532 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.533 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:207144 6.19 6:9fc2644a:::benchmark_data_smithi096_167087_object89916:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:51:41.533 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:43.053032+0000", 2024-09-13T14:51:41.533 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 238.36464667300001, 2024-09-13T14:51:41.533 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.89552253000000004, 2024-09-13T14:51:41.533 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.533 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.533 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.533 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.533 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.533 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 207144 2024-09-13T14:51:41.533 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.533 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.533 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.533 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.534 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.053032+0000", 2024-09-13T14:51:41.534 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.534 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.534 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.534 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.534 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.053032+0000", 2024-09-13T14:51:41.534 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.534 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.534 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.534 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.534 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.053031+0000", 2024-09-13T14:51:41.534 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:51:41.534 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.534 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.535 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.535 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.053035+0000", 2024-09-13T14:51:41.535 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7210000000000002e-06 2024-09-13T14:51:41.535 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.535 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.535 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.535 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.053044+0000", 2024-09-13T14:51:41.535 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8030000000000003e-06 2024-09-13T14:51:41.535 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.535 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.535 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.535 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.053051+0000", 2024-09-13T14:51:41.535 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7150000000000001e-06 2024-09-13T14:51:41.535 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.536 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.536 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.536 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.053097+0000", 2024-09-13T14:51:41.536 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.651e-05 2024-09-13T14:51:41.536 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.536 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.536 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.536 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.053105+0000", 2024-09-13T14:51:41.536 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.3799999999999996e-06 2024-09-13T14:51:41.536 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.536 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.536 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.536 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.937860+0000", 2024-09-13T14:51:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.88475545600000005 2024-09-13T14:51:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.937951+0000", 2024-09-13T14:51:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.0956000000000001e-05 2024-09-13T14:51:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.938116+0000", 2024-09-13T14:51:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000165219 2024-09-13T14:51:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.538 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.943666+0000", 2024-09-13T14:51:41.538 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0055493230000000001 2024-09-13T14:51:41.538 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.538 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.538 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.538 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948430+0000", 2024-09-13T14:51:41.538 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0047644580000000001 2024-09-13T14:51:41.538 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.538 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.538 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.538 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948475+0000", 2024-09-13T14:51:41.538 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5049999999999997e-05 2024-09-13T14:51:41.538 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.539 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.539 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.539 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948519+0000", 2024-09-13T14:51:41.539 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3563000000000003e-05 2024-09-13T14:51:41.539 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.539 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.539 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.539 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948554+0000", 2024-09-13T14:51:41.539 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5274999999999997e-05 2024-09-13T14:51:41.539 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.539 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.539 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.539 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.539 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.540 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:207150 6.19 6:9babf97e:::benchmark_data_smithi096_167087_object89922:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:51:41.540 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:43.058413+0000", 2024-09-13T14:51:41.540 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 238.35926482599999, 2024-09-13T14:51:41.540 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.89029254499999999, 2024-09-13T14:51:41.540 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.540 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.540 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.540 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.540 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.540 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 207150 2024-09-13T14:51:41.540 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.540 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.540 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.540 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.541 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.058413+0000", 2024-09-13T14:51:41.541 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.541 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.541 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.541 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.541 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.058413+0000", 2024-09-13T14:51:41.541 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.541 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.541 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.541 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.541 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.058413+0000", 2024-09-13T14:51:41.541 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:51:41.541 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.541 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.542 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.542 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.058417+0000", 2024-09-13T14:51:41.542 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2699999999999998e-06 2024-09-13T14:51:41.542 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.542 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.542 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.542 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.058426+0000", 2024-09-13T14:51:41.542 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1509999999999993e-06 2024-09-13T14:51:41.542 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.542 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.542 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.542 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.058431+0000", 2024-09-13T14:51:41.542 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9989999999999999e-06 2024-09-13T14:51:41.543 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.543 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.543 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.543 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.058475+0000", 2024-09-13T14:51:41.543 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3853999999999997e-05 2024-09-13T14:51:41.543 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.543 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.543 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.543 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.058482+0000", 2024-09-13T14:51:41.543 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5870000000000004e-06 2024-09-13T14:51:41.543 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.544 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.544 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.544 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.938918+0000", 2024-09-13T14:51:41.544 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.88043586900000004 2024-09-13T14:51:41.544 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.544 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.544 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.544 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.938986+0000", 2024-09-13T14:51:41.544 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7902000000000001e-05 2024-09-13T14:51:41.544 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.544 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.544 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:41.544 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.939126+0000", 2024-09-13T14:51:41.545 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013982400000000001 2024-09-13T14:51:41.545 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.545 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.545 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.545 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.943671+0000", 2024-09-13T14:51:41.545 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0045446009999999997 2024-09-13T14:51:41.545 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.545 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.545 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.545 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948595+0000", 2024-09-13T14:51:41.545 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0049248499999999997 2024-09-13T14:51:41.545 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.545 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.545 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.546 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948635+0000", 2024-09-13T14:51:41.546 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9780999999999997e-05 2024-09-13T14:51:41.546 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.546 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.546 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.546 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948672+0000", 2024-09-13T14:51:41.546 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6709999999999999e-05 2024-09-13T14:51:41.546 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.546 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.546 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.546 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948706+0000", 2024-09-13T14:51:41.546 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4020000000000003e-05 2024-09-13T14:51:41.546 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.546 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.547 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.547 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.547 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.547 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:207199 6.19 6:980d416d:::benchmark_data_smithi096_167087_object89971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:51:41.547 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:43.105032+0000", 2024-09-13T14:51:41.547 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 238.31264660799999, 2024-09-13T14:51:41.547 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.84385194399999996, 2024-09-13T14:51:41.547 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.547 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.547 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.547 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.547 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.547 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 207199 2024-09-13T14:51:41.547 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.548 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.548 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.548 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.548 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.105032+0000", 2024-09-13T14:51:41.548 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.548 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.548 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.548 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.548 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.105032+0000", 2024-09-13T14:51:41.548 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.548 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.548 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.548 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.549 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.105030+0000", 2024-09-13T14:51:41.549 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:41.549 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.549 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.549 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.549 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.105037+0000", 2024-09-13T14:51:41.549 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7039999999999997e-06 2024-09-13T14:51:41.549 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.549 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.549 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.549 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.105051+0000", 2024-09-13T14:51:41.549 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3964999999999999e-05 2024-09-13T14:51:41.549 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.549 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.550 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.550 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.105056+0000", 2024-09-13T14:51:41.550 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9060000000000001e-06 2024-09-13T14:51:41.550 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.550 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.550 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.550 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.105102+0000", 2024-09-13T14:51:41.550 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6390000000000001e-05 2024-09-13T14:51:41.550 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.550 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.550 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.550 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.105112+0000", 2024-09-13T14:51:41.550 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1700000000000003e-06 2024-09-13T14:51:41.550 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.551 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.551 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.551 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.939850+0000", 2024-09-13T14:51:41.551 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.834738604 2024-09-13T14:51:41.551 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.551 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.551 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.551 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.939962+0000", 2024-09-13T14:51:41.551 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000111711 2024-09-13T14:51:41.551 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.551 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.551 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:41.551 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.940158+0000", 2024-09-13T14:51:41.551 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00019601800000000001 2024-09-13T14:51:41.552 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.552 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.552 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.552 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.946835+0000", 2024-09-13T14:51:41.552 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0066770930000000003 2024-09-13T14:51:41.552 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.552 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.552 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.552 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948762+0000", 2024-09-13T14:51:41.552 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0019265770000000001 2024-09-13T14:51:41.552 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.552 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.552 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.552 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948804+0000", 2024-09-13T14:51:41.553 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2873000000000001e-05 2024-09-13T14:51:41.553 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.553 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.553 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.553 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948846+0000", 2024-09-13T14:51:41.553 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1117e-05 2024-09-13T14:51:41.553 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.553 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.553 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.553 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948884+0000", 2024-09-13T14:51:41.553 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8071000000000002e-05 2024-09-13T14:51:41.553 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.553 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.554 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.554 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.554 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.554 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:207220 6.19 6:9f672050:::benchmark_data_smithi096_167087_object89992:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:51:41.554 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:43.123223+0000", 2024-09-13T14:51:41.554 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 238.29445481400001, 2024-09-13T14:51:41.554 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.82581941599999997, 2024-09-13T14:51:41.554 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.554 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.554 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.554 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.554 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.554 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 207220 2024-09-13T14:51:41.554 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.123223+0000", 2024-09-13T14:51:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.123223+0000", 2024-09-13T14:51:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.555 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.123222+0000", 2024-09-13T14:51:41.556 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:41.556 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.556 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.556 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.556 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.123229+0000", 2024-09-13T14:51:41.556 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9290000000000001e-06 2024-09-13T14:51:41.556 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.556 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.556 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.556 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.123243+0000", 2024-09-13T14:51:41.556 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4078e-05 2024-09-13T14:51:41.556 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.556 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.557 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.557 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.123250+0000", 2024-09-13T14:51:41.557 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6529999999999997e-06 2024-09-13T14:51:41.557 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.557 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.557 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.557 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.123300+0000", 2024-09-13T14:51:41.557 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0362999999999999e-05 2024-09-13T14:51:41.557 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.557 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.557 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.557 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.123309+0000", 2024-09-13T14:51:41.557 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.9930000000000001e-06 2024-09-13T14:51:41.557 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.558 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.558 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.558 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.940969+0000", 2024-09-13T14:51:41.558 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.81766029799999995 2024-09-13T14:51:41.558 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.558 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.558 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.558 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.941061+0000", 2024-09-13T14:51:41.558 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1910000000000006e-05 2024-09-13T14:51:41.558 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.558 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.558 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:41.558 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.941228+0000", 2024-09-13T14:51:41.559 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016647799999999999 2024-09-13T14:51:41.559 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.559 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.559 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.559 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.946840+0000", 2024-09-13T14:51:41.559 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.005612695 2024-09-13T14:51:41.559 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.559 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.559 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.559 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948939+0000", 2024-09-13T14:51:41.559 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002098815 2024-09-13T14:51:41.559 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.559 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.559 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.560 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.948974+0000", 2024-09-13T14:51:41.560 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4804000000000001e-05 2024-09-13T14:51:41.560 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.560 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.560 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.560 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.949010+0000", 2024-09-13T14:51:41.560 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6047999999999998e-05 2024-09-13T14:51:41.560 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.560 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.560 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.560 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.949043+0000", 2024-09-13T14:51:41.560 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2948999999999999e-05 2024-09-13T14:51:41.560 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.560 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.561 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.561 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.561 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.561 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:207223 6.19 6:99851a55:::benchmark_data_smithi096_167087_object89995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:51:41.561 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:43.124536+0000", 2024-09-13T14:51:41.561 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 238.29314205200001, 2024-09-13T14:51:41.561 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.82463943399999995, 2024-09-13T14:51:41.561 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:41.561 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:41.561 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:41.561 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:51:41.561 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:51:41.561 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 207223 2024-09-13T14:51:41.562 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.562 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:41.562 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.562 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:41.562 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.124536+0000", 2024-09-13T14:51:41.562 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.562 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.562 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.562 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:41.562 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.124536+0000", 2024-09-13T14:51:41.562 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:41.562 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.562 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.562 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:41.563 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.124535+0000", 2024-09-13T14:51:41.563 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:51:41.563 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.563 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.563 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:41.563 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.124544+0000", 2024-09-13T14:51:41.563 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7490000000000004e-06 2024-09-13T14:51:41.563 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.563 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.563 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:41.563 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.124555+0000", 2024-09-13T14:51:41.563 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0839000000000001e-05 2024-09-13T14:51:41.563 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.563 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.564 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:41.564 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.124560+0000", 2024-09-13T14:51:41.564 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2669999999999997e-06 2024-09-13T14:51:41.564 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.564 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.564 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.564 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.124597+0000", 2024-09-13T14:51:41.564 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6876999999999999e-05 2024-09-13T14:51:41.564 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.564 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.564 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:51:41.564 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.124603+0000", 2024-09-13T14:51:41.564 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.353e-06 2024-09-13T14:51:41.564 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.565 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.565 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:41.565 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.942044+0000", 2024-09-13T14:51:41.565 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.81744012899999996 2024-09-13T14:51:41.565 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.565 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.565 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:41.565 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.942162+0000", 2024-09-13T14:51:41.565 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011822100000000001 2024-09-13T14:51:41.565 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.565 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.565 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:41.565 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.942349+0000", 2024-09-13T14:51:41.565 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000187014 2024-09-13T14:51:41.566 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.566 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.566 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:41.566 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.946939+0000", 2024-09-13T14:51:41.566 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0045905169999999997 2024-09-13T14:51:41.566 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.566 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.566 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.566 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.949079+0000", 2024-09-13T14:51:41.566 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0021395659999999999 2024-09-13T14:51:41.566 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.566 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.566 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:41.567 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.949109+0000", 2024-09-13T14:51:41.567 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0446999999999998e-05 2024-09-13T14:51:41.567 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.567 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.567 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:51:41.567 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.949146+0000", 2024-09-13T14:51:41.567 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6161999999999997e-05 2024-09-13T14:51:41.567 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:41.567 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:41.567 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:51:41.567 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:43.949176+0000", 2024-09-13T14:51:41.567 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0147000000000001e-05 2024-09-13T14:51:41.567 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.567 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.568 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.568 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:41.568 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:41.568 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:51:41.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:41 smithi096 ceph-mon[98901]: pgmap v743: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 10803/157359 objects misplaced (6.865%); 3.8 MiB/s, 61 objects/s recovering 2024-09-13T14:51:41.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:41 smithi096 ceph-mon[102928]: pgmap v743: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 10803/157359 objects misplaced (6.865%); 3.8 MiB/s, 61 objects/s recovering 2024-09-13T14:51:41.608 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:41.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:41.709 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:41.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.779 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:41.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:41.880 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:41.948 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:41.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:42.051 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:42.079 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:42.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:42.181 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:42.197 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:42.220 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:51:42.224 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:42.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:42.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:42 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/799648642' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:51:42.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:42 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/799648642' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:51:42.340 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:51:42.341 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:42.376 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:42.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:42.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:42 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/799648642' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:51:42.478 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:42.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.500 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:42.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:42.601 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:42.639 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:42.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:42.668 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:42.741 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:42.781 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:42.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:42.883 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:42.922 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:42.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:43.024 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:43.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.048 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:43.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:43.149 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:43.186 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:43.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:43.288 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:43.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.329 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:43.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:43.430 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:43.468 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:43.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:43.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:43 smithi204 ceph-mon[87071]: pgmap v744: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 9766/152487 objects misplaced (6.404%); 6.6 MiB/s, 106 objects/s recovering 2024-09-13T14:51:43.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:43 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:43.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:43 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/799648642' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:51:43.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:43 smithi204 ceph-mon[87071]: osdmap e251: 8 total, 8 up, 8 in 2024-09-13T14:51:43.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:43 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/34120044' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:51:43.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:43 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:51:43.569 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:43.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:43 smithi096 ceph-mon[98901]: pgmap v744: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 9766/152487 objects misplaced (6.404%); 6.6 MiB/s, 106 objects/s recovering 2024-09-13T14:51:43.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:43 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:43.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:43 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/799648642' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:51:43.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:43 smithi096 ceph-mon[98901]: osdmap e251: 8 total, 8 up, 8 in 2024-09-13T14:51:43.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:43 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/34120044' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:51:43.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:43 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:51:43.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:43 smithi096 ceph-mon[102928]: pgmap v744: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 9766/152487 objects misplaced (6.404%); 6.6 MiB/s, 106 objects/s recovering 2024-09-13T14:51:43.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:43 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:43.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:43 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/799648642' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:51:43.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:43 smithi096 ceph-mon[102928]: osdmap e251: 8 total, 8 up, 8 in 2024-09-13T14:51:43.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:43 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/34120044' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:51:43.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:43 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:51:43.608 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:43.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:43.617 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T14:51:43.710 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:43.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.751 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:43.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:43.852 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:43.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.877 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:43.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:43.978 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:44.007 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:44.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:44.108 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:44.146 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:44.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:44.249 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:44.293 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:44.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:44.395 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:44.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.438 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:44.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:44.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:44 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:51:44.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:44 smithi204 ceph-mon[87071]: osdmap e252: 8 total, 8 up, 8 in 2024-09-13T14:51:44.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:44 smithi204 ceph-mon[87071]: osdmap e253: 8 total, 8 up, 8 in 2024-09-13T14:51:44.538 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:44.559 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:44.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:44.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:44 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:51:44.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:44 smithi096 ceph-mon[98901]: osdmap e252: 8 total, 8 up, 8 in 2024-09-13T14:51:44.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:44 smithi096 ceph-mon[98901]: osdmap e253: 8 total, 8 up, 8 in 2024-09-13T14:51:44.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:44 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:51:44.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:44 smithi096 ceph-mon[102928]: osdmap e252: 8 total, 8 up, 8 in 2024-09-13T14:51:44.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:44 smithi096 ceph-mon[102928]: osdmap e253: 8 total, 8 up, 8 in 2024-09-13T14:51:44.661 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:44.691 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:44.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:44.793 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:44.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.826 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:44.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:44.868 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:51:44.868 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:51:44.869 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:44.869 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193435 7.7 7:eeb2e43c:::benchmark_data_smithi096_196313_object73304:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-09-13T14:51:44.869 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:44.857601+0000", 2024-09-13T14:51:44.869 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0092721929999999998, 2024-09-13T14:51:44.869 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0092944889999999995, 2024-09-13T14:51:44.869 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:51:44.869 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:51:44.869 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:51:44.869 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:51:44.870 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:51:44.870 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193435 2024-09-13T14:51:44.870 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:44.870 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:51:44.870 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:44.870 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:51:44.870 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:44.857601+0000", 2024-09-13T14:51:44.870 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:44.870 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:44.870 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:44.870 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:51:44.870 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:44.857601+0000", 2024-09-13T14:51:44.871 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:51:44.871 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:44.871 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:44.871 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:51:44.871 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:44.857600+0000", 2024-09-13T14:51:44.872 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:44.872 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:44.872 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:44.873 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:51:44.873 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:44.857606+0000", 2024-09-13T14:51:44.873 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9880000000000001e-06 2024-09-13T14:51:44.873 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:44.873 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:44.873 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:51:44.873 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:44.857621+0000", 2024-09-13T14:51:44.874 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5096000000000001e-05 2024-09-13T14:51:44.874 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:44.874 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:44.874 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:51:44.874 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:44.857627+0000", 2024-09-13T14:51:44.874 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.161e-06 2024-09-13T14:51:44.874 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:44.875 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:44.875 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:51:44.875 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:44.857678+0000", 2024-09-13T14:51:44.875 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0717000000000002e-05 2024-09-13T14:51:44.876 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:44.876 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:44.876 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:51:44.876 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:44.857790+0000", 2024-09-13T14:51:44.876 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000112716 2024-09-13T14:51:44.876 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:44.877 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:44.877 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,1,3", 2024-09-13T14:51:44.877 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:44.857909+0000", 2024-09-13T14:51:44.877 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000118402 2024-09-13T14:51:44.877 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:44.877 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:44.878 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:51:44.878 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:44.862000+0000", 2024-09-13T14:51:44.878 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0040912120000000003 2024-09-13T14:51:44.878 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:51:44.878 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:51:44.878 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:44.879 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:44.866081+0000", 2024-09-13T14:51:44.879 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0040814759999999997 2024-09-13T14:51:44.879 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:44.879 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:51:44.879 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:44.879 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:51:44.880 INFO:teuthology.orchestra.run.smithi204.stdout: ], 2024-09-13T14:51:44.880 INFO:teuthology.orchestra.run.smithi204.stdout: "num_ops": 1 2024-09-13T14:51:44.880 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:51:44.927 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:44.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.964 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:44.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:45.065 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:45.093 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:45.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:45.195 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:45.234 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:45.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:45.336 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:45.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.367 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:45.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:45.468 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:45 smithi204 ceph-mon[87071]: pgmap v747: 99 pgs: 3 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 92 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 9554/146352 objects misplaced (6.528%); 4.8 MiB/s, 77 objects/s recovering 2024-09-13T14:51:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:45 smithi204 ceph-mon[87071]: osdmap e254: 8 total, 8 up, 8 in 2024-09-13T14:51:45.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:45 smithi096 ceph-mon[98901]: pgmap v747: 99 pgs: 3 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 92 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 9554/146352 objects misplaced (6.528%); 4.8 MiB/s, 77 objects/s recovering 2024-09-13T14:51:45.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:45 smithi096 ceph-mon[98901]: osdmap e254: 8 total, 8 up, 8 in 2024-09-13T14:51:45.482 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:45 smithi096 ceph-mon[102928]: pgmap v747: 99 pgs: 3 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 92 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 9554/146352 objects misplaced (6.528%); 4.8 MiB/s, 77 objects/s recovering 2024-09-13T14:51:45.482 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:45 smithi096 ceph-mon[102928]: osdmap e254: 8 total, 8 up, 8 in 2024-09-13T14:51:45.495 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:45.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:45.596 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:51:45.596 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:45.619 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:45.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:45.721 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:45.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.744 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:45.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:45.845 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:45.861 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:45.866 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:45.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:45.968 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:45.991 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:45.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:46.093 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:46.115 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:46.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:46.217 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:46.247 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:46.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:46.349 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:46.396 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:46.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:46.498 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:46.543 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:46.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:46.645 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:46.670 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:46.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:46.772 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:46.813 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:46.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:46.915 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:46.941 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:46.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:47.004 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T14:51:47.043 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:47.069 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:47.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:47.171 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:47.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.230 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:47.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:47.307 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:47 smithi096 ceph-mon[98901]: pgmap v750: 99 pgs: 3 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 92 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 9554/144879 objects misplaced (6.594%) 2024-09-13T14:51:47.307 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:47 smithi096 ceph-mon[98901]: osdmap e255: 8 total, 8 up, 8 in 2024-09-13T14:51:47.309 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:47 smithi096 ceph-mon[102928]: pgmap v750: 99 pgs: 3 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 92 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 9554/144879 objects misplaced (6.594%) 2024-09-13T14:51:47.309 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:47 smithi096 ceph-mon[102928]: osdmap e255: 8 total, 8 up, 8 in 2024-09-13T14:51:47.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:47.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.387 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:47.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:47.392 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:47 smithi204 ceph-mon[87071]: pgmap v750: 99 pgs: 3 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 92 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 9554/144879 objects misplaced (6.594%) 2024-09-13T14:51:47.392 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:47 smithi204 ceph-mon[87071]: osdmap e255: 8 total, 8 up, 8 in 2024-09-13T14:51:47.488 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:47.538 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:47.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:47.640 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:47.663 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:51:47.663 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:51:47.663 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T14:51:47.663 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T14:51:47.663 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:51:47.684 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:47.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:47.786 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:47.821 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:47.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:47.923 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:47.961 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:47.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:48.063 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:48.096 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:48.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:48.198 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:48.210 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:51:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:51:47] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.51.0" 2024-09-13T14:51:48.225 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:48.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:48.327 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:48.346 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:51:48.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.399 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:48.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:48.470 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:48 smithi096 ceph-mon[98901]: osdmap e256: 8 total, 8 up, 8 in 2024-09-13T14:51:48.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:48 smithi204 ceph-mon[87071]: osdmap e256: 8 total, 8 up, 8 in 2024-09-13T14:51:48.471 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:48 smithi096 ceph-mon[102928]: osdmap e256: 8 total, 8 up, 8 in 2024-09-13T14:51:48.500 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:48.532 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:51:48.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.553 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:48.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:48.654 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:48.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.681 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:48.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:48.686 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:48.782 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:48.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.808 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:48.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:48.888 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:48.909 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:48.938 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:48.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:48.975 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:51:48.975 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:51:49.016 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:51:49.016 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:51:49.040 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:49.055 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:51:49.056 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:51:49.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.066 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:49.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:49.099 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:51:49.099 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:51:49.129 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:51:49.130 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:51:49.165 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:51:49.165 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:51:49.167 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:49.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.192 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:49.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:49.194 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:51:49.195 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:51:49.237 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:51:49.237 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:51:49.293 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:49.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:49 smithi096 ceph-mon[98901]: pgmap v753: 99 pgs: 4 peering, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 92 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 8137/141372 objects misplaced (5.756%); 16 MiB/s, 252 objects/s recovering 2024-09-13T14:51:49.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:49 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2214116194' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:49 smithi096 ceph-mon[102928]: pgmap v753: 99 pgs: 4 peering, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 92 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 8137/141372 objects misplaced (5.756%); 16 MiB/s, 252 objects/s recovering 2024-09-13T14:51:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:49 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2214116194' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:49.338 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:49.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:49.440 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:49.468 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:49.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:49 smithi204 ceph-mon[87071]: pgmap v753: 99 pgs: 4 peering, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 92 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 8137/141372 objects misplaced (5.756%); 16 MiB/s, 252 objects/s recovering 2024-09-13T14:51:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:49 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2214116194' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:49.570 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:49.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.606 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:49.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:49.707 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:49.739 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:49.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:49.841 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:49.866 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:49.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:49.911 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:49.968 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:50.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.014 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:50.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:50.115 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:50.146 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:50.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:50.248 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:50.274 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:50.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:50.376 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:50.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.419 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:50.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:51:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:51:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:50.520 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:50.558 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:50.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:50.660 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:50.696 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:50.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:50.798 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:50.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.837 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:50.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:50.938 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:50.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.969 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:50.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:51.070 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:51.109 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:51.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:51.210 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:51.241 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:51.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:51.343 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:51.348 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:51 smithi204 ceph-mon[87071]: pgmap v754: 99 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 7911/136500 objects misplaced (5.796%); 11 MiB/s, 176 objects/s recovering 2024-09-13T14:51:51.348 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:51.348 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:51.383 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:51.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:51.384 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:51:51.385 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:51:51.485 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:51.522 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:51.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:51.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:51 smithi096 ceph-mon[98901]: pgmap v754: 99 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 7911/136500 objects misplaced (5.796%); 11 MiB/s, 176 objects/s recovering 2024-09-13T14:51:51.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:51.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:51.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:51 smithi096 ceph-mon[102928]: pgmap v754: 99 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 7911/136500 objects misplaced (5.796%); 11 MiB/s, 176 objects/s recovering 2024-09-13T14:51:51.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:51:51.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:51.626 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:51.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.666 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:51.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:51.767 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:51.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.802 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:51.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:51.903 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:51.936 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:51.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:52.038 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:52.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.079 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:52.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:52.132 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:51:52.181 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:52.216 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:52.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:52.318 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:52.353 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:52.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:52.386 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:51:52.455 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:52.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.491 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:52.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:52.536 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:52.593 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:52.619 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:52.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:52.721 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:52.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.754 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:52.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:52.855 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:52.886 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:52.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:52.988 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:53.028 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:53.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:53.130 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:53.158 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:53.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:53.260 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:53.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.307 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:53.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:53.397 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:53 smithi096 ceph-mon[98901]: pgmap v755: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 6974/131649 objects misplaced (5.297%); 14 MiB/s, 231 objects/s recovering 2024-09-13T14:51:53.397 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:53 smithi096 ceph-mon[102928]: pgmap v755: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 6974/131649 objects misplaced (5.297%); 14 MiB/s, 231 objects/s recovering 2024-09-13T14:51:53.409 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:53.451 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:53.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:53.553 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:53.613 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:53.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:53.708 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:53 smithi204 ceph-mon[87071]: pgmap v755: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 6974/131649 objects misplaced (5.297%); 14 MiB/s, 231 objects/s recovering 2024-09-13T14:51:53.715 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:53.740 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:53.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:53.842 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:53.894 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:53.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:53.929 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:53.996 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:54.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.044 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:54.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:54.145 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:54.175 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:54.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:54.234 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T14:51:54.276 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:54.303 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:54.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:54.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:54 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/271896881' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:51:54.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:54 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/271896881' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:51:54.404 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:54.434 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:54.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:54.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:54 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/271896881' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:51:54.536 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:54.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.584 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:54.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:54.685 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:54.715 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:54.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:54.817 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:54.851 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:54.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:54.953 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:54.982 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:54.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:55.084 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:55.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.111 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:55.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:55.212 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:55.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.240 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:55.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:55.241 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:51:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[98901]: pgmap v756: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 6735/124896 objects misplaced (5.392%); 12 MiB/s, 194 objects/s recovering 2024-09-13T14:51:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/271896881' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:51:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[98901]: osdmap e257: 8 total, 8 up, 8 in 2024-09-13T14:51:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1023316753' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:55.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[98901]: osdmap e258: 8 total, 8 up, 8 in 2024-09-13T14:51:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[102928]: pgmap v756: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 6735/124896 objects misplaced (5.392%); 12 MiB/s, 194 objects/s recovering 2024-09-13T14:51:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/271896881' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:51:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[102928]: osdmap e257: 8 total, 8 up, 8 in 2024-09-13T14:51:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1023316753' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:55 smithi096 ceph-mon[102928]: osdmap e258: 8 total, 8 up, 8 in 2024-09-13T14:51:55.341 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:55.363 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:55.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:55.465 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:55.485 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:55.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:55.587 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:55.612 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:55.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:55.714 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:55.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:55 smithi204 ceph-mon[87071]: pgmap v756: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 6735/124896 objects misplaced (5.392%); 12 MiB/s, 194 objects/s recovering 2024-09-13T14:51:55.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:55 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/271896881' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:51:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:55 smithi204 ceph-mon[87071]: osdmap e257: 8 total, 8 up, 8 in 2024-09-13T14:51:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:55 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1023316753' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:55 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:55 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:55 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:55 smithi204 ceph-mon[87071]: osdmap e258: 8 total, 8 up, 8 in 2024-09-13T14:51:55.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.743 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:55.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:55.773 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:51:55.773 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:51:55.845 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:55.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.889 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:55.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:55.990 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:56.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.021 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:56.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:56.122 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:56.156 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:56.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:56.258 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:56.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.299 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:56.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:56.400 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:56.428 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:56.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:56.530 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:56.563 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:56.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:56.641 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T14:51:56.641 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:51:56.666 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:56.745 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:56.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:56.847 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:56.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.888 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:56.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:56.989 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:57.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.035 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:57.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:57.136 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:57.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.170 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:57.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:57.243 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:51:57.244 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:51:57.271 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:57.291 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:51:57.292 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:51:57.326 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:51:57.327 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:51:57.334 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:57.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:57.335 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:57.368 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:51:57.369 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:51:57.401 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:51:57.401 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:51:57.436 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:57.437 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:51:57.438 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:51:57.438 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:57 smithi096 ceph-mon[98901]: pgmap v759: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 6735/123384 objects misplaced (5.459%); 4.2 MiB/s, 67 objects/s recovering 2024-09-13T14:51:57.438 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:57 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1346939732' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:57.438 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:57 smithi096 ceph-mon[102928]: pgmap v759: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 6735/123384 objects misplaced (5.459%); 4.2 MiB/s, 67 objects/s recovering 2024-09-13T14:51:57.439 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:57 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1346939732' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:57.472 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:51:57.473 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:51:57.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.494 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:57.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:57.510 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:51:57.510 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:51:57.555 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:57 smithi204 ceph-mon[87071]: pgmap v759: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 6735/123384 objects misplaced (5.459%); 4.2 MiB/s, 67 objects/s recovering 2024-09-13T14:51:57.555 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:57 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1346939732' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:57.590 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:51:57.596 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:57.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.659 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:57.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:57.761 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:57.796 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:57.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:57.898 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:57.943 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:57.943 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:57.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:58.045 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:58.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:51:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:51:57] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.51.0" 2024-09-13T14:51:58.079 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:58.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:58.181 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:58.185 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:51:58.218 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:58.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:58.320 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:58.354 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:58.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:58.456 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:58.497 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:58.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:58.599 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:58.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.642 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:58.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:58.743 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:58.776 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:58.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:58.878 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:58.938 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:58.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:59.040 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:59.074 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:59.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:59.176 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:59.216 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:59.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:59.319 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:59.361 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:59.378 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:59.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:51:59 smithi204 ceph-mon[87071]: pgmap v760: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 5920/119379 objects misplaced (4.959%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:51:59.479 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:59.512 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:51:59 smithi096 ceph-mon[98901]: pgmap v760: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 5920/119379 objects misplaced (4.959%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:51:59.512 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:51:59 smithi096 ceph-mon[102928]: pgmap v760: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 5920/119379 objects misplaced (4.959%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:51:59.517 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:59.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:59.619 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:59.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.653 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:59.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:59.756 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:59.795 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:51:59.796 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:51:59.796 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T14:51:59.796 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T14:51:59.796 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:51:59.798 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:51:59.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:59.900 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:59.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.930 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:51:59.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:00.031 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:00.056 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:00.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:00.158 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:00.202 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:00.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:00.304 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:00.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.344 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:00.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:00.433 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:52:00.444 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:52:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:52:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:00.631 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:52:00.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.676 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:00.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:00.778 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:00.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.828 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:00.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:00.929 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:00.973 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:00.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:00.999 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:52:01.041 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:52:01.049 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:52:01.075 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:01.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.119 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:01.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:01.221 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:01.247 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:01.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:01.349 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:01.375 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:01.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:01 smithi204 ceph-mon[87071]: pgmap v761: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 5674/114054 objects misplaced (4.975%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T14:52:01.477 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:01.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.530 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:01.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:01.548 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:01 smithi096 ceph-mon[98901]: pgmap v761: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 5674/114054 objects misplaced (4.975%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T14:52:01.548 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:01 smithi096 ceph-mon[102928]: pgmap v761: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 5674/114054 objects misplaced (4.975%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T14:52:01.636 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:01.706 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:01.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:01.809 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:01.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.860 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:01.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:01.962 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:02.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.001 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:02.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:02.102 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:02.124 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:02.131 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:02.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:02.234 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:02.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.289 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:02.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:02.372 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:02 smithi096 ceph-mon[98901]: from='client.55246 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:02.372 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:02 smithi096 ceph-mon[102928]: from='client.55246 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:02.391 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:02.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.418 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:02.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:02.519 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:02.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.546 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:02.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:02.604 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":761,"stamp":"2024-09-13T14:51:59.898970+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2494437349,"num_objects":38018,"num_object_clones":0,"num_object_copies":114054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5674,"num_objects_unfound":0,"num_objects_dirty":38018,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":203175,"num_write_kb":7700816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5419,"num_bytes_recovered":355811154,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127863,"ondisk_log_size":127863,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":300,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11453268,"kb_used_data":7511236,"kb_used_omap":35,"kb_used_meta":3941788,"kb_avail":738507948,"statfs":{"total":767960285184,"available":756232138752,"internally_reserved":0,"allocated":7691505664,"data_stored":7677711869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36830,"internal_metadata":4036390946},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-384368640,"num_objects":-5865,"num_object_clones":0,"num_object_copies":-17595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1300,"num_objects_unfound":0,"num_objects_dirty":-5865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5865,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":483,"num_bytes_recovered":31653888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003342"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":384,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.242203+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:51:55.242203+0000","last_peered":"2024-09-13T14:51:55.242203+0000","last_clean":"2024-09-13T14:51:55.242203+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:51:55.242203+0000","last_undegraded":"2024-09-13T14:51:55.242203+0000","last_fullsized":"2024-09-13T14:51:55.242203+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":224,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.240184+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:51:54.240184+0000","last_peered":"2024-09-13T14:51:54.240184+0000","last_clean":"2024-09-13T14:51:54.240184+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:51:54.240184+0000","last_undegraded":"2024-09-13T14:51:54.240184+0000","last_fullsized":"2024-09-13T14:51:54.240184+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":221,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.255027+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:51:54.255027+0000","last_peered":"2024-09-13T14:51:54.255027+0000","last_clean":"2024-09-13T14:51:54.255027+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:51:54.255027+0000","last_undegraded":"2024-09-13T14:51:54.255027+0000","last_fullsized":"2024-09-13T14:51:54.255027+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":398,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.245605+0000","last_change":"2024-09-13T14:51:44.196384+0000","last_active":"2024-09-13T14:51:55.245605+0000","last_peered":"2024-09-13T14:51:55.245605+0000","last_clean":"2024-09-13T14:51:55.245605+0000","last_became_active":"2024-09-13T14:51:44.196082+0000","last_became_peered":"2024-09-13T14:51:44.196082+0000","last_unstale":"2024-09-13T14:51:55.245605+0000","last_undegraded":"2024-09-13T14:51:55.245605+0000","last_fullsized":"2024-09-13T14:51:55.245605+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:11:03.757803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":392,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.246134+0000","last_change":"2024-09-13T14:51:44.199134+0000","last_active":"2024-09-13T14:51:55.246134+0000","last_peered":"2024-09-13T14:51:55.246134+0000","last_clean":"2024-09-13T14:51:55.246134+0000","last_became_active":"2024-09-13T14:51:44.198525+0000","last_became_peered":"2024-09-13T14:51:44.198525+0000","last_unstale":"2024-09-13T14:51:55.246134+0000","last_undegraded":"2024-09-13T14:51:55.246134+0000","last_fullsized":"2024-09-13T14:51:55.246134+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:44.009127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":225,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.240102+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T14:51:54.240102+0000","last_peered":"2024-09-13T14:51:54.240102+0000","last_clean":"2024-09-13T14:51:54.240102+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T14:51:54.240102+0000","last_undegraded":"2024-09-13T14:51:54.240102+0000","last_fullsized":"2024-09-13T14:51:54.240102+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":45,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.241573+0000","last_change":"2024-09-13T14:51:44.207246+0000","last_active":"2024-09-13T14:51:55.241573+0000","last_peered":"2024-09-13T14:51:55.241573+0000","last_clean":"2024-09-13T14:51:55.241573+0000","last_became_active":"2024-09-13T14:51:44.206637+0000","last_became_peered":"2024-09-13T14:51:44.206637+0000","last_unstale":"2024-09-13T14:51:55.241573+0000","last_undegraded":"2024-09-13T14:51:55.241573+0000","last_fullsized":"2024-09-13T14:51:55.241573+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:42.020301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":52,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.255863+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T14:51:54.255863+0000","last_peered":"2024-09-13T14:51:54.255863+0000","last_clean":"2024-09-13T14:51:54.255863+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T14:51:54.255863+0000","last_undegraded":"2024-09-13T14:51:54.255863+0000","last_fullsized":"2024-09-13T14:51:54.255863+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"258'12479","reported_seq":12567,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:57.933567+0000","last_change":"2024-09-13T14:48:32.204190+0000","last_active":"2024-09-13T14:51:57.933567+0000","last_peered":"2024-09-13T14:51:57.933567+0000","last_clean":"2024-09-13T14:51:57.933567+0000","last_became_active":"2024-09-13T14:48:32.203935+0000","last_became_peered":"2024-09-13T14:48:32.203935+0000","last_unstale":"2024-09-13T14:51:57.933567+0000","last_undegraded":"2024-09-13T14:51:57.933567+0000","last_fullsized":"2024-09-13T14:51:57.933567+0000","mapping_epoch":225,"log_start":"233'4800","ondisk_log_start":"233'4800","created":223,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7679,"log_dups_size":0,"ondisk_log_size":7679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:29:24.200434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153157632,"num_objects":2337,"num_object_clones":0,"num_object_copies":7011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12479,"num_write_kb":474112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":367,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.255265+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:51:54.255265+0000","last_peered":"2024-09-13T14:51:54.255265+0000","last_clean":"2024-09-13T14:51:54.255265+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:51:54.255265+0000","last_undegraded":"2024-09-13T14:51:54.255265+0000","last_fullsized":"2024-09-13T14:51:54.255265+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":244,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.254760+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T14:51:54.254760+0000","last_peered":"2024-09-13T14:51:54.254760+0000","last_clean":"2024-09-13T14:51:54.254760+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T14:51:54.254760+0000","last_undegraded":"2024-09-13T14:51:54.254760+0000","last_fullsized":"2024-09-13T14:51:54.254760+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"258'12647","reported_seq":12729,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:58.108336+0000","last_change":"2024-09-13T14:48:30.196807+0000","last_active":"2024-09-13T14:51:58.108336+0000","last_peered":"2024-09-13T14:51:58.108336+0000","last_clean":"2024-09-13T14:51:58.108336+0000","last_became_active":"2024-09-13T14:48:30.195433+0000","last_became_peered":"2024-09-13T14:48:30.195433+0000","last_unstale":"2024-09-13T14:51:58.108336+0000","last_undegraded":"2024-09-13T14:51:58.108336+0000","last_fullsized":"2024-09-13T14:51:58.108336+0000","mapping_epoch":223,"log_start":"234'5668","ondisk_log_start":"234'5668","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":6979,"log_dups_size":0,"ondisk_log_size":6979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:31:16.623107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146341888,"num_objects":2233,"num_object_clones":0,"num_object_copies":6699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12647,"num_write_kb":476160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":224,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.240506+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:51:54.240506+0000","last_peered":"2024-09-13T14:51:54.240506+0000","last_clean":"2024-09-13T14:51:54.240506+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:51:54.240506+0000","last_undegraded":"2024-09-13T14:51:54.240506+0000","last_fullsized":"2024-09-13T14:51:54.240506+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":279,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.252765+0000","last_change":"2024-09-13T14:51:11.059483+0000","last_active":"2024-09-13T14:51:54.252765+0000","last_peered":"2024-09-13T14:51:54.252765+0000","last_clean":"2024-09-13T14:51:54.252765+0000","last_became_active":"2024-09-13T14:51:11.059262+0000","last_became_peered":"2024-09-13T14:51:11.059262+0000","last_unstale":"2024-09-13T14:51:54.252765+0000","last_undegraded":"2024-09-13T14:51:54.252765+0000","last_fullsized":"2024-09-13T14:51:54.252765+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:45:35.152631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"258'12517","reported_seq":12599,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:58.254503+0000","last_change":"2024-09-13T14:48:30.196414+0000","last_active":"2024-09-13T14:51:58.254503+0000","last_peered":"2024-09-13T14:51:58.254503+0000","last_clean":"2024-09-13T14:51:58.254503+0000","last_became_active":"2024-09-13T14:48:30.195744+0000","last_became_peered":"2024-09-13T14:48:30.195744+0000","last_unstale":"2024-09-13T14:51:58.254503+0000","last_undegraded":"2024-09-13T14:51:58.254503+0000","last_fullsized":"2024-09-13T14:51:58.254503+0000","mapping_epoch":223,"log_start":"234'5100","ondisk_log_start":"234'5100","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7417,"log_dups_size":0,"ondisk_log_size":7417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.893543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153550848,"num_objects":2343,"num_object_clones":0,"num_object_copies":7029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12517,"num_write_kb":475520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":234,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.240478+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T14:51:54.240478+0000","last_peered":"2024-09-13T14:51:54.240478+0000","last_clean":"2024-09-13T14:51:54.240478+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T14:51:54.240478+0000","last_undegraded":"2024-09-13T14:51:54.240478+0000","last_fullsized":"2024-09-13T14:51:54.240478+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":52,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.255967+0000","last_change":"2024-09-13T14:51:44.190879+0000","last_active":"2024-09-13T14:51:54.255967+0000","last_peered":"2024-09-13T14:51:54.255967+0000","last_clean":"2024-09-13T14:51:54.255967+0000","last_became_active":"2024-09-13T14:51:44.190617+0000","last_became_peered":"2024-09-13T14:51:44.190617+0000","last_unstale":"2024-09-13T14:51:54.255967+0000","last_undegraded":"2024-09-13T14:51:54.255967+0000","last_fullsized":"2024-09-13T14:51:54.255967+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:19.536298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"257'12676","reported_seq":12756,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.140702+0000","last_change":"2024-09-13T14:48:30.193569+0000","last_active":"2024-09-13T14:51:55.140702+0000","last_peered":"2024-09-13T14:51:55.140702+0000","last_clean":"2024-09-13T14:51:55.140702+0000","last_became_active":"2024-09-13T14:48:30.191205+0000","last_became_peered":"2024-09-13T14:48:30.191205+0000","last_unstale":"2024-09-13T14:51:55.140702+0000","last_undegraded":"2024-09-13T14:51:55.140702+0000","last_fullsized":"2024-09-13T14:51:55.140702+0000","mapping_epoch":223,"log_start":"233'3968","ondisk_log_start":"233'3968","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8708,"log_dups_size":0,"ondisk_log_size":8708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:27.050525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164364288,"num_objects":2508,"num_object_clones":0,"num_object_copies":7524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12676,"num_write_kb":485888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":242,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.255026+0000","last_change":"2024-09-13T14:48:32.206148+0000","last_active":"2024-09-13T14:51:54.255026+0000","last_peered":"2024-09-13T14:51:54.255026+0000","last_clean":"2024-09-13T14:51:54.255026+0000","last_became_active":"2024-09-13T14:48:32.205832+0000","last_became_peered":"2024-09-13T14:48:32.205832+0000","last_unstale":"2024-09-13T14:51:54.255026+0000","last_undegraded":"2024-09-13T14:51:54.255026+0000","last_fullsized":"2024-09-13T14:51:54.255026+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:55.026094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":379,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.245869+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:51:55.245869+0000","last_peered":"2024-09-13T14:51:55.245869+0000","last_clean":"2024-09-13T14:51:55.245869+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:51:55.245869+0000","last_undegraded":"2024-09-13T14:51:55.245869+0000","last_fullsized":"2024-09-13T14:51:55.245869+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"258'12745","reported_seq":12827,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:58.282113+0000","last_change":"2024-09-13T14:48:30.198101+0000","last_active":"2024-09-13T14:51:58.282113+0000","last_peered":"2024-09-13T14:51:58.282113+0000","last_clean":"2024-09-13T14:51:58.282113+0000","last_became_active":"2024-09-13T14:48:30.197770+0000","last_became_peered":"2024-09-13T14:48:30.197770+0000","last_unstale":"2024-09-13T14:51:58.282113+0000","last_undegraded":"2024-09-13T14:51:58.282113+0000","last_fullsized":"2024-09-13T14:51:58.282113+0000","mapping_epoch":223,"log_start":"234'5400","ondisk_log_start":"234'5400","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7345,"log_dups_size":0,"ondisk_log_size":7345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:26:49.717826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154075136,"num_objects":2351,"num_object_clones":0,"num_object_copies":7053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12745,"num_write_kb":483072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":405,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.240013+0000","last_change":"2024-09-13T14:51:01.996634+0000","last_active":"2024-09-13T14:51:54.240013+0000","last_peered":"2024-09-13T14:51:54.240013+0000","last_clean":"2024-09-13T14:51:54.240013+0000","last_became_active":"2024-09-13T14:51:01.996264+0000","last_became_peered":"2024-09-13T14:51:01.996264+0000","last_unstale":"2024-09-13T14:51:54.240013+0000","last_undegraded":"2024-09-13T14:51:54.240013+0000","last_fullsized":"2024-09-13T14:51:54.240013+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:30.083115+0000","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":382,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.242732+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:51:55.242732+0000","last_peered":"2024-09-13T14:51:55.242732+0000","last_clean":"2024-09-13T14:51:55.242732+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:51:55.242732+0000","last_undegraded":"2024-09-13T14:51:55.242732+0000","last_fullsized":"2024-09-13T14:51:55.242732+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":242,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.253115+0000","last_change":"2024-09-13T14:48:32.196993+0000","last_active":"2024-09-13T14:51:54.253115+0000","last_peered":"2024-09-13T14:51:54.253115+0000","last_clean":"2024-09-13T14:51:54.253115+0000","last_became_active":"2024-09-13T14:48:32.196681+0000","last_became_peered":"2024-09-13T14:48:32.196681+0000","last_unstale":"2024-09-13T14:51:54.253115+0000","last_undegraded":"2024-09-13T14:51:54.253115+0000","last_fullsized":"2024-09-13T14:51:54.253115+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:56.040494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"257'12522","reported_seq":12602,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.141458+0000","last_change":"2024-09-13T14:48:30.197998+0000","last_active":"2024-09-13T14:51:55.141458+0000","last_peered":"2024-09-13T14:51:55.141458+0000","last_clean":"2024-09-13T14:51:55.141458+0000","last_became_active":"2024-09-13T14:48:30.197697+0000","last_became_peered":"2024-09-13T14:48:30.197697+0000","last_unstale":"2024-09-13T14:51:55.141458+0000","last_undegraded":"2024-09-13T14:51:55.141458+0000","last_fullsized":"2024-09-13T14:51:55.141458+0000","mapping_epoch":223,"log_start":"233'3885","ondisk_log_start":"233'3885","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8637,"log_dups_size":0,"ondisk_log_size":8637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:17.417819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162660352,"num_objects":2482,"num_object_clones":0,"num_object_copies":7446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12522,"num_write_kb":480128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":165,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.254647+0000","last_change":"2024-09-13T14:40:27.868402+0000","last_active":"2024-09-13T14:51:54.254647+0000","last_peered":"2024-09-13T14:51:54.254647+0000","last_clean":"2024-09-13T14:51:54.254647+0000","last_became_active":"2024-09-13T14:40:22.966301+0000","last_became_peered":"2024-09-13T14:40:22.966301+0000","last_unstale":"2024-09-13T14:51:54.254647+0000","last_undegraded":"2024-09-13T14:51:54.254647+0000","last_fullsized":"2024-09-13T14:51:54.254647+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:18.660186+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":42,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.256045+0000","last_change":"2024-09-13T14:48:32.200521+0000","last_active":"2024-09-13T14:51:54.256045+0000","last_peered":"2024-09-13T14:51:54.256045+0000","last_clean":"2024-09-13T14:51:54.256045+0000","last_became_active":"2024-09-13T14:48:32.200256+0000","last_became_peered":"2024-09-13T14:48:32.200256+0000","last_unstale":"2024-09-13T14:51:54.256045+0000","last_undegraded":"2024-09-13T14:51:54.256045+0000","last_fullsized":"2024-09-13T14:51:54.256045+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:38.899629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":239,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.253192+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:51:54.253192+0000","last_peered":"2024-09-13T14:51:54.253192+0000","last_clean":"2024-09-13T14:51:54.253192+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:51:54.253192+0000","last_undegraded":"2024-09-13T14:51:54.253192+0000","last_fullsized":"2024-09-13T14:51:54.253192+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"258'12611","reported_seq":12653,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:56.604167+0000","last_change":"2024-09-13T14:48:32.204769+0000","last_active":"2024-09-13T14:51:56.604167+0000","last_peered":"2024-09-13T14:51:56.604167+0000","last_clean":"2024-09-13T14:51:56.604167+0000","last_became_active":"2024-09-13T14:48:32.204464+0000","last_became_peered":"2024-09-13T14:48:32.204464+0000","last_unstale":"2024-09-13T14:51:56.604167+0000","last_undegraded":"2024-09-13T14:51:56.604167+0000","last_fullsized":"2024-09-13T14:51:56.604167+0000","mapping_epoch":225,"log_start":"233'4500","ondisk_log_start":"233'4500","created":223,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8111,"log_dups_size":0,"ondisk_log_size":8111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:46.636678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159383576,"num_objects":2433,"num_object_clones":0,"num_object_copies":7299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2433,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12611,"num_write_kb":481345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":365,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.245467+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:51:55.245467+0000","last_peered":"2024-09-13T14:51:55.245467+0000","last_clean":"2024-09-13T14:51:55.245467+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:51:55.245467+0000","last_undegraded":"2024-09-13T14:51:55.245467+0000","last_fullsized":"2024-09-13T14:51:55.245467+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":242,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.253108+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T14:51:54.253108+0000","last_peered":"2024-09-13T14:51:54.253108+0000","last_clean":"2024-09-13T14:51:54.253108+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T14:51:54.253108+0000","last_undegraded":"2024-09-13T14:51:54.253108+0000","last_fullsized":"2024-09-13T14:51:54.253108+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"258'12545","reported_seq":19905,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:56.627736+0000","last_change":"2024-09-13T14:51:46.210926+0000","last_active":"2024-09-13T14:51:56.627736+0000","last_peered":"2024-09-13T14:51:56.627736+0000","last_clean":"2024-09-13T14:51:56.627736+0000","last_became_active":"2024-09-13T14:51:46.187150+0000","last_became_peered":"2024-09-13T14:51:46.187150+0000","last_unstale":"2024-09-13T14:51:56.627736+0000","last_undegraded":"2024-09-13T14:51:56.627736+0000","last_fullsized":"2024-09-13T14:51:56.627736+0000","mapping_epoch":254,"log_start":"233'4400","ondisk_log_start":"233'4400","created":223,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8145,"log_dups_size":0,"ondisk_log_size":8145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:35:28.550999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156303360,"num_objects":2385,"num_object_clones":0,"num_object_copies":7155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12545,"num_write_kb":477760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3647,"num_bytes_recovered":239009792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":168,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.254731+0000","last_change":"2024-09-13T14:48:32.205288+0000","last_active":"2024-09-13T14:51:54.254731+0000","last_peered":"2024-09-13T14:51:54.254731+0000","last_clean":"2024-09-13T14:51:54.254731+0000","last_became_active":"2024-09-13T14:48:32.204657+0000","last_became_peered":"2024-09-13T14:48:32.204657+0000","last_unstale":"2024-09-13T14:51:54.254731+0000","last_undegraded":"2024-09-13T14:51:54.254731+0000","last_fullsized":"2024-09-13T14:51:54.254731+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:52.513329+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":38,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.241241+0000","last_change":"2024-09-13T14:51:11.059644+0000","last_active":"2024-09-13T14:51:55.241241+0000","last_peered":"2024-09-13T14:51:55.241241+0000","last_clean":"2024-09-13T14:51:55.241241+0000","last_became_active":"2024-09-13T14:51:11.058874+0000","last_became_peered":"2024-09-13T14:51:11.058874+0000","last_unstale":"2024-09-13T14:51:55.241241+0000","last_undegraded":"2024-09-13T14:51:55.241241+0000","last_fullsized":"2024-09-13T14:51:55.241241+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:15:14.736476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":366,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.242911+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:51:55.242911+0000","last_peered":"2024-09-13T14:51:55.242911+0000","last_clean":"2024-09-13T14:51:55.242911+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:51:55.242911+0000","last_undegraded":"2024-09-13T14:51:55.242911+0000","last_fullsized":"2024-09-13T14:51:55.242911+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"258'12866","reported_seq":12952,"reported_epoch":258,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:51:58.279447+0000","last_change":"2024-09-13T14:51:12.318778+0000","last_active":"2024-09-13T14:51:58.279447+0000","last_peered":"2024-09-13T14:51:58.279447+0000","last_clean":"2024-09-13T14:51:10.011417+0000","last_became_active":"2024-09-13T14:51:12.271357+0000","last_became_peered":"2024-09-13T14:51:12.271357+0000","last_unstale":"2024-09-13T14:51:58.279447+0000","last_undegraded":"2024-09-13T14:51:58.279447+0000","last_fullsized":"2024-09-13T14:51:58.279447+0000","mapping_epoch":245,"log_start":"234'5500","ondisk_log_start":"234'5500","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7366,"log_dups_size":0,"ondisk_log_size":7366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150470656,"num_objects":2296,"num_object_clones":0,"num_object_copies":6888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2296,"num_objects_unfound":0,"num_objects_dirty":2296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12866,"num_write_kb":485184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":2296}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":386,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.243004+0000","last_change":"2024-09-13T14:48:32.204164+0000","last_active":"2024-09-13T14:51:54.243004+0000","last_peered":"2024-09-13T14:51:54.243004+0000","last_clean":"2024-09-13T14:51:54.243004+0000","last_became_active":"2024-09-13T14:48:32.203815+0000","last_became_peered":"2024-09-13T14:48:32.203815+0000","last_unstale":"2024-09-13T14:51:54.243004+0000","last_undegraded":"2024-09-13T14:51:54.243004+0000","last_fullsized":"2024-09-13T14:51:54.243004+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:22.639760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:22.639760+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:21.985178+0000","scrub_duration":0.0051120610000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":784,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.254939+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:51:54.254939+0000","last_peered":"2024-09-13T14:51:54.254939+0000","last_clean":"2024-09-13T14:51:54.254939+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:51:54.254939+0000","last_undegraded":"2024-09-13T14:51:54.254939+0000","last_fullsized":"2024-09-13T14:51:54.254939+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":38,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.241725+0000","last_change":"2024-09-13T14:51:11.059953+0000","last_active":"2024-09-13T14:51:55.241725+0000","last_peered":"2024-09-13T14:51:55.241725+0000","last_clean":"2024-09-13T14:51:55.241725+0000","last_became_active":"2024-09-13T14:51:11.059396+0000","last_became_peered":"2024-09-13T14:51:11.059396+0000","last_unstale":"2024-09-13T14:51:55.241725+0000","last_undegraded":"2024-09-13T14:51:55.241725+0000","last_fullsized":"2024-09-13T14:51:55.241725+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:11:51.119142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":42,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.256272+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T14:51:54.256272+0000","last_peered":"2024-09-13T14:51:54.256272+0000","last_clean":"2024-09-13T14:51:54.256272+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T14:51:54.256272+0000","last_undegraded":"2024-09-13T14:51:54.256272+0000","last_fullsized":"2024-09-13T14:51:54.256272+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"258'12746","reported_seq":12789,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:56.856889+0000","last_change":"2024-09-13T14:48:30.197655+0000","last_active":"2024-09-13T14:51:56.856889+0000","last_peered":"2024-09-13T14:51:56.856889+0000","last_clean":"2024-09-13T14:51:56.856889+0000","last_became_active":"2024-09-13T14:48:30.197442+0000","last_became_peered":"2024-09-13T14:48:30.197442+0000","last_unstale":"2024-09-13T14:51:56.856889+0000","last_undegraded":"2024-09-13T14:51:56.856889+0000","last_fullsized":"2024-09-13T14:51:56.856889+0000","mapping_epoch":223,"log_start":"233'5032","ondisk_log_start":"233'5032","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7714,"log_dups_size":0,"ondisk_log_size":7714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:44.452905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157679616,"num_objects":2406,"num_object_clones":0,"num_object_copies":7218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12746,"num_write_kb":484864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":505,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.243017+0000","last_change":"2024-09-13T14:48:32.204587+0000","last_active":"2024-09-13T14:51:55.243017+0000","last_peered":"2024-09-13T14:51:55.243017+0000","last_clean":"2024-09-13T14:51:55.243017+0000","last_became_active":"2024-09-13T14:48:32.204153+0000","last_became_peered":"2024-09-13T14:48:32.204153+0000","last_unstale":"2024-09-13T14:51:55.243017+0000","last_undegraded":"2024-09-13T14:51:55.243017+0000","last_fullsized":"2024-09-13T14:51:55.243017+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.784263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.784263+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:11:12.651038+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":469,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.242955+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:51:54.242955+0000","last_peered":"2024-09-13T14:51:54.242955+0000","last_clean":"2024-09-13T14:51:54.242955+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:51:54.242955+0000","last_undegraded":"2024-09-13T14:51:54.242955+0000","last_fullsized":"2024-09-13T14:51:54.242955+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":400,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.245527+0000","last_change":"2024-09-13T14:51:11.061677+0000","last_active":"2024-09-13T14:51:55.245527+0000","last_peered":"2024-09-13T14:51:55.245527+0000","last_clean":"2024-09-13T14:51:55.245527+0000","last_became_active":"2024-09-13T14:51:11.061205+0000","last_became_peered":"2024-09-13T14:51:11.061205+0000","last_unstale":"2024-09-13T14:51:55.245527+0000","last_undegraded":"2024-09-13T14:51:55.245527+0000","last_fullsized":"2024-09-13T14:51:55.245527+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:38.521784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":404,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.246233+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T14:51:55.246233+0000","last_peered":"2024-09-13T14:51:55.246233+0000","last_clean":"2024-09-13T14:51:55.246233+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T14:51:55.246233+0000","last_undegraded":"2024-09-13T14:51:55.246233+0000","last_fullsized":"2024-09-13T14:51:55.246233+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"258'12828","reported_seq":12910,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:58.292034+0000","last_change":"2024-09-13T14:48:30.196036+0000","last_active":"2024-09-13T14:51:58.292034+0000","last_peered":"2024-09-13T14:51:58.292034+0000","last_clean":"2024-09-13T14:51:58.292034+0000","last_became_active":"2024-09-13T14:48:30.195478+0000","last_became_peered":"2024-09-13T14:48:30.195478+0000","last_unstale":"2024-09-13T14:51:58.292034+0000","last_undegraded":"2024-09-13T14:51:58.292034+0000","last_fullsized":"2024-09-13T14:51:58.292034+0000","mapping_epoch":223,"log_start":"234'5500","ondisk_log_start":"234'5500","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7328,"log_dups_size":0,"ondisk_log_size":7328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:21:44.624753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153354240,"num_objects":2340,"num_object_clones":0,"num_object_copies":7020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12828,"num_write_kb":485376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":15,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.240171+0000","last_change":"2024-09-13T14:51:44.190446+0000","last_active":"2024-09-13T14:51:54.240171+0000","last_peered":"2024-09-13T14:51:54.240171+0000","last_clean":"2024-09-13T14:51:54.240171+0000","last_became_active":"2024-09-13T14:51:44.190115+0000","last_became_peered":"2024-09-13T14:51:44.190115+0000","last_unstale":"2024-09-13T14:51:54.240171+0000","last_undegraded":"2024-09-13T14:51:54.240171+0000","last_fullsized":"2024-09-13T14:51:54.240171+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:45:49.425578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:45:49.425578+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:43.724060+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":850,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.248480+0000","last_change":"2024-09-13T14:51:11.072066+0000","last_active":"2024-09-13T14:51:54.248480+0000","last_peered":"2024-09-13T14:51:54.248480+0000","last_clean":"2024-09-13T14:51:54.248480+0000","last_became_active":"2024-09-13T14:51:11.061307+0000","last_became_peered":"2024-09-13T14:51:11.061307+0000","last_unstale":"2024-09-13T14:51:54.248480+0000","last_undegraded":"2024-09-13T14:51:54.248480+0000","last_fullsized":"2024-09-13T14:51:54.248480+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:06.774106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":381,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.241449+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T14:51:55.241449+0000","last_peered":"2024-09-13T14:51:55.241449+0000","last_clean":"2024-09-13T14:51:55.241449+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T14:51:55.241449+0000","last_undegraded":"2024-09-13T14:51:55.241449+0000","last_fullsized":"2024-09-13T14:51:55.241449+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":399,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.242827+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:51:55.242827+0000","last_peered":"2024-09-13T14:51:55.242827+0000","last_clean":"2024-09-13T14:51:55.242827+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:51:55.242827+0000","last_undegraded":"2024-09-13T14:51:55.242827+0000","last_fullsized":"2024-09-13T14:51:55.242827+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"258'12688","reported_seq":12741,"reported_epoch":258,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:51:56.618332+0000","last_change":"2024-09-13T14:51:12.278325+0000","last_active":"2024-09-13T14:51:56.618332+0000","last_peered":"2024-09-13T14:51:56.618332+0000","last_clean":"2024-09-13T14:51:09.999835+0000","last_became_active":"2024-09-13T14:51:12.256335+0000","last_became_peered":"2024-09-13T14:51:12.256335+0000","last_unstale":"2024-09-13T14:51:56.618332+0000","last_undegraded":"2024-09-13T14:51:56.618332+0000","last_fullsized":"2024-09-13T14:51:56.618332+0000","mapping_epoch":245,"log_start":"233'4500","ondisk_log_start":"233'4500","created":223,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8188,"log_dups_size":0,"ondisk_log_size":8188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:40:16.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160169984,"num_objects":2444,"num_object_clones":0,"num_object_copies":7332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2444,"num_objects_unfound":0,"num_objects_dirty":2444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12688,"num_write_kb":484224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,1],"avail_no_missing":["6","0","1"],"object_location_counts":[{"shards":"0,1,6","objects":2444}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":158,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.254678+0000","last_change":"2024-09-13T14:40:26.896691+0000","last_active":"2024-09-13T14:51:54.254678+0000","last_peered":"2024-09-13T14:51:54.254678+0000","last_clean":"2024-09-13T14:51:54.254678+0000","last_became_active":"2024-09-13T14:40:22.949772+0000","last_became_peered":"2024-09-13T14:40:22.949772+0000","last_unstale":"2024-09-13T14:51:54.254678+0000","last_undegraded":"2024-09-13T14:51:54.254678+0000","last_fullsized":"2024-09-13T14:51:54.254678+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:26.896613+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:26.896613+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:01.615433+0000","scrub_duration":0.004120828,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":884,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.254886+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T14:51:54.254886+0000","last_peered":"2024-09-13T14:51:54.254886+0000","last_clean":"2024-09-13T14:51:54.254886+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T14:51:54.254886+0000","last_undegraded":"2024-09-13T14:51:54.254886+0000","last_fullsized":"2024-09-13T14:51:54.254886+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":243,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.253108+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T14:51:54.253108+0000","last_peered":"2024-09-13T14:51:54.253108+0000","last_clean":"2024-09-13T14:51:54.253108+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T14:51:54.253108+0000","last_undegraded":"2024-09-13T14:51:54.253108+0000","last_fullsized":"2024-09-13T14:51:54.253108+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":399,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.242113+0000","last_change":"2024-09-13T14:51:11.060994+0000","last_active":"2024-09-13T14:51:55.242113+0000","last_peered":"2024-09-13T14:51:55.242113+0000","last_clean":"2024-09-13T14:51:55.242113+0000","last_became_active":"2024-09-13T14:51:11.060666+0000","last_became_peered":"2024-09-13T14:51:11.060666+0000","last_unstale":"2024-09-13T14:51:55.242113+0000","last_undegraded":"2024-09-13T14:51:55.242113+0000","last_fullsized":"2024-09-13T14:51:55.242113+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:40:15.828766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"258'12477","reported_seq":15851,"reported_epoch":258,"state":"active+remapped+backfilling","last_fresh":"2024-09-13T14:51:56.858121+0000","last_change":"2024-09-13T14:51:45.011068+0000","last_active":"2024-09-13T14:51:56.858121+0000","last_peered":"2024-09-13T14:51:56.858121+0000","last_clean":"2024-09-13T14:51:09.961543+0000","last_became_active":"2024-09-13T14:51:12.277498+0000","last_became_peered":"2024-09-13T14:51:12.277498+0000","last_unstale":"2024-09-13T14:51:56.858121+0000","last_undegraded":"2024-09-13T14:51:56.858121+0000","last_fullsized":"2024-09-13T14:51:56.858121+0000","mapping_epoch":245,"log_start":"233'4100","ondisk_log_start":"233'4100","created":223,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8377,"log_dups_size":0,"ondisk_log_size":8377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:08.459300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156041216,"num_objects":2381,"num_object_clones":0,"num_object_copies":7143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":934,"num_objects_unfound":0,"num_objects_dirty":2381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12477,"num_write_kb":475456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1654,"num_bytes_recovered":108396544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6,1],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":2381}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":62,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.257007+0000","last_change":"2024-09-13T14:51:11.061598+0000","last_active":"2024-09-13T14:51:54.257007+0000","last_peered":"2024-09-13T14:51:54.257007+0000","last_clean":"2024-09-13T14:51:54.257007+0000","last_became_active":"2024-09-13T14:51:11.061164+0000","last_became_peered":"2024-09-13T14:51:11.061164+0000","last_unstale":"2024-09-13T14:51:54.257007+0000","last_undegraded":"2024-09-13T14:51:54.257007+0000","last_fullsized":"2024-09-13T14:51:54.257007+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:48:34.095505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:48:34.095505+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:03.076025+0000","scrub_duration":0.001382681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":239,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.255182+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:51:54.255182+0000","last_peered":"2024-09-13T14:51:54.255182+0000","last_clean":"2024-09-13T14:51:54.255182+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:51:54.255182+0000","last_undegraded":"2024-09-13T14:51:54.255182+0000","last_fullsized":"2024-09-13T14:51:54.255182+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":884,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.242297+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:51:55.242297+0000","last_peered":"2024-09-13T14:51:55.242297+0000","last_clean":"2024-09-13T14:51:55.242297+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:51:55.242297+0000","last_undegraded":"2024-09-13T14:51:55.242297+0000","last_fullsized":"2024-09-13T14:51:55.242297+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":356,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.242997+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:51:55.242997+0000","last_peered":"2024-09-13T14:51:55.242997+0000","last_clean":"2024-09-13T14:51:55.242997+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:51:55.242997+0000","last_undegraded":"2024-09-13T14:51:55.242997+0000","last_fullsized":"2024-09-13T14:51:55.242997+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"258'12349","reported_seq":12392,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:56.963145+0000","last_change":"2024-09-13T14:48:30.192664+0000","last_active":"2024-09-13T14:51:56.963145+0000","last_peered":"2024-09-13T14:51:56.963145+0000","last_clean":"2024-09-13T14:51:56.963145+0000","last_became_active":"2024-09-13T14:48:30.192434+0000","last_became_peered":"2024-09-13T14:48:30.192434+0000","last_unstale":"2024-09-13T14:51:56.963145+0000","last_undegraded":"2024-09-13T14:51:56.963145+0000","last_fullsized":"2024-09-13T14:51:56.963145+0000","mapping_epoch":223,"log_start":"232'2600","ondisk_log_start":"232'2600","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":9749,"log_dups_size":0,"ondisk_log_size":9749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:58.920518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157220864,"num_objects":2399,"num_object_clones":0,"num_object_copies":7197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12349,"num_write_kb":471936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":52,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.256268+0000","last_change":"2024-09-13T14:51:11.060132+0000","last_active":"2024-09-13T14:51:54.256268+0000","last_peered":"2024-09-13T14:51:54.256268+0000","last_clean":"2024-09-13T14:51:54.256268+0000","last_became_active":"2024-09-13T14:51:11.059785+0000","last_became_peered":"2024-09-13T14:51:11.059785+0000","last_unstale":"2024-09-13T14:51:54.256268+0000","last_undegraded":"2024-09-13T14:51:54.256268+0000","last_fullsized":"2024-09-13T14:51:54.256268+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.334379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.334379+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:44.298431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":381,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.242207+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T14:51:55.242207+0000","last_peered":"2024-09-13T14:51:55.242207+0000","last_clean":"2024-09-13T14:51:55.242207+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T14:51:55.242207+0000","last_undegraded":"2024-09-13T14:51:55.242207+0000","last_fullsized":"2024-09-13T14:51:55.242207+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":471,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.240484+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:51:54.240484+0000","last_peered":"2024-09-13T14:51:54.240484+0000","last_clean":"2024-09-13T14:51:54.240484+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:51:54.240484+0000","last_undegraded":"2024-09-13T14:51:54.240484+0000","last_fullsized":"2024-09-13T14:51:54.240484+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":368,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.245779+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:51:55.245779+0000","last_peered":"2024-09-13T14:51:55.245779+0000","last_clean":"2024-09-13T14:51:55.245779+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:51:55.245779+0000","last_undegraded":"2024-09-13T14:51:55.245779+0000","last_fullsized":"2024-09-13T14:51:55.245779+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"258'12901","reported_seq":12983,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:58.136448+0000","last_change":"2024-09-13T14:48:30.197226+0000","last_active":"2024-09-13T14:51:58.136448+0000","last_peered":"2024-09-13T14:51:58.136448+0000","last_clean":"2024-09-13T14:51:58.136448+0000","last_became_active":"2024-09-13T14:48:30.195243+0000","last_became_peered":"2024-09-13T14:48:30.195243+0000","last_unstale":"2024-09-13T14:51:58.136448+0000","last_undegraded":"2024-09-13T14:51:58.136448+0000","last_fullsized":"2024-09-13T14:51:58.136448+0000","mapping_epoch":223,"log_start":"234'5917","ondisk_log_start":"234'5917","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:46:06.507375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150798336,"num_objects":2301,"num_object_clones":0,"num_object_copies":6903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12901,"num_write_kb":486464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":189,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.252857+0000","last_change":"2024-09-13T14:51:01.998101+0000","last_active":"2024-09-13T14:51:54.252857+0000","last_peered":"2024-09-13T14:51:54.252857+0000","last_clean":"2024-09-13T14:51:54.252857+0000","last_became_active":"2024-09-13T14:51:01.997688+0000","last_became_peered":"2024-09-13T14:51:01.997688+0000","last_unstale":"2024-09-13T14:51:54.252857+0000","last_undegraded":"2024-09-13T14:51:54.252857+0000","last_fullsized":"2024-09-13T14:51:54.252857+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:40:30.500046+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:02:51.444049+0000","scrub_duration":0.00248832,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":382,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.242978+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:51:55.242978+0000","last_peered":"2024-09-13T14:51:55.242978+0000","last_clean":"2024-09-13T14:51:55.242978+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:51:55.242978+0000","last_undegraded":"2024-09-13T14:51:55.242978+0000","last_fullsized":"2024-09-13T14:51:55.242978+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":738,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.242462+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T14:51:55.242462+0000","last_peered":"2024-09-13T14:51:55.242462+0000","last_clean":"2024-09-13T14:51:55.242462+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T14:51:55.242462+0000","last_undegraded":"2024-09-13T14:51:55.242462+0000","last_fullsized":"2024-09-13T14:51:55.242462+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":45,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.241359+0000","last_change":"2024-09-13T14:51:44.203186+0000","last_active":"2024-09-13T14:51:55.241359+0000","last_peered":"2024-09-13T14:51:55.241359+0000","last_clean":"2024-09-13T14:51:55.241359+0000","last_became_active":"2024-09-13T14:51:44.202851+0000","last_became_peered":"2024-09-13T14:51:44.202851+0000","last_unstale":"2024-09-13T14:51:55.241359+0000","last_undegraded":"2024-09-13T14:51:55.241359+0000","last_fullsized":"2024-09-13T14:51:55.241359+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:05:49.173340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"258'12671","reported_seq":12753,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:58.124487+0000","last_change":"2024-09-13T14:48:30.195027+0000","last_active":"2024-09-13T14:51:58.124487+0000","last_peered":"2024-09-13T14:51:58.124487+0000","last_clean":"2024-09-13T14:51:58.124487+0000","last_became_active":"2024-09-13T14:48:30.194672+0000","last_became_peered":"2024-09-13T14:48:30.194672+0000","last_unstale":"2024-09-13T14:51:58.124487+0000","last_undegraded":"2024-09-13T14:51:58.124487+0000","last_fullsized":"2024-09-13T14:51:58.124487+0000","mapping_epoch":223,"log_start":"234'5667","ondisk_log_start":"234'5667","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7004,"log_dups_size":0,"ondisk_log_size":7004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:38:50.182084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154075136,"num_objects":2351,"num_object_clones":0,"num_object_copies":7053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12671,"num_write_kb":480704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":312,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.241958+0000","last_change":"2024-09-13T14:40:24.124750+0000","last_active":"2024-09-13T14:51:55.241958+0000","last_peered":"2024-09-13T14:51:55.241958+0000","last_clean":"2024-09-13T14:51:55.241958+0000","last_became_active":"2024-09-13T14:40:22.958820+0000","last_became_peered":"2024-09-13T14:40:22.958820+0000","last_unstale":"2024-09-13T14:51:55.241958+0000","last_undegraded":"2024-09-13T14:51:55.241958+0000","last_fullsized":"2024-09-13T14:51:55.241958+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:24.124531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:24.124531+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:47:30.267886+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":390,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.246028+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:51:55.246028+0000","last_peered":"2024-09-13T14:51:55.246028+0000","last_clean":"2024-09-13T14:51:55.246028+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:51:55.246028+0000","last_undegraded":"2024-09-13T14:51:55.246028+0000","last_fullsized":"2024-09-13T14:51:55.246028+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1071,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.260452+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T14:51:54.260452+0000","last_peered":"2024-09-13T14:51:54.260452+0000","last_clean":"2024-09-13T14:51:54.260452+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T14:51:54.260452+0000","last_undegraded":"2024-09-13T14:51:54.260452+0000","last_fullsized":"2024-09-13T14:51:54.260452+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":225,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.252640+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T14:51:54.252640+0000","last_peered":"2024-09-13T14:51:54.252640+0000","last_clean":"2024-09-13T14:51:54.252640+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T14:51:54.252640+0000","last_undegraded":"2024-09-13T14:51:54.252640+0000","last_fullsized":"2024-09-13T14:51:54.252640+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":242,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.253033+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T14:51:54.253033+0000","last_peered":"2024-09-13T14:51:54.253033+0000","last_clean":"2024-09-13T14:51:54.253033+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T14:51:54.253033+0000","last_undegraded":"2024-09-13T14:51:54.253033+0000","last_fullsized":"2024-09-13T14:51:54.253033+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":397,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.245628+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T14:51:55.245628+0000","last_peered":"2024-09-13T14:51:55.245628+0000","last_clean":"2024-09-13T14:51:55.245628+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T14:51:55.245628+0000","last_undegraded":"2024-09-13T14:51:55.245628+0000","last_fullsized":"2024-09-13T14:51:55.245628+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":380,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.241627+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T14:51:55.241627+0000","last_peered":"2024-09-13T14:51:55.241627+0000","last_clean":"2024-09-13T14:51:55.241627+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T14:51:55.241627+0000","last_undegraded":"2024-09-13T14:51:55.241627+0000","last_fullsized":"2024-09-13T14:51:55.241627+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":238,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.240374+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:51:54.240374+0000","last_peered":"2024-09-13T14:51:54.240374+0000","last_clean":"2024-09-13T14:51:54.240374+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:51:54.240374+0000","last_undegraded":"2024-09-13T14:51:54.240374+0000","last_fullsized":"2024-09-13T14:51:54.240374+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":222,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.257999+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:51:54.257999+0000","last_peered":"2024-09-13T14:51:54.257999+0000","last_clean":"2024-09-13T14:51:54.257999+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:51:54.257999+0000","last_undegraded":"2024-09-13T14:51:54.257999+0000","last_fullsized":"2024-09-13T14:51:54.257999+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":349,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.241808+0000","last_change":"2024-09-13T14:48:32.200953+0000","last_active":"2024-09-13T14:51:55.241808+0000","last_peered":"2024-09-13T14:51:55.241808+0000","last_clean":"2024-09-13T14:51:55.241808+0000","last_became_active":"2024-09-13T14:48:32.200634+0000","last_became_peered":"2024-09-13T14:48:32.200634+0000","last_unstale":"2024-09-13T14:51:55.241808+0000","last_undegraded":"2024-09-13T14:51:55.241808+0000","last_fullsized":"2024-09-13T14:51:55.241808+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:18.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":377,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.246030+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:51:55.246030+0000","last_peered":"2024-09-13T14:51:55.246030+0000","last_clean":"2024-09-13T14:51:55.246030+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:51:55.246030+0000","last_undegraded":"2024-09-13T14:51:55.246030+0000","last_fullsized":"2024-09-13T14:51:55.246030+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":398,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.241748+0000","last_change":"2024-09-13T14:48:32.201812+0000","last_active":"2024-09-13T14:51:55.241748+0000","last_peered":"2024-09-13T14:51:55.241748+0000","last_clean":"2024-09-13T14:51:55.241748+0000","last_became_active":"2024-09-13T14:48:32.201486+0000","last_became_peered":"2024-09-13T14:48:32.201486+0000","last_unstale":"2024-09-13T14:51:55.241748+0000","last_undegraded":"2024-09-13T14:51:55.241748+0000","last_fullsized":"2024-09-13T14:51:55.241748+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:23:29.693906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":42,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.257075+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T14:51:54.257075+0000","last_peered":"2024-09-13T14:51:54.257075+0000","last_clean":"2024-09-13T14:51:54.257075+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T14:51:54.257075+0000","last_undegraded":"2024-09-13T14:51:54.257075+0000","last_fullsized":"2024-09-13T14:51:54.257075+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":402,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.241570+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:51:55.241570+0000","last_peered":"2024-09-13T14:51:55.241570+0000","last_clean":"2024-09-13T14:51:55.241570+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:51:55.241570+0000","last_undegraded":"2024-09-13T14:51:55.241570+0000","last_fullsized":"2024-09-13T14:51:55.241570+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":401,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.246230+0000","last_change":"2024-09-13T14:51:44.198771+0000","last_active":"2024-09-13T14:51:55.246230+0000","last_peered":"2024-09-13T14:51:55.246230+0000","last_clean":"2024-09-13T14:51:55.246230+0000","last_became_active":"2024-09-13T14:51:44.198466+0000","last_became_peered":"2024-09-13T14:51:44.198466+0000","last_unstale":"2024-09-13T14:51:55.246230+0000","last_undegraded":"2024-09-13T14:51:55.246230+0000","last_fullsized":"2024-09-13T14:51:55.246230+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:41:16.099954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":366,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.244281+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:51:55.244281+0000","last_peered":"2024-09-13T14:51:55.244281+0000","last_clean":"2024-09-13T14:51:55.244281+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:51:55.244281+0000","last_undegraded":"2024-09-13T14:51:55.244281+0000","last_fullsized":"2024-09-13T14:51:55.244281+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":398,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.242434+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:51:55.242434+0000","last_peered":"2024-09-13T14:51:55.242434+0000","last_clean":"2024-09-13T14:51:55.242434+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:51:55.242434+0000","last_undegraded":"2024-09-13T14:51:55.242434+0000","last_fullsized":"2024-09-13T14:51:55.242434+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":241,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.252673+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:51:54.252673+0000","last_peered":"2024-09-13T14:51:54.252673+0000","last_clean":"2024-09-13T14:51:54.252673+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:51:54.252673+0000","last_undegraded":"2024-09-13T14:51:54.252673+0000","last_fullsized":"2024-09-13T14:51:54.252673+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":40,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.254800+0000","last_change":"2024-09-13T14:51:02.029301+0000","last_active":"2024-09-13T14:51:54.254800+0000","last_peered":"2024-09-13T14:51:54.254800+0000","last_clean":"2024-09-13T14:51:54.254800+0000","last_became_active":"2024-09-13T14:51:02.029032+0000","last_became_peered":"2024-09-13T14:51:02.029032+0000","last_unstale":"2024-09-13T14:51:54.254800+0000","last_undegraded":"2024-09-13T14:51:54.254800+0000","last_fullsized":"2024-09-13T14:51:54.254800+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:40.150320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":370,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.245784+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:51:55.245784+0000","last_peered":"2024-09-13T14:51:55.245784+0000","last_clean":"2024-09-13T14:51:55.245784+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:51:55.245784+0000","last_undegraded":"2024-09-13T14:51:55.245784+0000","last_fullsized":"2024-09-13T14:51:55.245784+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":239,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.252961+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:51:54.252961+0000","last_peered":"2024-09-13T14:51:54.252961+0000","last_clean":"2024-09-13T14:51:54.252961+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:51:54.252961+0000","last_undegraded":"2024-09-13T14:51:54.252961+0000","last_fullsized":"2024-09-13T14:51:54.252961+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":224,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.254946+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:51:54.254946+0000","last_peered":"2024-09-13T14:51:54.254946+0000","last_clean":"2024-09-13T14:51:54.254946+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:51:54.254946+0000","last_undegraded":"2024-09-13T14:51:54.254946+0000","last_fullsized":"2024-09-13T14:51:54.254946+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":239,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.252599+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:51:54.252599+0000","last_peered":"2024-09-13T14:51:54.252599+0000","last_clean":"2024-09-13T14:51:54.252599+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:51:54.252599+0000","last_undegraded":"2024-09-13T14:51:54.252599+0000","last_fullsized":"2024-09-13T14:51:54.252599+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":263,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.255109+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T14:51:54.255109+0000","last_peered":"2024-09-13T14:51:54.255109+0000","last_clean":"2024-09-13T14:51:54.255109+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T14:51:54.255109+0000","last_undegraded":"2024-09-13T14:51:54.255109+0000","last_fullsized":"2024-09-13T14:51:54.255109+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":213,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.254681+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:51:54.254681+0000","last_peered":"2024-09-13T14:51:54.254681+0000","last_clean":"2024-09-13T14:51:54.254681+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:51:54.254681+0000","last_undegraded":"2024-09-13T14:51:54.254681+0000","last_fullsized":"2024-09-13T14:51:54.254681+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":405,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.242708+0000","last_change":"2024-09-13T14:48:32.202404+0000","last_active":"2024-09-13T14:51:55.242708+0000","last_peered":"2024-09-13T14:51:55.242708+0000","last_clean":"2024-09-13T14:51:55.242708+0000","last_became_active":"2024-09-13T14:48:32.202064+0000","last_became_peered":"2024-09-13T14:48:32.202064+0000","last_unstale":"2024-09-13T14:51:55.242708+0000","last_undegraded":"2024-09-13T14:51:55.242708+0000","last_fullsized":"2024-09-13T14:51:55.242708+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:25.674819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":245,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.252368+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T14:51:54.252368+0000","last_peered":"2024-09-13T14:51:54.252368+0000","last_clean":"2024-09-13T14:51:54.252368+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T14:51:54.252368+0000","last_undegraded":"2024-09-13T14:51:54.252368+0000","last_fullsized":"2024-09-13T14:51:54.252368+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":421,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-09-13T14:51:55.246004+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T14:51:55.246004+0000","last_peered":"2024-09-13T14:51:55.246004+0000","last_clean":"2024-09-13T14:51:55.246004+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T14:51:55.246004+0000","last_undegraded":"2024-09-13T14:51:55.246004+0000","last_fullsized":"2024-09-13T14:51:55.246004+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2489647128,"num_objects":37990,"num_object_clones":0,"num_object_copies":113970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5674,"num_objects_unfound":0,"num_objects_dirty":37990,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":202268,"num_write_kb":7688193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5301,"num_bytes_recovered":347406336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7579578368,"data_stored":7579566152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125731,"ondisk_log_size":125731,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":16172,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397816,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1070180,"kb_used_data":772820,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":92674972,"statfs":{"total":95995035648,"available":94899171328,"internally_reserved":0,"allocated":791367680,"data_stored":789635534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594045,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1085976,"kb_used_data":784464,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92659176,"statfs":{"total":95995035648,"available":94882996224,"internally_reserved":0,"allocated":803291136,"data_stored":801562682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790274,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1367120,"kb_used_data":781256,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":92378032,"statfs":{"total":95995035648,"available":94595104768,"internally_reserved":0,"allocated":800006144,"data_stored":798282299,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986503,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1372624,"kb_used_data":774408,"kb_used_omap":1,"kb_used_meta":598206,"kb_avail":92372528,"statfs":{"total":95995035648,"available":94589468672,"internally_reserved":0,"allocated":792993792,"data_stored":791269134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612563288},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598951,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1493508,"kb_used_data":1000316,"kb_used_omap":6,"kb_used_meta":493177,"kb_avail":92251644,"statfs":{"total":95995035648,"available":94465683456,"internally_reserved":0,"allocated":1024323584,"data_stored":1022611396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":505014240},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762477,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1551108,"kb_used_data":946744,"kb_used_omap":8,"kb_used_meta":604343,"kb_avail":92194044,"statfs":{"total":95995035648,"available":94406701056,"internally_reserved":0,"allocated":969465856,"data_stored":967749699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":618847690},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926004,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1783184,"kb_used_data":1230020,"kb_used_omap":7,"kb_used_meta":553144,"kb_avail":91961968,"statfs":{"total":95995035648,"available":94169055232,"internally_reserved":0,"allocated":1259540480,"data_stored":1257819843,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":566420169},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122244,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1729568,"kb_used_data":1221208,"kb_used_omap":7,"kb_used_meta":508344,"kb_avail":92015584,"statfs":{"total":95995035648,"available":94223958016,"internally_reserved":0,"allocated":1250516992,"data_stored":1248781282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":520544847},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1227558912,"data_stored":1227554840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1235357696,"data_stored":1235353624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":965345280,"data_stored":965345280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1013710848,"data_stored":1013710848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":786038784,"data_stored":786038784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":775749632,"data_stored":775749632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":789581824,"data_stored":789577752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":786235392,"data_stored":786235392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:52:02.611 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:52:02.642 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:02 smithi204 ceph-mon[87071]: from='client.55246 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:02.647 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:02.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.675 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:02.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:02.777 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:02.875 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:02.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:02.978 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:03.055 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:03.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:03.157 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:03.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.196 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:03.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:03.253 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:52:03.253 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T14:52:03.253 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T14:52:03.253 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:52:03.253 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.253 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:34135 6.3 6:c2de75f9:::benchmark_data_smithi096_167087_object34134:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:52:03.253 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.699103+0000", 2024-09-13T14:52:03.253 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 460.54999969400001, 2024-09-13T14:52:03.254 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19865819200000001, 2024-09-13T14:52:03.254 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.254 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:03.254 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:52:03.254 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:52:03.254 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:52:03.254 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 34135 2024-09-13T14:52:03.254 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.254 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.254 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.254 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.254 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699103+0000", 2024-09-13T14:52:03.255 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.255 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.255 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.255 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.255 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699103+0000", 2024-09-13T14:52:03.255 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.255 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.255 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.255 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.255 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699102+0000", 2024-09-13T14:52:03.255 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:52:03.256 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.256 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.256 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.256 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699154+0000", 2024-09-13T14:52:03.256 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2114000000000003e-05 2024-09-13T14:52:03.256 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.256 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.256 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.256 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699161+0000", 2024-09-13T14:52:03.256 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1899999999999998e-06 2024-09-13T14:52:03.257 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.257 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.257 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.257 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699165+0000", 2024-09-13T14:52:03.257 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.597e-06 2024-09-13T14:52:03.257 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.257 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.257 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.257 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699198+0000", 2024-09-13T14:52:03.257 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3531000000000001e-05 2024-09-13T14:52:03.257 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.257 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.257 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.258 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699264+0000", 2024-09-13T14:52:03.258 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5819000000000001e-05 2024-09-13T14:52:03.258 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.258 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.258 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:52:03.258 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699341+0000", 2024-09-13T14:52:03.258 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6790000000000002e-05 2024-09-13T14:52:03.258 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.258 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.258 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:52:03.258 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.701095+0000", 2024-09-13T14:52:03.258 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001754434 2024-09-13T14:52:03.259 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.259 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.259 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.259 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897558+0000", 2024-09-13T14:52:03.259 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19646306199999999 2024-09-13T14:52:03.259 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.259 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.259 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.259 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897605+0000", 2024-09-13T14:52:03.259 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7012000000000003e-05 2024-09-13T14:52:03.259 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.259 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.259 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:52:03.260 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897648+0000", 2024-09-13T14:52:03.260 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3058e-05 2024-09-13T14:52:03.260 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.260 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.260 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.260 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897761+0000", 2024-09-13T14:52:03.260 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011252999999999999 2024-09-13T14:52:03.260 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.260 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.260 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.260 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.261 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.261 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:34139 6.3 6:cd715ee8:::benchmark_data_smithi096_167087_object34138:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:52:03.261 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.699339+0000", 2024-09-13T14:52:03.261 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 460.54976344900001, 2024-09-13T14:52:03.261 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19895058299999999, 2024-09-13T14:52:03.261 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.261 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:03.261 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:52:03.261 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:52:03.261 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:52:03.261 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 34139 2024-09-13T14:52:03.261 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.261 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.262 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.262 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.262 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699339+0000", 2024-09-13T14:52:03.262 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.262 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.262 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.262 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.262 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699339+0000", 2024-09-13T14:52:03.262 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.262 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.262 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.262 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.262 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699338+0000", 2024-09-13T14:52:03.262 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:52:03.263 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.263 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.263 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.263 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699447+0000", 2024-09-13T14:52:03.263 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000109077 2024-09-13T14:52:03.263 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.263 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.263 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.263 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699456+0000", 2024-09-13T14:52:03.263 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3520000000000007e-06 2024-09-13T14:52:03.264 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.264 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.264 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.264 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.699460+0000", 2024-09-13T14:52:03.264 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6920000000000001e-06 2024-09-13T14:52:03.264 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.264 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.264 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.264 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.700283+0000", 2024-09-13T14:52:03.264 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000822742 2024-09-13T14:52:03.264 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.264 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.265 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.265 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.700351+0000", 2024-09-13T14:52:03.265 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8039000000000004e-05 2024-09-13T14:52:03.265 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.265 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.265 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:52:03.265 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.700433+0000", 2024-09-13T14:52:03.265 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2398999999999995e-05 2024-09-13T14:52:03.265 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.265 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.265 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:52:03.265 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.701174+0000", 2024-09-13T14:52:03.266 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00074037599999999995 2024-09-13T14:52:03.266 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.266 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.266 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.266 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897584+0000", 2024-09-13T14:52:03.266 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19641035100000001 2024-09-13T14:52:03.266 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.266 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.266 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.266 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.898214+0000", 2024-09-13T14:52:03.266 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00063014600000000005 2024-09-13T14:52:03.266 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.266 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.266 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:52:03.267 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.898255+0000", 2024-09-13T14:52:03.267 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1083e-05 2024-09-13T14:52:03.267 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.267 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.267 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.267 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.898289+0000", 2024-09-13T14:52:03.267 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4069000000000002e-05 2024-09-13T14:52:03.267 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.267 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.267 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.268 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.268 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.268 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:34136 6.c e150/145 6:3e63118a:::benchmark_data_smithi096_167087_object34135:head v 150'2081, mlcod=150'2079)", 2024-09-13T14:52:03.268 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.703607+0000", 2024-09-13T14:52:03.268 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 460.54549516200001, 2024-09-13T14:52:03.268 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19718830400000001, 2024-09-13T14:52:03.268 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.268 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:03.268 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.268 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.268 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.268 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703607+0000", 2024-09-13T14:52:03.268 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.269 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.269 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.269 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.269 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703607+0000", 2024-09-13T14:52:03.269 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.269 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.269 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.269 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.269 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703606+0000", 2024-09-13T14:52:03.269 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:03.270 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.270 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.270 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.270 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703652+0000", 2024-09-13T14:52:03.270 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6789000000000002e-05 2024-09-13T14:52:03.270 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.270 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.270 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.270 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703674+0000", 2024-09-13T14:52:03.270 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1297e-05 2024-09-13T14:52:03.271 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.271 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.271 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.271 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.703683+0000", 2024-09-13T14:52:03.271 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5559999999999996e-06 2024-09-13T14:52:03.271 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.271 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.271 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.271 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897235+0000", 2024-09-13T14:52:03.271 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19355149599999999 2024-09-13T14:52:03.271 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.271 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.271 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.271 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897275+0000", 2024-09-13T14:52:03.272 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0275000000000003e-05 2024-09-13T14:52:03.272 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.272 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.272 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:52:03.272 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.900770+0000", 2024-09-13T14:52:03.272 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0034945810000000001 2024-09-13T14:52:03.272 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.272 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.272 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.272 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.900795+0000", 2024-09-13T14:52:03.272 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5803e-05 2024-09-13T14:52:03.272 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.272 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.273 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.273 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.273 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.273 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.45167.0:34135 6.3 e150/145 ondisk, result = 0)", 2024-09-13T14:52:03.273 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.704116+0000", 2024-09-13T14:52:03.273 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 460.54498620200002, 2024-09-13T14:52:03.273 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19344439799999999, 2024-09-13T14:52:03.273 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.273 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:52:03.273 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.273 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.273 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.274 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704116+0000", 2024-09-13T14:52:03.274 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.274 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.274 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.274 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.274 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704116+0000", 2024-09-13T14:52:03.274 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.274 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.274 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.274 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.274 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704115+0000", 2024-09-13T14:52:03.274 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:52:03.274 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.274 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.275 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.275 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704119+0000", 2024-09-13T14:52:03.275 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9960000000000004e-06 2024-09-13T14:52:03.275 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.275 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.275 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.275 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897303+0000", 2024-09-13T14:52:03.275 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19318347899999999 2024-09-13T14:52:03.275 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.275 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.276 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.276 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897312+0000", 2024-09-13T14:52:03.277 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8969999999999997e-06 2024-09-13T14:52:03.278 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.278 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.278 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.278 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897545+0000", 2024-09-13T14:52:03.278 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00023321799999999999 2024-09-13T14:52:03.278 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.278 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.278 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.278 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897552+0000", 2024-09-13T14:52:03.278 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.396e-06 2024-09-13T14:52:03.278 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.278 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.279 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.279 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897560+0000", 2024-09-13T14:52:03.279 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0050000000000004e-06 2024-09-13T14:52:03.280 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.280 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.280 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.280 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.280 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.281 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:34141 6.b e150/145 6:d45fd98b:::benchmark_data_smithi096_167087_object34140:head v 150'2166, mlcod=150'2163)", 2024-09-13T14:52:03.281 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:22.704766+0000", 2024-09-13T14:52:03.281 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 460.54433595900002, 2024-09-13T14:52:03.281 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.197159529, 2024-09-13T14:52:03.282 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.282 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:03.282 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.282 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.282 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.283 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704766+0000", 2024-09-13T14:52:03.283 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.283 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.283 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.283 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.283 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704766+0000", 2024-09-13T14:52:03.284 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.284 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.284 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.284 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.284 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704766+0000", 2024-09-13T14:52:03.284 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:52:03.285 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.285 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.285 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.285 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.704832+0000", 2024-09-13T14:52:03.285 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5964999999999996e-05 2024-09-13T14:52:03.285 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.286 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.286 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.286 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897456+0000", 2024-09-13T14:52:03.286 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.192624657 2024-09-13T14:52:03.286 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.287 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.287 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.287 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897466+0000", 2024-09-13T14:52:03.287 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5510000000000005e-06 2024-09-13T14:52:03.287 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.287 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.288 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.288 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897783+0000", 2024-09-13T14:52:03.288 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00031745699999999999 2024-09-13T14:52:03.288 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.288 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.288 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.289 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.897800+0000", 2024-09-13T14:52:03.289 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6619000000000001e-05 2024-09-13T14:52:03.291 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.292 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.292 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:52:03.292 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.901888+0000", 2024-09-13T14:52:03.292 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0040879230000000003 2024-09-13T14:52:03.292 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.292 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.292 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.292 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:22.901926+0000", 2024-09-13T14:52:03.292 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7832e-05 2024-09-13T14:52:03.292 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.292 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.292 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.293 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.293 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.293 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:37473 6.6 e150/145 6:6dedaf54:::benchmark_data_smithi096_167087_object37472:head v 150'2324, mlcod=150'2322)", 2024-09-13T14:52:03.293 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:44:25.100530+0000", 2024-09-13T14:52:03.293 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 458.14857190999999, 2024-09-13T14:52:03.293 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.17121808399999999, 2024-09-13T14:52:03.293 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.293 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:03.293 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.293 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.294 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.294 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100530+0000", 2024-09-13T14:52:03.294 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.294 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.294 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.294 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.294 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100530+0000", 2024-09-13T14:52:03.294 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.294 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.294 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.294 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.294 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100530+0000", 2024-09-13T14:52:03.294 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:52:03.295 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.295 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.295 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.295 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100650+0000", 2024-09-13T14:52:03.295 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011967 2024-09-13T14:52:03.295 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.295 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.295 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.297 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100656+0000", 2024-09-13T14:52:03.297 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1380000000000004e-06 2024-09-13T14:52:03.298 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.298 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.298 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.298 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100659+0000", 2024-09-13T14:52:03.298 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9249999999999999e-06 2024-09-13T14:52:03.298 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.298 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.298 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.298 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100689+0000", 2024-09-13T14:52:03.299 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0114999999999999e-05 2024-09-13T14:52:03.299 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.299 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.299 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.299 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.100704+0000", 2024-09-13T14:52:03.299 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5502000000000002e-05 2024-09-13T14:52:03.299 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.299 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.299 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:52:03.299 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271712+0000", 2024-09-13T14:52:03.299 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.17100812600000001 2024-09-13T14:52:03.300 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.300 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.300 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.300 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:44:25.271748+0000", 2024-09-13T14:52:03.300 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6117e-05 2024-09-13T14:52:03.300 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.300 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.300 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.300 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.300 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.300 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156611 6.3 6.ccd62063 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-13T14:52:03.301 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.006819+0000", 2024-09-13T14:52:03.301 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 331.24228332199999, 2024-09-13T14:52:03.301 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.30103220400000003, 2024-09-13T14:52:03.301 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.301 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:52:03.301 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:52:03.301 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:52:03.301 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:52:03.301 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156611 2024-09-13T14:52:03.301 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.302 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.302 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.302 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.302 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006819+0000", 2024-09-13T14:52:03.302 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.302 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.302 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.302 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.302 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006819+0000", 2024-09-13T14:52:03.302 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.302 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.303 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.303 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.303 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006816+0000", 2024-09-13T14:52:03.303 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999971 2024-09-13T14:52:03.303 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.303 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.303 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.303 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006831+0000", 2024-09-13T14:52:03.303 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5e-05 2024-09-13T14:52:03.303 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.303 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.304 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.304 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006848+0000", 2024-09-13T14:52:03.304 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6429000000000001e-05 2024-09-13T14:52:03.304 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.304 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.304 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.304 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006856+0000", 2024-09-13T14:52:03.304 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8420000000000005e-06 2024-09-13T14:52:03.304 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.304 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.304 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.305 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.307840+0000", 2024-09-13T14:52:03.305 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.30098485899999999 2024-09-13T14:52:03.305 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.305 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.305 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.305 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.307851+0000", 2024-09-13T14:52:03.305 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0763e-05 2024-09-13T14:52:03.305 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.305 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.305 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.305 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.306 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.306 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:156604 6.9 e173/169)", 2024-09-13T14:52:03.306 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.013215+0000", 2024-09-13T14:52:03.306 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 331.235887601, 2024-09-13T14:52:03.306 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.26933052299999999, 2024-09-13T14:52:03.306 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.306 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:52:03.306 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.306 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.306 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.306 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013215+0000", 2024-09-13T14:52:03.306 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.307 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.307 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.307 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.307 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013215+0000", 2024-09-13T14:52:03.307 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.307 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.307 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.307 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.307 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013214+0000", 2024-09-13T14:52:03.307 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:52:03.307 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.307 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.308 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.308 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013223+0000", 2024-09-13T14:52:03.308 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.0750000000000004e-06 2024-09-13T14:52:03.308 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.308 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.308 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.308 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013236+0000", 2024-09-13T14:52:03.308 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2925000000000001e-05 2024-09-13T14:52:03.308 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.309 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.309 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.309 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013242+0000", 2024-09-13T14:52:03.309 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0839999999999997e-06 2024-09-13T14:52:03.309 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.309 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.309 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.309 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282529+0000", 2024-09-13T14:52:03.309 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.26928756599999998 2024-09-13T14:52:03.310 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.310 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.310 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.310 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282545+0000", 2024-09-13T14:52:03.310 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5888000000000001e-05 2024-09-13T14:52:03.310 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.310 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.310 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.310 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.310 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.311 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:156612 6.9 e173/169)", 2024-09-13T14:52:03.311 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.023006+0000", 2024-09-13T14:52:03.311 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 331.22609599899999, 2024-09-13T14:52:03.311 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.25955990099999998, 2024-09-13T14:52:03.311 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.311 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:52:03.311 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.311 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.311 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.312 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023006+0000", 2024-09-13T14:52:03.312 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.312 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.312 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.312 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.312 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023006+0000", 2024-09-13T14:52:03.312 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.312 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.312 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.312 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.312 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023004+0000", 2024-09-13T14:52:03.313 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999976 2024-09-13T14:52:03.313 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.313 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.313 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.313 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023023+0000", 2024-09-13T14:52:03.313 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9327999999999999e-05 2024-09-13T14:52:03.313 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.313 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.313 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.313 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023054+0000", 2024-09-13T14:52:03.313 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0972000000000001e-05 2024-09-13T14:52:03.313 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.314 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.314 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.314 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023064+0000", 2024-09-13T14:52:03.314 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0198e-05 2024-09-13T14:52:03.314 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.314 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.314 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.314 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282559+0000", 2024-09-13T14:52:03.314 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.25949471600000001 2024-09-13T14:52:03.314 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.314 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.315 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.315 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282566+0000", 2024-09-13T14:52:03.315 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1570000000000003e-06 2024-09-13T14:52:03.315 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.315 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.315 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.315 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.315 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.315 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156611 6.3 6:c6046b33:::benchmark_data_smithi096_167087_object39383:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:52:03.315 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.023013+0000", 2024-09-13T14:52:03.315 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 331.22608929, 2024-09-13T14:52:03.315 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0343409539999999, 2024-09-13T14:52:03.316 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.316 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:03.316 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:52:03.316 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:52:03.316 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:52:03.316 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156611 2024-09-13T14:52:03.316 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.316 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.316 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.316 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.316 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023013+0000", 2024-09-13T14:52:03.317 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.317 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.317 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.317 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.317 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023013+0000", 2024-09-13T14:52:03.317 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.317 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.317 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.317 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.317 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023004+0000", 2024-09-13T14:52:03.318 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999909 2024-09-13T14:52:03.318 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.318 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.318 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.318 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023024+0000", 2024-09-13T14:52:03.318 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9893000000000001e-05 2024-09-13T14:52:03.318 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.318 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.318 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.318 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.039994+0000", 2024-09-13T14:52:03.318 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.016970180000000001 2024-09-13T14:52:03.318 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.319 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.319 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.319 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040019+0000", 2024-09-13T14:52:03.319 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4919e-05 2024-09-13T14:52:03.319 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.319 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.319 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.319 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.308192+0000", 2024-09-13T14:52:03.319 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.26817356999999997 2024-09-13T14:52:03.320 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.320 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.320 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:52:03.320 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.308207+0000", 2024-09-13T14:52:03.320 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4960000000000001e-05 2024-09-13T14:52:03.320 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.320 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.320 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.320 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.043149+0000", 2024-09-13T14:52:03.320 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.73494118500000005 2024-09-13T14:52:03.320 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.321 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.321 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.321 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.043279+0000", 2024-09-13T14:52:03.321 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013044200000000001 2024-09-13T14:52:03.321 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.321 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.321 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:52:03.321 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.043839+0000", 2024-09-13T14:52:03.321 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00056042099999999997 2024-09-13T14:52:03.321 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.321 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.321 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:52:03.322 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.047415+0000", 2024-09-13T14:52:03.322 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035758959999999998 2024-09-13T14:52:03.322 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.322 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.322 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.322 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.052702+0000", 2024-09-13T14:52:03.322 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0052870019999999998 2024-09-13T14:52:03.322 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.322 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.322 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.322 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.057268+0000", 2024-09-13T14:52:03.323 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0045653200000000003 2024-09-13T14:52:03.323 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.323 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.323 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:52:03.323 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.057316+0000", 2024-09-13T14:52:03.323 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8798000000000003e-05 2024-09-13T14:52:03.323 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.323 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.323 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.323 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.057354+0000", 2024-09-13T14:52:03.323 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7571999999999999e-05 2024-09-13T14:52:03.324 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.324 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.324 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.324 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.324 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.324 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156616 6.15 6:ac5310a6:::benchmark_data_smithi096_167087_object39388:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:52:03.324 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.040042+0000", 2024-09-13T14:52:03.324 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 331.20906065200001, 2024-09-13T14:52:03.324 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1222388599999999, 2024-09-13T14:52:03.324 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.324 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:03.324 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:52:03.325 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:52:03.325 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:52:03.325 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156616 2024-09-13T14:52:03.325 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.325 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.325 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.325 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.325 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040042+0000", 2024-09-13T14:52:03.325 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.325 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.325 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.326 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.326 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040042+0000", 2024-09-13T14:52:03.326 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.326 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.326 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.326 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.326 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040039+0000", 2024-09-13T14:52:03.326 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999976 2024-09-13T14:52:03.326 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.326 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.326 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.326 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040051+0000", 2024-09-13T14:52:03.327 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2446000000000001e-05 2024-09-13T14:52:03.327 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.327 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.327 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.327 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040068+0000", 2024-09-13T14:52:03.327 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6059e-05 2024-09-13T14:52:03.327 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.327 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.327 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.327 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040072+0000", 2024-09-13T14:52:03.327 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2370000000000003e-06 2024-09-13T14:52:03.327 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.328 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.328 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.328 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.309500+0000", 2024-09-13T14:52:03.328 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.26942850299999999 2024-09-13T14:52:03.328 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.328 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.328 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:52:03.328 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.309506+0000", 2024-09-13T14:52:03.328 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.049e-06 2024-09-13T14:52:03.328 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.328 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.329 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.329 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.158349+0000", 2024-09-13T14:52:03.329 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.84884293700000002 2024-09-13T14:52:03.329 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.329 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.329 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.329 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.158451+0000", 2024-09-13T14:52:03.329 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000101679 2024-09-13T14:52:03.329 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.329 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.329 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:52:03.329 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.158782+0000", 2024-09-13T14:52:03.330 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00033148900000000002 2024-09-13T14:52:03.330 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.330 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.330 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:52:03.330 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.160769+0000", 2024-09-13T14:52:03.330 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0019864119999999999 2024-09-13T14:52:03.330 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.330 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.330 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.330 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.161676+0000", 2024-09-13T14:52:03.330 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00090709000000000004 2024-09-13T14:52:03.331 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.331 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.331 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.331 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.162206+0000", 2024-09-13T14:52:03.331 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00053007 2024-09-13T14:52:03.331 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.331 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.331 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:52:03.331 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.162249+0000", 2024-09-13T14:52:03.331 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3250000000000001e-05 2024-09-13T14:52:03.331 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.331 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.332 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.332 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.162281+0000", 2024-09-13T14:52:03.332 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1241000000000001e-05 2024-09-13T14:52:03.332 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.332 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.332 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.332 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.332 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.332 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156618 6.8 6:13ebd03e:::benchmark_data_smithi096_167087_object39390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:52:03.332 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.040084+0000", 2024-09-13T14:52:03.332 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 331.20901834699998, 2024-09-13T14:52:03.333 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.023429146, 2024-09-13T14:52:03.333 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.333 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:03.333 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:52:03.333 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:52:03.333 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:52:03.333 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156618 2024-09-13T14:52:03.333 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.333 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.333 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.334 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.334 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040084+0000", 2024-09-13T14:52:03.334 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.334 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.334 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.334 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.334 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040084+0000", 2024-09-13T14:52:03.334 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.334 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.334 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.334 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.335 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040084+0000", 2024-09-13T14:52:03.335 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:52:03.335 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.335 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.335 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.335 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040087+0000", 2024-09-13T14:52:03.335 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0809999999999998e-06 2024-09-13T14:52:03.335 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.335 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.335 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.335 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040095+0000", 2024-09-13T14:52:03.336 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9529999999999999e-06 2024-09-13T14:52:03.336 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.336 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.336 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.336 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040098+0000", 2024-09-13T14:52:03.336 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4599999999999999e-06 2024-09-13T14:52:03.336 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.336 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.336 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.337 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.304957+0000", 2024-09-13T14:52:03.337 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.26485901699999997 2024-09-13T14:52:03.337 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.337 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.337 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:52:03.337 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.304969+0000", 2024-09-13T14:52:03.337 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1610999999999999e-05 2024-09-13T14:52:03.337 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.337 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.337 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.337 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.046503+0000", 2024-09-13T14:52:03.338 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.74153483200000003 2024-09-13T14:52:03.338 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.338 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.338 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.338 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.046619+0000", 2024-09-13T14:52:03.338 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000115473 2024-09-13T14:52:03.338 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.338 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.338 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:52:03.338 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.047071+0000", 2024-09-13T14:52:03.338 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00045168200000000002 2024-09-13T14:52:03.339 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.339 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.339 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:52:03.339 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.049454+0000", 2024-09-13T14:52:03.339 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002383399 2024-09-13T14:52:03.339 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.339 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.339 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.339 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.057281+0000", 2024-09-13T14:52:03.339 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0078272889999999994 2024-09-13T14:52:03.339 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.340 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.340 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.063435+0000", 2024-09-13T14:52:03.340 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0061532130000000003 2024-09-13T14:52:03.340 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.340 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:52:03.340 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.063479+0000", 2024-09-13T14:52:03.340 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4530000000000002e-05 2024-09-13T14:52:03.340 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.341 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.341 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.063513+0000", 2024-09-13T14:52:03.341 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4041999999999997e-05 2024-09-13T14:52:03.341 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.341 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.341 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.341 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.341 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.341 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:156614 6.1 e173/169)", 2024-09-13T14:52:03.341 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.040126+0000", 2024-09-13T14:52:03.342 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 331.20897637799999, 2024-09-13T14:52:03.342 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.24246019299999999, 2024-09-13T14:52:03.342 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.342 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:52:03.342 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.342 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.342 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.342 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040126+0000", 2024-09-13T14:52:03.342 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.342 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.343 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.343 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.343 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040126+0000", 2024-09-13T14:52:03.343 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.343 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.343 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.343 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.343 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040125+0000", 2024-09-13T14:52:03.343 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:03.343 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.343 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.344 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.344 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040134+0000", 2024-09-13T14:52:03.344 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.3610000000000007e-06 2024-09-13T14:52:03.344 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.344 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.344 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.344 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040144+0000", 2024-09-13T14:52:03.344 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0173000000000001e-05 2024-09-13T14:52:03.344 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.344 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.345 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.345 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040148+0000", 2024-09-13T14:52:03.345 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6169999999999998e-06 2024-09-13T14:52:03.345 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.345 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.345 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.345 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282579+0000", 2024-09-13T14:52:03.345 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.242431116 2024-09-13T14:52:03.345 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.345 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.345 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.346 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282586+0000", 2024-09-13T14:52:03.346 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.216e-06 2024-09-13T14:52:03.346 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.346 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.346 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.346 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.346 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.346 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156624 6.13 6:c852d4ec:::benchmark_data_smithi096_167087_object39396:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:52:03.346 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.372301+0000", 2024-09-13T14:52:03.346 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 330.87680169999999, 2024-09-13T14:52:03.346 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.7720032, 2024-09-13T14:52:03.346 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.347 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:03.347 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:52:03.347 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:52:03.347 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:52:03.347 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156624 2024-09-13T14:52:03.347 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.347 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.347 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.347 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.347 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372301+0000", 2024-09-13T14:52:03.347 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.347 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.348 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.348 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.348 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372301+0000", 2024-09-13T14:52:03.348 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.348 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.348 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.348 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.348 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372299+0000", 2024-09-13T14:52:03.348 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:03.348 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.348 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.349 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.349 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372306+0000", 2024-09-13T14:52:03.349 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.221e-06 2024-09-13T14:52:03.349 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.349 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.349 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.349 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372320+0000", 2024-09-13T14:52:03.349 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3293e-05 2024-09-13T14:52:03.349 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.349 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.349 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.350 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372325+0000", 2024-09-13T14:52:03.350 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7089999999999999e-06 2024-09-13T14:52:03.350 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.350 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.350 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.350 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372366+0000", 2024-09-13T14:52:03.350 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0308000000000001e-05 2024-09-13T14:52:03.350 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.350 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.350 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:52:03.350 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372372+0000", 2024-09-13T14:52:03.350 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8179999999999998e-06 2024-09-13T14:52:03.351 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.351 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.351 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.351 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.070086+0000", 2024-09-13T14:52:03.351 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.69771321799999997 2024-09-13T14:52:03.351 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.351 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.351 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.351 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.070229+0000", 2024-09-13T14:52:03.351 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014322 2024-09-13T14:52:03.351 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.351 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.352 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:52:03.352 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.070517+0000", 2024-09-13T14:52:03.352 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00028809400000000002 2024-09-13T14:52:03.352 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.352 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.352 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:52:03.352 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.072169+0000", 2024-09-13T14:52:03.352 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0016519390000000001 2024-09-13T14:52:03.352 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.352 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.352 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.353 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.073034+0000", 2024-09-13T14:52:03.353 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00086496100000000003 2024-09-13T14:52:03.353 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.353 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.353 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.353 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.144218+0000", 2024-09-13T14:52:03.353 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.071183739999999995 2024-09-13T14:52:03.353 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.353 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.353 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:52:03.353 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.144268+0000", 2024-09-13T14:52:03.354 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0076e-05 2024-09-13T14:52:03.354 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.354 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.354 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.354 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.144304+0000", 2024-09-13T14:52:03.354 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6125999999999999e-05 2024-09-13T14:52:03.354 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.354 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.354 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.354 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.354 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.355 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:182590 6.15 6:aa32eba5:::benchmark_data_smithi096_167087_object65362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2024-09-13T14:52:03.355 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:19.162570+0000", 2024-09-13T14:52:03.355 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 284.08653247900003, 2024-09-13T14:52:03.355 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95580658399999996, 2024-09-13T14:52:03.355 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.355 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:03.355 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:52:03.355 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:52:03.355 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:52:03.355 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 182590 2024-09-13T14:52:03.355 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.355 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.356 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.356 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.356 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162570+0000", 2024-09-13T14:52:03.356 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.356 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.356 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.356 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.356 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162570+0000", 2024-09-13T14:52:03.356 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.356 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.356 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.356 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.357 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162569+0000", 2024-09-13T14:52:03.357 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:52:03.357 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.357 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.357 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.357 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162574+0000", 2024-09-13T14:52:03.357 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3739999999999997e-06 2024-09-13T14:52:03.357 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.357 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.357 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.358 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162587+0000", 2024-09-13T14:52:03.358 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2448e-05 2024-09-13T14:52:03.358 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.358 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.358 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.358 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162593+0000", 2024-09-13T14:52:03.358 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.669e-06 2024-09-13T14:52:03.358 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.358 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.358 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.358 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162634+0000", 2024-09-13T14:52:03.359 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0627000000000003e-05 2024-09-13T14:52:03.359 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.359 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.359 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:52:03.359 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162642+0000", 2024-09-13T14:52:03.359 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2460000000000003e-06 2024-09-13T14:52:03.359 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.359 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.359 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.359 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.113644+0000", 2024-09-13T14:52:03.359 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95100172500000002 2024-09-13T14:52:03.359 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.360 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.360 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.360 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.113728+0000", 2024-09-13T14:52:03.360 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4036000000000007e-05 2024-09-13T14:52:03.360 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.360 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.360 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:52:03.360 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.113982+0000", 2024-09-13T14:52:03.360 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00025374799999999998 2024-09-13T14:52:03.360 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.360 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.360 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.361 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.116230+0000", 2024-09-13T14:52:03.361 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002248613 2024-09-13T14:52:03.361 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.361 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.361 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:52:03.361 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118234+0000", 2024-09-13T14:52:03.361 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0020033669999999998 2024-09-13T14:52:03.361 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.361 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.361 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.361 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118305+0000", 2024-09-13T14:52:03.361 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1443e-05 2024-09-13T14:52:03.362 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.362 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.362 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:52:03.362 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118344+0000", 2024-09-13T14:52:03.362 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8488999999999997e-05 2024-09-13T14:52:03.362 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.362 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.362 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.362 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118376+0000", 2024-09-13T14:52:03.362 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2659e-05 2024-09-13T14:52:03.362 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.362 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.363 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.363 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.363 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.363 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:182611 6.15 6:ae0b3e38:::benchmark_data_smithi096_167087_object65383:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2024-09-13T14:52:03.363 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:19.183753+0000", 2024-09-13T14:52:03.363 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 284.06534911599999, 2024-09-13T14:52:03.363 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93487196400000006, 2024-09-13T14:52:03.363 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.363 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:03.363 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:52:03.363 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:52:03.364 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:52:03.364 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 182611 2024-09-13T14:52:03.364 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.364 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.364 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.364 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.364 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183753+0000", 2024-09-13T14:52:03.364 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.364 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.364 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.364 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.365 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183753+0000", 2024-09-13T14:52:03.365 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.365 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.365 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.365 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.365 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183752+0000", 2024-09-13T14:52:03.365 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:52:03.365 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.365 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.365 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.365 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183757+0000", 2024-09-13T14:52:03.366 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0479999999999998e-06 2024-09-13T14:52:03.366 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.366 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.366 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.366 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183767+0000", 2024-09-13T14:52:03.366 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7410000000000004e-06 2024-09-13T14:52:03.366 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.366 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.366 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.366 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183771+0000", 2024-09-13T14:52:03.366 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1799999999999998e-06 2024-09-13T14:52:03.366 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.367 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.367 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.367 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183805+0000", 2024-09-13T14:52:03.367 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4193000000000003e-05 2024-09-13T14:52:03.367 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.367 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.367 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:52:03.367 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183812+0000", 2024-09-13T14:52:03.367 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1349999999999997e-06 2024-09-13T14:52:03.367 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.367 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.368 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.368 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.116254+0000", 2024-09-13T14:52:03.368 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93244249599999995 2024-09-13T14:52:03.368 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.368 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.368 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.368 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.116358+0000", 2024-09-13T14:52:03.368 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000104342 2024-09-13T14:52:03.368 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.368 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.368 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:52:03.368 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.116545+0000", 2024-09-13T14:52:03.369 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000186152 2024-09-13T14:52:03.369 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.369 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.369 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.369 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118435+0000", 2024-09-13T14:52:03.369 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0018904519999999999 2024-09-13T14:52:03.369 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.369 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.369 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:52:03.369 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118536+0000", 2024-09-13T14:52:03.369 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000101067 2024-09-13T14:52:03.370 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.370 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.370 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.370 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118571+0000", 2024-09-13T14:52:03.370 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4502000000000003e-05 2024-09-13T14:52:03.370 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.370 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.370 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:52:03.370 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118608+0000", 2024-09-13T14:52:03.371 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7051999999999997e-05 2024-09-13T14:52:03.371 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.371 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.371 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.371 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118625+0000", 2024-09-13T14:52:03.371 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7555999999999999e-05 2024-09-13T14:52:03.371 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.371 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.371 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.371 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.371 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.372 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:125782 7.5 7:a83dc0eb:::benchmark_data_smithi096_196313_object5651:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2024-09-13T14:52:03.372 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:50:16.277859+0000", 2024-09-13T14:52:03.372 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 106.971243644, 2024-09-13T14:52:03.372 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.34631596799999997, 2024-09-13T14:52:03.372 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.372 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:03.372 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:52:03.372 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:52:03.372 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:52:03.372 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 125782 2024-09-13T14:52:03.373 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.373 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.373 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.373 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.373 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277859+0000", 2024-09-13T14:52:03.373 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.373 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.373 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.373 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.373 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277859+0000", 2024-09-13T14:52:03.373 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.374 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.374 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.374 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.374 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277858+0000", 2024-09-13T14:52:03.374 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:52:03.374 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.374 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.374 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.374 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277862+0000", 2024-09-13T14:52:03.374 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6859999999999997e-06 2024-09-13T14:52:03.374 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.375 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.375 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.375 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277872+0000", 2024-09-13T14:52:03.375 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6989999999999998e-06 2024-09-13T14:52:03.375 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.375 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.375 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.375 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277876+0000", 2024-09-13T14:52:03.375 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.391e-06 2024-09-13T14:52:03.375 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.375 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.375 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.376 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.278833+0000", 2024-09-13T14:52:03.376 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00095691899999999998 2024-09-13T14:52:03.376 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.376 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.376 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.376 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.278950+0000", 2024-09-13T14:52:03.376 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000116446 2024-09-13T14:52:03.376 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.376 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.376 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:52:03.376 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.279046+0000", 2024-09-13T14:52:03.377 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6119999999999995e-05 2024-09-13T14:52:03.377 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.377 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.377 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:52:03.377 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.282554+0000", 2024-09-13T14:52:03.377 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035082289999999999 2024-09-13T14:52:03.377 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.377 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.377 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.377 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287518+0000", 2024-09-13T14:52:03.377 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0049643049999999996 2024-09-13T14:52:03.378 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.378 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.378 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.378 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624079+0000", 2024-09-13T14:52:03.378 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.33656060599999998 2024-09-13T14:52:03.378 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.378 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.378 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:52:03.378 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624136+0000", 2024-09-13T14:52:03.378 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6737000000000002e-05 2024-09-13T14:52:03.378 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.379 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.379 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.379 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624175+0000", 2024-09-13T14:52:03.379 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8862000000000002e-05 2024-09-13T14:52:03.379 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.379 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.379 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.379 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.379 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.380 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:125792 7.5 7:a137e5c2:::benchmark_data_smithi096_196313_object5661:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2024-09-13T14:52:03.380 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:50:16.287604+0000", 2024-09-13T14:52:03.380 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 106.96149822, 2024-09-13T14:52:03.380 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.33670364200000003, 2024-09-13T14:52:03.380 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.380 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:03.380 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:52:03.380 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:52:03.380 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:52:03.380 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 125792 2024-09-13T14:52:03.380 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.381 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.381 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.381 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.381 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287604+0000", 2024-09-13T14:52:03.381 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.381 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.381 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.381 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.381 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287604+0000", 2024-09-13T14:52:03.381 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.381 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.382 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.382 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.382 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287603+0000", 2024-09-13T14:52:03.382 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:52:03.382 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.382 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.382 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.382 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287609+0000", 2024-09-13T14:52:03.382 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6489999999999996e-06 2024-09-13T14:52:03.382 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.382 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.382 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.383 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287618+0000", 2024-09-13T14:52:03.383 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5820000000000008e-06 2024-09-13T14:52:03.383 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.383 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.383 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.383 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287626+0000", 2024-09-13T14:52:03.383 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1780000000000003e-06 2024-09-13T14:52:03.383 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.383 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.383 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.383 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287652+0000", 2024-09-13T14:52:03.383 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5395e-05 2024-09-13T14:52:03.384 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.384 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.384 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.384 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287735+0000", 2024-09-13T14:52:03.384 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2979000000000006e-05 2024-09-13T14:52:03.384 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.384 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.384 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:52:03.384 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287800+0000", 2024-09-13T14:52:03.384 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5335999999999999e-05 2024-09-13T14:52:03.384 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.385 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.385 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:52:03.385 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.296078+0000", 2024-09-13T14:52:03.385 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:03.386 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:52:03.387 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0082774900000000002 2024-09-13T14:52:03.387 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.387 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.387 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.387 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.299594+0000", 2024-09-13T14:52:03.387 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035163949999999999 2024-09-13T14:52:03.387 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.387 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.387 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.387 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624220+0000", 2024-09-13T14:52:03.388 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.32462581499999998 2024-09-13T14:52:03.388 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.388 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.388 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:52:03.388 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624269+0000", 2024-09-13T14:52:03.388 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9626000000000002e-05 2024-09-13T14:52:03.388 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.388 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.388 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.388 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624308+0000", 2024-09-13T14:52:03.389 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8374999999999998e-05 2024-09-13T14:52:03.389 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.389 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.389 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.389 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.389 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.389 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:125795 7.5 7:acb77273:::benchmark_data_smithi096_196313_object5664:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2024-09-13T14:52:03.389 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:50:16.290753+0000", 2024-09-13T14:52:03.390 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 106.95834977, 2024-09-13T14:52:03.390 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.33410231899999998, 2024-09-13T14:52:03.390 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.390 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:03.390 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:52:03.390 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:52:03.390 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:52:03.390 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 125795 2024-09-13T14:52:03.390 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.390 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.391 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.391 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.391 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290753+0000", 2024-09-13T14:52:03.391 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.391 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.391 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.391 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.391 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290753+0000", 2024-09-13T14:52:03.391 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.391 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.391 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.392 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.392 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290751+0000", 2024-09-13T14:52:03.392 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:03.392 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.392 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.392 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.392 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290758+0000", 2024-09-13T14:52:03.392 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.238e-06 2024-09-13T14:52:03.392 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.392 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.393 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.393 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290778+0000", 2024-09-13T14:52:03.393 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9643000000000001e-05 2024-09-13T14:52:03.393 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.393 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.393 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.393 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290786+0000", 2024-09-13T14:52:03.393 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5220000000000001e-06 2024-09-13T14:52:03.393 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.394 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.394 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.394 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.291631+0000", 2024-09-13T14:52:03.394 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000845617 2024-09-13T14:52:03.394 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.394 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.394 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.394 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.291741+0000", 2024-09-13T14:52:03.394 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000109487 2024-09-13T14:52:03.394 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.394 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.395 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:52:03.395 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.291842+0000", 2024-09-13T14:52:03.395 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000101823 2024-09-13T14:52:03.395 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.395 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.395 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:52:03.395 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.296685+0000", 2024-09-13T14:52:03.395 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0048421799999999997 2024-09-13T14:52:03.395 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.395 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.396 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.396 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.299630+0000", 2024-09-13T14:52:03.396 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002945544 2024-09-13T14:52:03.396 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.396 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.396 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.396 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624773+0000", 2024-09-13T14:52:03.396 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.32514234600000003 2024-09-13T14:52:03.396 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.396 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.396 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:52:03.397 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624815+0000", 2024-09-13T14:52:03.397 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2979999999999998e-05 2024-09-13T14:52:03.397 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.397 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.397 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.397 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624855+0000", 2024-09-13T14:52:03.397 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9347000000000001e-05 2024-09-13T14:52:03.397 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.397 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.397 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.397 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.398 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.398 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:172721 7.5 7:a4933ecd:::benchmark_data_smithi096_196313_object52590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-09-13T14:52:03.398 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:16.905298+0000", 2024-09-13T14:52:03.398 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 46.343803954999998, 2024-09-13T14:52:03.398 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18517154299999999, 2024-09-13T14:52:03.398 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:52:03.398 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:03.398 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:52:03.398 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:52:03.398 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:52:03.398 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 172721 2024-09-13T14:52:03.399 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.399 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:52:03.399 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.399 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:52:03.399 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905298+0000", 2024-09-13T14:52:03.399 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.399 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.399 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.399 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:52:03.399 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905298+0000", 2024-09-13T14:52:03.399 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:52:03.400 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.400 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.400 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:52:03.400 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905297+0000", 2024-09-13T14:52:03.400 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:03.400 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.400 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.400 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:52:03.400 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905304+0000", 2024-09-13T14:52:03.400 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5290000000000003e-06 2024-09-13T14:52:03.400 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.401 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.401 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:52:03.401 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905320+0000", 2024-09-13T14:52:03.401 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5852e-05 2024-09-13T14:52:03.401 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.401 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.401 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:52:03.401 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905327+0000", 2024-09-13T14:52:03.401 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5889999999999999e-06 2024-09-13T14:52:03.401 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.401 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.401 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:52:03.402 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905375+0000", 2024-09-13T14:52:03.402 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8362000000000002e-05 2024-09-13T14:52:03.402 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.402 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.402 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:52:03.402 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905529+0000", 2024-09-13T14:52:03.402 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015435700000000001 2024-09-13T14:52:03.402 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.402 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.402 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:52:03.402 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905659+0000", 2024-09-13T14:52:03.402 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000129231 2024-09-13T14:52:03.403 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.403 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.403 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:52:03.403 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.909873+0000", 2024-09-13T14:52:03.403 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.004214158 2024-09-13T14:52:03.403 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.403 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.403 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.403 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.910053+0000", 2024-09-13T14:52:03.403 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018025800000000001 2024-09-13T14:52:03.403 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.403 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.404 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:03.404 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:17.090381+0000", 2024-09-13T14:52:03.404 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18032772599999999 2024-09-13T14:52:03.404 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.404 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.404 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:52:03.404 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:17.090429+0000", 2024-09-13T14:52:03.404 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8335999999999999e-05 2024-09-13T14:52:03.404 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:52:03.404 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:52:03.404 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:52:03.405 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:17.090470+0000", 2024-09-13T14:52:03.405 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0713000000000001e-05 2024-09-13T14:52:03.405 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.405 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.405 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.405 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:52:03.405 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:52:03.405 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:52:03.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.432 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:03.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:03.533 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:03.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.563 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:03.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:03 smithi096 ceph-mon[98901]: pgmap v762: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 4835/109230 objects misplaced (4.426%); 7.9 MiB/s, 126 objects/s recovering 2024-09-13T14:52:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:03 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2692036735' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:52:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:03 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:52:03.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:03 smithi096 ceph-mon[102928]: pgmap v762: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 4835/109230 objects misplaced (4.426%); 7.9 MiB/s, 126 objects/s recovering 2024-09-13T14:52:03.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:03 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2692036735' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:52:03.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:03 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:52:03.664 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:03.668 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:03 smithi204 ceph-mon[87071]: pgmap v762: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 4835/109230 objects misplaced (4.426%); 7.9 MiB/s, 126 objects/s recovering 2024-09-13T14:52:03.668 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:03 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2692036735' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:52:03.668 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:03 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:52:03.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.695 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:03.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:03.796 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:03.826 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:03.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:03.928 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:03.969 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:03.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:04.071 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:04.100 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:04.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:04.202 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:04.228 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:52:04.242 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:04.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:04.344 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:04.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.385 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:04.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:04.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:04 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:04.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:04 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:52:04.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:04 smithi204 ceph-mon[87071]: osdmap e259: 8 total, 8 up, 8 in 2024-09-13T14:52:04.485 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:04.509 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:04.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:04.555 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:04 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:04.555 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:04 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:52:04.555 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:04 smithi096 ceph-mon[98901]: osdmap e259: 8 total, 8 up, 8 in 2024-09-13T14:52:04.555 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:04 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:04.555 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:04 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:52:04.555 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:04 smithi096 ceph-mon[102928]: osdmap e259: 8 total, 8 up, 8 in 2024-09-13T14:52:04.611 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:04.639 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:04.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:04.741 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:04.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.771 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:04.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:04.872 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:04.901 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:04.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:05.004 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:05.029 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:05.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:05.131 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:05.157 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:05.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:05.259 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:05.285 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:05.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:05.387 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:05.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.425 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:05.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:05.526 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:05.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.549 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:05.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:05.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:05 smithi096 ceph-mon[102928]: pgmap v764: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 4595/103365 objects misplaced (4.445%); 7.3 MiB/s, 116 objects/s recovering 2024-09-13T14:52:05.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:05 smithi096 ceph-mon[98901]: pgmap v764: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 4595/103365 objects misplaced (4.445%); 7.3 MiB/s, 116 objects/s recovering 2024-09-13T14:52:05.650 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:05.667 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:05.680 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:05.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:05.691 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:05 smithi204 ceph-mon[87071]: pgmap v764: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 4595/103365 objects misplaced (4.445%); 7.3 MiB/s, 116 objects/s recovering 2024-09-13T14:52:05.782 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:05.804 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:05.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:05.906 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:05.927 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:05.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:06.029 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:06.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.051 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:06.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:06.152 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:06.174 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:06.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:06.276 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:06.301 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:06.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:06.403 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:06.438 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:06.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:06.540 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:06.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:06 smithi096 ceph-mon[98901]: osdmap e260: 8 total, 8 up, 8 in 2024-09-13T14:52:06.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:06.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:06 smithi096 ceph-mon[102928]: osdmap e260: 8 total, 8 up, 8 in 2024-09-13T14:52:06.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:06.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.579 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:06.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:06.680 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:06 smithi204 ceph-mon[87071]: osdmap e260: 8 total, 8 up, 8 in 2024-09-13T14:52:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:06.721 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:06.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:06.823 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:06.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.850 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:06.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:06.886 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:52:06.887 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:52:06.930 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:52:06.930 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:52:06.951 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:06.971 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:52:06.971 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:52:06.975 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:06.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:07.008 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:52:07.008 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:52:07.027 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:52:07.038 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:52:07.038 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:52:07.069 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:52:07.069 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:52:07.077 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:07.099 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:52:07.100 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:52:07.136 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:07.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:07.138 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:52:07.138 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:52:07.237 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:07.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.308 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:07.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:07.410 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:07.437 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:07.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:07.539 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:07.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:07 smithi096 ceph-mon[98901]: pgmap v766: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 4595/101991 objects misplaced (4.505%); 4.1 MiB/s, 66 objects/s recovering 2024-09-13T14:52:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:07 smithi096 ceph-mon[98901]: osdmap e261: 8 total, 8 up, 8 in 2024-09-13T14:52:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:07 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4264144043' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:07 smithi096 ceph-mon[102928]: pgmap v766: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 4595/101991 objects misplaced (4.505%); 4.1 MiB/s, 66 objects/s recovering 2024-09-13T14:52:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:07 smithi096 ceph-mon[102928]: osdmap e261: 8 total, 8 up, 8 in 2024-09-13T14:52:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:07 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4264144043' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:07.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.585 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:07.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:07.686 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:07.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.714 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:07.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:07.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:07 smithi204 ceph-mon[87071]: pgmap v766: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 4595/101991 objects misplaced (4.505%); 4.1 MiB/s, 66 objects/s recovering 2024-09-13T14:52:07.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:07 smithi204 ceph-mon[87071]: osdmap e261: 8 total, 8 up, 8 in 2024-09-13T14:52:07.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:07 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4264144043' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:07.815 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:07.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.853 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:07.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:07.954 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:07.981 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:07.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:08.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:52:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:52:07] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.51.0" 2024-09-13T14:52:08.083 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:08.108 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:08.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:08.210 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:08.248 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:08.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:08.349 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:08.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.378 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:08.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:08.479 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:08.505 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:08.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:08.606 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:08.635 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:08.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:08.737 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:08.773 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:08.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:08.875 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:08.904 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:08.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:09.006 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:09.027 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:09.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:09.128 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:09.156 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:09.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:09.258 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:09.321 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:09.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:09.423 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:09.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.469 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:09.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:09.570 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:09.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:09 smithi096 ceph-mon[98901]: pgmap v768: 99 pgs: 2 active+remapped+backfill_wait, 97 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 4064/98475 objects misplaced (4.127%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:52:09.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:09 smithi096 ceph-mon[102928]: pgmap v768: 99 pgs: 2 active+remapped+backfill_wait, 97 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 4064/98475 objects misplaced (4.127%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:52:09.596 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:09.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:09.698 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:09.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:09 smithi204 ceph-mon[87071]: pgmap v768: 99 pgs: 2 active+remapped+backfill_wait, 97 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 4064/98475 objects misplaced (4.127%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:52:09.727 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:09.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:09.829 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:09.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.876 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:09.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:09.977 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:10.013 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:10.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:10.116 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:10.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.137 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:10.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:10.238 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:10.275 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:10.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:10.377 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:10.411 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:10.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:52:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:52:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:10.513 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:10.543 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:10.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:10.645 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:10.667 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:10.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:10.769 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:10.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.796 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:10.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:10.897 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:10.922 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:10.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:11.024 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:11.050 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:11.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:11.151 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:11.174 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:11.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:11.276 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:11.300 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:11.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:11.310 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:52:11.401 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:11.427 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:11.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:11.529 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:11.550 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:11.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:11.652 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:11.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.696 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:11.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:11 smithi204 ceph-mon[87071]: pgmap v769: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.9 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3215/93693 objects misplaced (3.431%); 9.5 MiB/s, 151 objects/s recovering 2024-09-13T14:52:11.797 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:11.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:11 smithi096 ceph-mon[98901]: pgmap v769: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.9 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3215/93693 objects misplaced (3.431%); 9.5 MiB/s, 151 objects/s recovering 2024-09-13T14:52:11.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:11 smithi096 ceph-mon[102928]: pgmap v769: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.9 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3215/93693 objects misplaced (3.431%); 9.5 MiB/s, 151 objects/s recovering 2024-09-13T14:52:11.826 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:11.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:11.928 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:11.950 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:11.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:12.052 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:12.096 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:12.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:12.198 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:12.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.238 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:12.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:12.340 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:12.365 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:12.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:12.468 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:12.511 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:12.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:12.613 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:12.654 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:12.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:12.756 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:12.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.778 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:12.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:12.879 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:12.907 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:12.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:13.008 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:13.049 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:13.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:13.151 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:13.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.178 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:13.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:13.279 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:13.314 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:13.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:13.416 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:13.441 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:13.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:13.542 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:13.564 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:13.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:13.666 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:13.690 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:13.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:13.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:13 smithi204 ceph-mon[87071]: pgmap v770: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 2979/89124 objects misplaced (3.343%); 7.8 MiB/s, 125 objects/s recovering 2024-09-13T14:52:13.792 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:13.817 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:13.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:13.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:13 smithi096 ceph-mon[98901]: pgmap v770: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 2979/89124 objects misplaced (3.343%); 7.8 MiB/s, 125 objects/s recovering 2024-09-13T14:52:13.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:13 smithi096 ceph-mon[102928]: pgmap v770: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 2979/89124 objects misplaced (3.343%); 7.8 MiB/s, 125 objects/s recovering 2024-09-13T14:52:13.919 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:13.948 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:13.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:14.052 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:14.081 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:14.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:14.183 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:14.208 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:14.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:14.310 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:14.331 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:14.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:14.433 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:14.465 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:14.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:14.567 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:14.619 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:14.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:14.721 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:14.747 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:14.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:14.848 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:14.872 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:14.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:14.974 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:14.996 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:14.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:15.098 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:15.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.149 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:15.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:15.250 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:15.297 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:15.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:15.399 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:15.423 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:15.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:15.525 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:15.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.568 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:15.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:15.617 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:15 smithi096 ceph-mon[98901]: pgmap v771: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 2579/82986 objects misplaced (3.108%); 9.2 MiB/s, 147 objects/s recovering 2024-09-13T14:52:15.617 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:15 smithi096 ceph-mon[102928]: pgmap v771: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 2579/82986 objects misplaced (3.108%); 9.2 MiB/s, 147 objects/s recovering 2024-09-13T14:52:15.669 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:15.694 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:15.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:15.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:15 smithi204 ceph-mon[87071]: pgmap v771: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 2579/82986 objects misplaced (3.108%); 9.2 MiB/s, 147 objects/s recovering 2024-09-13T14:52:15.796 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:15.823 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:15.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:15.925 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:15.978 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:15.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:16.036 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:52:16.037 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:52:16.080 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:16.127 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:16.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:16.198 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:52:16.198 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:52:16.229 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:16.278 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:16.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:16.380 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:16.414 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:16.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:16.516 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:16.562 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:16.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:16.664 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:16.692 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:16.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:16.793 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:16.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.836 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:16.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:16.907 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":772,"stamp":"2024-09-13T14:52:15.916634+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1786648549,"num_objects":27218,"num_object_clones":0,"num_object_copies":81654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2579,"num_objects_unfound":0,"num_objects_dirty":27218,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":213975,"num_write_kb":7700816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7216,"num_bytes_recovered":473579346,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128663,"ondisk_log_size":128663,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":299,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9349140,"kb_used_data":5372564,"kb_used_omap":35,"kb_used_meta":3976476,"kb_avail":740612076,"statfs":{"total":767960285184,"available":758386765824,"internally_reserved":0,"allocated":5501505536,"data_stored":5487519184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36830,"internal_metadata":4071911458},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-444268544,"num_objects":-6779,"num_object_clones":0,"num_object_copies":-20337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2016,"num_objects_unfound":0,"num_objects_dirty":-6779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6779,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1266,"num_bytes_recovered":82968576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006675"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":391,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.322931+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:52:06.322931+0000","last_peered":"2024-09-13T14:52:06.322931+0000","last_clean":"2024-09-13T14:52:06.322931+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:52:06.322931+0000","last_undegraded":"2024-09-13T14:52:06.322931+0000","last_fullsized":"2024-09-13T14:52:06.322931+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":227,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.317513+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:52:05.317513+0000","last_peered":"2024-09-13T14:52:05.317513+0000","last_clean":"2024-09-13T14:52:05.317513+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:52:05.317513+0000","last_undegraded":"2024-09-13T14:52:05.317513+0000","last_fullsized":"2024-09-13T14:52:05.317513+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":224,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.332941+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:52:05.332941+0000","last_peered":"2024-09-13T14:52:05.332941+0000","last_clean":"2024-09-13T14:52:05.332941+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:52:05.332941+0000","last_undegraded":"2024-09-13T14:52:05.332941+0000","last_fullsized":"2024-09-13T14:52:05.332941+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":405,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.327571+0000","last_change":"2024-09-13T14:51:44.196384+0000","last_active":"2024-09-13T14:52:06.327571+0000","last_peered":"2024-09-13T14:52:06.327571+0000","last_clean":"2024-09-13T14:52:06.327571+0000","last_became_active":"2024-09-13T14:51:44.196082+0000","last_became_peered":"2024-09-13T14:51:44.196082+0000","last_unstale":"2024-09-13T14:52:06.327571+0000","last_undegraded":"2024-09-13T14:52:06.327571+0000","last_fullsized":"2024-09-13T14:52:06.327571+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:11:03.757803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":398,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.328895+0000","last_change":"2024-09-13T14:51:44.199134+0000","last_active":"2024-09-13T14:52:06.328895+0000","last_peered":"2024-09-13T14:52:06.328895+0000","last_clean":"2024-09-13T14:52:06.328895+0000","last_became_active":"2024-09-13T14:51:44.198525+0000","last_became_peered":"2024-09-13T14:51:44.198525+0000","last_unstale":"2024-09-13T14:52:06.328895+0000","last_undegraded":"2024-09-13T14:52:06.328895+0000","last_fullsized":"2024-09-13T14:52:06.328895+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:44.009127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":228,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.317346+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T14:52:05.317346+0000","last_peered":"2024-09-13T14:52:05.317346+0000","last_clean":"2024-09-13T14:52:05.317346+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T14:52:05.317346+0000","last_undegraded":"2024-09-13T14:52:05.317346+0000","last_fullsized":"2024-09-13T14:52:05.317346+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":51,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.323025+0000","last_change":"2024-09-13T14:51:44.207246+0000","last_active":"2024-09-13T14:52:06.323025+0000","last_peered":"2024-09-13T14:52:06.323025+0000","last_clean":"2024-09-13T14:52:06.323025+0000","last_became_active":"2024-09-13T14:51:44.206637+0000","last_became_peered":"2024-09-13T14:51:44.206637+0000","last_unstale":"2024-09-13T14:52:06.323025+0000","last_undegraded":"2024-09-13T14:52:06.323025+0000","last_fullsized":"2024-09-13T14:52:06.323025+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:42.020301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":55,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.319449+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T14:52:05.319449+0000","last_peered":"2024-09-13T14:52:05.319449+0000","last_clean":"2024-09-13T14:52:05.319449+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T14:52:05.319449+0000","last_undegraded":"2024-09-13T14:52:05.319449+0000","last_fullsized":"2024-09-13T14:52:05.319449+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"261'13107","reported_seq":13201,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:13.027814+0000","last_change":"2024-09-13T14:48:32.204190+0000","last_active":"2024-09-13T14:52:13.027814+0000","last_peered":"2024-09-13T14:52:13.027814+0000","last_clean":"2024-09-13T14:52:13.027814+0000","last_became_active":"2024-09-13T14:48:32.203935+0000","last_became_peered":"2024-09-13T14:48:32.203935+0000","last_unstale":"2024-09-13T14:52:13.027814+0000","last_undegraded":"2024-09-13T14:52:13.027814+0000","last_fullsized":"2024-09-13T14:52:13.027814+0000","mapping_epoch":225,"log_start":"234'5200","ondisk_log_start":"234'5200","created":223,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7907,"log_dups_size":0,"ondisk_log_size":7907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:29:24.200434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112001024,"num_objects":1709,"num_object_clones":0,"num_object_copies":5127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13107,"num_write_kb":474112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":375,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.360169+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:52:06.360169+0000","last_peered":"2024-09-13T14:52:06.360169+0000","last_clean":"2024-09-13T14:52:06.360169+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:52:06.360169+0000","last_undegraded":"2024-09-13T14:52:06.360169+0000","last_fullsized":"2024-09-13T14:52:06.360169+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":247,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.332847+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T14:52:05.332847+0000","last_peered":"2024-09-13T14:52:05.332847+0000","last_clean":"2024-09-13T14:52:05.332847+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T14:52:05.332847+0000","last_undegraded":"2024-09-13T14:52:05.332847+0000","last_fullsized":"2024-09-13T14:52:05.332847+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"261'13241","reported_seq":13329,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:13.197661+0000","last_change":"2024-09-13T14:48:30.196807+0000","last_active":"2024-09-13T14:52:13.197661+0000","last_peered":"2024-09-13T14:52:13.197661+0000","last_clean":"2024-09-13T14:52:13.197661+0000","last_became_active":"2024-09-13T14:48:30.195433+0000","last_became_peered":"2024-09-13T14:48:30.195433+0000","last_unstale":"2024-09-13T14:52:13.197661+0000","last_undegraded":"2024-09-13T14:52:13.197661+0000","last_fullsized":"2024-09-13T14:52:13.197661+0000","mapping_epoch":223,"log_start":"235'6168","ondisk_log_start":"235'6168","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7073,"log_dups_size":0,"ondisk_log_size":7073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:31:16.623107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107413504,"num_objects":1639,"num_object_clones":0,"num_object_copies":4917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13241,"num_write_kb":476160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":227,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.317870+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:52:05.317870+0000","last_peered":"2024-09-13T14:52:05.317870+0000","last_clean":"2024-09-13T14:52:05.317870+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:52:05.317870+0000","last_undegraded":"2024-09-13T14:52:05.317870+0000","last_fullsized":"2024-09-13T14:52:05.317870+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":282,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.316720+0000","last_change":"2024-09-13T14:51:11.059483+0000","last_active":"2024-09-13T14:52:05.316720+0000","last_peered":"2024-09-13T14:52:05.316720+0000","last_clean":"2024-09-13T14:52:05.316720+0000","last_became_active":"2024-09-13T14:51:11.059262+0000","last_became_peered":"2024-09-13T14:51:11.059262+0000","last_unstale":"2024-09-13T14:52:05.316720+0000","last_undegraded":"2024-09-13T14:52:05.316720+0000","last_fullsized":"2024-09-13T14:52:05.316720+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:45:35.152631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"261'13187","reported_seq":13275,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:13.307698+0000","last_change":"2024-09-13T14:48:30.196414+0000","last_active":"2024-09-13T14:52:13.307698+0000","last_peered":"2024-09-13T14:52:13.307698+0000","last_clean":"2024-09-13T14:52:13.307698+0000","last_became_active":"2024-09-13T14:48:30.195744+0000","last_became_peered":"2024-09-13T14:48:30.195744+0000","last_unstale":"2024-09-13T14:52:13.307698+0000","last_undegraded":"2024-09-13T14:52:13.307698+0000","last_fullsized":"2024-09-13T14:52:13.307698+0000","mapping_epoch":223,"log_start":"234'5800","ondisk_log_start":"234'5800","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7387,"log_dups_size":0,"ondisk_log_size":7387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.893543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109641728,"num_objects":1673,"num_object_clones":0,"num_object_copies":5019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13187,"num_write_kb":475520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":237,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.317717+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T14:52:05.317717+0000","last_peered":"2024-09-13T14:52:05.317717+0000","last_clean":"2024-09-13T14:52:05.317717+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T14:52:05.317717+0000","last_undegraded":"2024-09-13T14:52:05.317717+0000","last_fullsized":"2024-09-13T14:52:05.317717+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":55,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.319585+0000","last_change":"2024-09-13T14:51:44.190879+0000","last_active":"2024-09-13T14:52:05.319585+0000","last_peered":"2024-09-13T14:52:05.319585+0000","last_clean":"2024-09-13T14:52:05.319585+0000","last_became_active":"2024-09-13T14:51:44.190617+0000","last_became_peered":"2024-09-13T14:51:44.190617+0000","last_unstale":"2024-09-13T14:52:05.319585+0000","last_undegraded":"2024-09-13T14:52:05.319585+0000","last_fullsized":"2024-09-13T14:52:05.319585+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:19.536298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"261'13551","reported_seq":13639,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:15.161285+0000","last_change":"2024-09-13T14:48:30.193569+0000","last_active":"2024-09-13T14:52:15.161285+0000","last_peered":"2024-09-13T14:52:15.161285+0000","last_clean":"2024-09-13T14:52:15.161285+0000","last_became_active":"2024-09-13T14:48:30.191205+0000","last_became_peered":"2024-09-13T14:48:30.191205+0000","last_unstale":"2024-09-13T14:52:15.161285+0000","last_undegraded":"2024-09-13T14:52:15.161285+0000","last_fullsized":"2024-09-13T14:52:15.161285+0000","mapping_epoch":223,"log_start":"233'4668","ondisk_log_start":"233'4668","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8883,"log_dups_size":0,"ondisk_log_size":8883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:27.050525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107020288,"num_objects":1633,"num_object_clones":0,"num_object_copies":4899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13551,"num_write_kb":485888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":245,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.332886+0000","last_change":"2024-09-13T14:48:32.206148+0000","last_active":"2024-09-13T14:52:05.332886+0000","last_peered":"2024-09-13T14:52:05.332886+0000","last_clean":"2024-09-13T14:52:05.332886+0000","last_became_active":"2024-09-13T14:48:32.205832+0000","last_became_peered":"2024-09-13T14:48:32.205832+0000","last_unstale":"2024-09-13T14:52:05.332886+0000","last_undegraded":"2024-09-13T14:52:05.332886+0000","last_fullsized":"2024-09-13T14:52:05.332886+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:55.026094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":385,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.327791+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:52:06.327791+0000","last_peered":"2024-09-13T14:52:06.327791+0000","last_clean":"2024-09-13T14:52:06.327791+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:52:06.327791+0000","last_undegraded":"2024-09-13T14:52:06.327791+0000","last_fullsized":"2024-09-13T14:52:06.327791+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"261'13425","reported_seq":13513,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:13.261298+0000","last_change":"2024-09-13T14:48:30.198101+0000","last_active":"2024-09-13T14:52:13.261298+0000","last_peered":"2024-09-13T14:52:13.261298+0000","last_clean":"2024-09-13T14:52:13.261298+0000","last_became_active":"2024-09-13T14:48:30.197770+0000","last_became_peered":"2024-09-13T14:48:30.197770+0000","last_unstale":"2024-09-13T14:52:13.261298+0000","last_undegraded":"2024-09-13T14:52:13.261298+0000","last_fullsized":"2024-09-13T14:52:13.261298+0000","mapping_epoch":223,"log_start":"235'6100","ondisk_log_start":"235'6100","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7325,"log_dups_size":0,"ondisk_log_size":7325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:26:49.717826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109510656,"num_objects":1671,"num_object_clones":0,"num_object_copies":5013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13425,"num_write_kb":483072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":408,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.317263+0000","last_change":"2024-09-13T14:51:01.996634+0000","last_active":"2024-09-13T14:52:05.317263+0000","last_peered":"2024-09-13T14:52:05.317263+0000","last_clean":"2024-09-13T14:52:05.317263+0000","last_became_active":"2024-09-13T14:51:01.996264+0000","last_became_peered":"2024-09-13T14:51:01.996264+0000","last_unstale":"2024-09-13T14:52:05.317263+0000","last_undegraded":"2024-09-13T14:52:05.317263+0000","last_fullsized":"2024-09-13T14:52:05.317263+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:30.083115+0000","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":388,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.322601+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:52:06.322601+0000","last_peered":"2024-09-13T14:52:06.322601+0000","last_clean":"2024-09-13T14:52:06.322601+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:52:06.322601+0000","last_undegraded":"2024-09-13T14:52:06.322601+0000","last_fullsized":"2024-09-13T14:52:06.322601+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":245,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.316312+0000","last_change":"2024-09-13T14:48:32.196993+0000","last_active":"2024-09-13T14:52:05.316312+0000","last_peered":"2024-09-13T14:52:05.316312+0000","last_clean":"2024-09-13T14:52:05.316312+0000","last_became_active":"2024-09-13T14:48:32.196681+0000","last_became_peered":"2024-09-13T14:48:32.196681+0000","last_unstale":"2024-09-13T14:52:05.316312+0000","last_undegraded":"2024-09-13T14:52:05.316312+0000","last_fullsized":"2024-09-13T14:52:05.316312+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:56.040494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"261'13373","reported_seq":13461,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:15.141927+0000","last_change":"2024-09-13T14:48:30.197998+0000","last_active":"2024-09-13T14:52:15.141927+0000","last_peered":"2024-09-13T14:52:15.141927+0000","last_clean":"2024-09-13T14:52:15.141927+0000","last_became_active":"2024-09-13T14:48:30.197697+0000","last_became_peered":"2024-09-13T14:48:30.197697+0000","last_unstale":"2024-09-13T14:52:15.141927+0000","last_undegraded":"2024-09-13T14:52:15.141927+0000","last_fullsized":"2024-09-13T14:52:15.141927+0000","mapping_epoch":223,"log_start":"233'4485","ondisk_log_start":"233'4485","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8888,"log_dups_size":0,"ondisk_log_size":8888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:17.417819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106889216,"num_objects":1631,"num_object_clones":0,"num_object_copies":4893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13373,"num_write_kb":480128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":168,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.332579+0000","last_change":"2024-09-13T14:40:27.868402+0000","last_active":"2024-09-13T14:52:05.332579+0000","last_peered":"2024-09-13T14:52:05.332579+0000","last_clean":"2024-09-13T14:52:05.332579+0000","last_became_active":"2024-09-13T14:40:22.966301+0000","last_became_peered":"2024-09-13T14:40:22.966301+0000","last_unstale":"2024-09-13T14:52:05.332579+0000","last_undegraded":"2024-09-13T14:52:05.332579+0000","last_fullsized":"2024-09-13T14:52:05.332579+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:18.660186+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":45,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.319701+0000","last_change":"2024-09-13T14:48:32.200521+0000","last_active":"2024-09-13T14:52:05.319701+0000","last_peered":"2024-09-13T14:52:05.319701+0000","last_clean":"2024-09-13T14:52:05.319701+0000","last_became_active":"2024-09-13T14:48:32.200256+0000","last_became_peered":"2024-09-13T14:48:32.200256+0000","last_unstale":"2024-09-13T14:52:05.319701+0000","last_undegraded":"2024-09-13T14:52:05.319701+0000","last_fullsized":"2024-09-13T14:52:05.319701+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:38.899629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":242,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.316643+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:52:05.316643+0000","last_peered":"2024-09-13T14:52:05.316643+0000","last_clean":"2024-09-13T14:52:05.316643+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:52:05.316643+0000","last_undegraded":"2024-09-13T14:52:05.316643+0000","last_fullsized":"2024-09-13T14:52:05.316643+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"261'13251","reported_seq":13297,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:11.621158+0000","last_change":"2024-09-13T14:48:32.204769+0000","last_active":"2024-09-13T14:52:11.621158+0000","last_peered":"2024-09-13T14:52:11.621158+0000","last_clean":"2024-09-13T14:52:11.621158+0000","last_became_active":"2024-09-13T14:48:32.204464+0000","last_became_peered":"2024-09-13T14:48:32.204464+0000","last_unstale":"2024-09-13T14:52:11.621158+0000","last_undegraded":"2024-09-13T14:52:11.621158+0000","last_fullsized":"2024-09-13T14:52:11.621158+0000","mapping_epoch":225,"log_start":"233'5100","ondisk_log_start":"233'5100","created":223,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8151,"log_dups_size":0,"ondisk_log_size":8151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:46.636678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117440536,"num_objects":1793,"num_object_clones":0,"num_object_copies":5379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1793,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13251,"num_write_kb":481345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":371,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.327471+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:52:06.327471+0000","last_peered":"2024-09-13T14:52:06.327471+0000","last_clean":"2024-09-13T14:52:06.327471+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:52:06.327471+0000","last_undegraded":"2024-09-13T14:52:06.327471+0000","last_fullsized":"2024-09-13T14:52:06.327471+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":245,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.316495+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T14:52:05.316495+0000","last_peered":"2024-09-13T14:52:05.316495+0000","last_clean":"2024-09-13T14:52:05.316495+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T14:52:05.316495+0000","last_undegraded":"2024-09-13T14:52:05.316495+0000","last_fullsized":"2024-09-13T14:52:05.316495+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"261'13176","reported_seq":20539,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:11.609198+0000","last_change":"2024-09-13T14:51:46.210926+0000","last_active":"2024-09-13T14:52:11.609198+0000","last_peered":"2024-09-13T14:52:11.609198+0000","last_clean":"2024-09-13T14:52:11.609198+0000","last_became_active":"2024-09-13T14:51:46.187150+0000","last_became_peered":"2024-09-13T14:51:46.187150+0000","last_unstale":"2024-09-13T14:52:11.609198+0000","last_undegraded":"2024-09-13T14:52:11.609198+0000","last_fullsized":"2024-09-13T14:52:11.609198+0000","mapping_epoch":254,"log_start":"233'5000","ondisk_log_start":"233'5000","created":223,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8176,"log_dups_size":0,"ondisk_log_size":8176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:35:28.550999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114950144,"num_objects":1754,"num_object_clones":0,"num_object_copies":5262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13176,"num_write_kb":477760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3647,"num_bytes_recovered":239009792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":171,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.332604+0000","last_change":"2024-09-13T14:48:32.205288+0000","last_active":"2024-09-13T14:52:05.332604+0000","last_peered":"2024-09-13T14:52:05.332604+0000","last_clean":"2024-09-13T14:52:05.332604+0000","last_became_active":"2024-09-13T14:48:32.204657+0000","last_became_peered":"2024-09-13T14:48:32.204657+0000","last_unstale":"2024-09-13T14:52:05.332604+0000","last_undegraded":"2024-09-13T14:52:05.332604+0000","last_fullsized":"2024-09-13T14:52:05.332604+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:52.513329+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":44,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.322059+0000","last_change":"2024-09-13T14:51:11.059644+0000","last_active":"2024-09-13T14:52:06.322059+0000","last_peered":"2024-09-13T14:52:06.322059+0000","last_clean":"2024-09-13T14:52:06.322059+0000","last_became_active":"2024-09-13T14:51:11.058874+0000","last_became_peered":"2024-09-13T14:51:11.058874+0000","last_unstale":"2024-09-13T14:52:06.322059+0000","last_undegraded":"2024-09-13T14:52:06.322059+0000","last_fullsized":"2024-09-13T14:52:06.322059+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:15:14.736476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":372,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.370112+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:52:06.370112+0000","last_peered":"2024-09-13T14:52:06.370112+0000","last_clean":"2024-09-13T14:52:06.370112+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:52:06.370112+0000","last_undegraded":"2024-09-13T14:52:06.370112+0000","last_fullsized":"2024-09-13T14:52:06.370112+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"261'13537","reported_seq":15523,"reported_epoch":261,"state":"active+remapped+backfilling","last_fresh":"2024-09-13T14:52:13.326350+0000","last_change":"2024-09-13T14:52:05.158594+0000","last_active":"2024-09-13T14:52:13.326350+0000","last_peered":"2024-09-13T14:52:13.326350+0000","last_clean":"2024-09-13T14:51:10.011417+0000","last_became_active":"2024-09-13T14:51:12.271357+0000","last_became_peered":"2024-09-13T14:51:12.271357+0000","last_unstale":"2024-09-13T14:52:13.326350+0000","last_undegraded":"2024-09-13T14:52:13.326350+0000","last_fullsized":"2024-09-13T14:52:13.326350+0000","mapping_epoch":245,"log_start":"235'6200","ondisk_log_start":"235'6200","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7337,"log_dups_size":0,"ondisk_log_size":7337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106496000,"num_objects":1625,"num_object_clones":0,"num_object_copies":4875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":807,"num_objects_unfound":0,"num_objects_dirty":1625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13537,"num_write_kb":485184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":947,"num_bytes_recovered":62062592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":1625}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":389,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.317401+0000","last_change":"2024-09-13T14:48:32.204164+0000","last_active":"2024-09-13T14:52:05.317401+0000","last_peered":"2024-09-13T14:52:05.317401+0000","last_clean":"2024-09-13T14:52:05.317401+0000","last_became_active":"2024-09-13T14:48:32.203815+0000","last_became_peered":"2024-09-13T14:48:32.203815+0000","last_unstale":"2024-09-13T14:52:05.317401+0000","last_undegraded":"2024-09-13T14:52:05.317401+0000","last_fullsized":"2024-09-13T14:52:05.317401+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:22.639760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:22.639760+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:21.985178+0000","scrub_duration":0.0051120610000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":787,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.332745+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:52:05.332745+0000","last_peered":"2024-09-13T14:52:05.332745+0000","last_clean":"2024-09-13T14:52:05.332745+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:52:05.332745+0000","last_undegraded":"2024-09-13T14:52:05.332745+0000","last_fullsized":"2024-09-13T14:52:05.332745+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":44,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.322392+0000","last_change":"2024-09-13T14:51:11.059953+0000","last_active":"2024-09-13T14:52:06.322392+0000","last_peered":"2024-09-13T14:52:06.322392+0000","last_clean":"2024-09-13T14:52:06.322392+0000","last_became_active":"2024-09-13T14:51:11.059396+0000","last_became_peered":"2024-09-13T14:51:11.059396+0000","last_unstale":"2024-09-13T14:52:06.322392+0000","last_undegraded":"2024-09-13T14:52:06.322392+0000","last_fullsized":"2024-09-13T14:52:06.322392+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:11:51.119142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":45,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.319790+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T14:52:05.319790+0000","last_peered":"2024-09-13T14:52:05.319790+0000","last_clean":"2024-09-13T14:52:05.319790+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T14:52:05.319790+0000","last_undegraded":"2024-09-13T14:52:05.319790+0000","last_fullsized":"2024-09-13T14:52:05.319790+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"261'13398","reported_seq":13444,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:11.810227+0000","last_change":"2024-09-13T14:48:30.197655+0000","last_active":"2024-09-13T14:52:11.810227+0000","last_peered":"2024-09-13T14:52:11.810227+0000","last_clean":"2024-09-13T14:52:11.810227+0000","last_became_active":"2024-09-13T14:48:30.197442+0000","last_became_peered":"2024-09-13T14:48:30.197442+0000","last_unstale":"2024-09-13T14:52:11.810227+0000","last_undegraded":"2024-09-13T14:52:11.810227+0000","last_fullsized":"2024-09-13T14:52:11.810227+0000","mapping_epoch":223,"log_start":"234'5632","ondisk_log_start":"234'5632","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7766,"log_dups_size":0,"ondisk_log_size":7766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:44.452905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114950144,"num_objects":1754,"num_object_clones":0,"num_object_copies":5262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13398,"num_write_kb":484864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":511,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.368959+0000","last_change":"2024-09-13T14:48:32.204587+0000","last_active":"2024-09-13T14:52:06.368959+0000","last_peered":"2024-09-13T14:52:06.368959+0000","last_clean":"2024-09-13T14:52:06.368959+0000","last_became_active":"2024-09-13T14:48:32.204153+0000","last_became_peered":"2024-09-13T14:48:32.204153+0000","last_unstale":"2024-09-13T14:52:06.368959+0000","last_undegraded":"2024-09-13T14:52:06.368959+0000","last_fullsized":"2024-09-13T14:52:06.368959+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.784263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.784263+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:11:12.651038+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":472,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.317319+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:52:05.317319+0000","last_peered":"2024-09-13T14:52:05.317319+0000","last_clean":"2024-09-13T14:52:05.317319+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:52:05.317319+0000","last_undegraded":"2024-09-13T14:52:05.317319+0000","last_fullsized":"2024-09-13T14:52:05.317319+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":406,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.327515+0000","last_change":"2024-09-13T14:51:11.061677+0000","last_active":"2024-09-13T14:52:06.327515+0000","last_peered":"2024-09-13T14:52:06.327515+0000","last_clean":"2024-09-13T14:52:06.327515+0000","last_became_active":"2024-09-13T14:51:11.061205+0000","last_became_peered":"2024-09-13T14:51:11.061205+0000","last_unstale":"2024-09-13T14:52:06.327515+0000","last_undegraded":"2024-09-13T14:52:06.327515+0000","last_fullsized":"2024-09-13T14:52:06.327515+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:38.521784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":410,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.328952+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T14:52:06.328952+0000","last_peered":"2024-09-13T14:52:06.328952+0000","last_clean":"2024-09-13T14:52:06.328952+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T14:52:06.328952+0000","last_undegraded":"2024-09-13T14:52:06.328952+0000","last_fullsized":"2024-09-13T14:52:06.328952+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"261'13469","reported_seq":13557,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:13.321976+0000","last_change":"2024-09-13T14:48:30.196036+0000","last_active":"2024-09-13T14:52:13.321976+0000","last_peered":"2024-09-13T14:52:13.321976+0000","last_clean":"2024-09-13T14:52:13.321976+0000","last_became_active":"2024-09-13T14:48:30.195478+0000","last_became_peered":"2024-09-13T14:48:30.195478+0000","last_unstale":"2024-09-13T14:52:13.321976+0000","last_undegraded":"2024-09-13T14:52:13.321976+0000","last_fullsized":"2024-09-13T14:52:13.321976+0000","mapping_epoch":223,"log_start":"235'6100","ondisk_log_start":"235'6100","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7369,"log_dups_size":0,"ondisk_log_size":7369,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:21:44.624753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111345664,"num_objects":1699,"num_object_clones":0,"num_object_copies":5097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13469,"num_write_kb":485376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":18,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.317400+0000","last_change":"2024-09-13T14:51:44.190446+0000","last_active":"2024-09-13T14:52:05.317400+0000","last_peered":"2024-09-13T14:52:05.317400+0000","last_clean":"2024-09-13T14:52:05.317400+0000","last_became_active":"2024-09-13T14:51:44.190115+0000","last_became_peered":"2024-09-13T14:51:44.190115+0000","last_unstale":"2024-09-13T14:52:05.317400+0000","last_undegraded":"2024-09-13T14:52:05.317400+0000","last_fullsized":"2024-09-13T14:52:05.317400+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:45:49.425578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:45:49.425578+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:43.724060+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":859,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.359692+0000","last_change":"2024-09-13T14:51:11.072066+0000","last_active":"2024-09-13T14:52:06.359692+0000","last_peered":"2024-09-13T14:52:06.359692+0000","last_clean":"2024-09-13T14:52:06.359692+0000","last_became_active":"2024-09-13T14:51:11.061307+0000","last_became_peered":"2024-09-13T14:51:11.061307+0000","last_unstale":"2024-09-13T14:52:06.359692+0000","last_undegraded":"2024-09-13T14:52:06.359692+0000","last_fullsized":"2024-09-13T14:52:06.359692+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:06.774106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":387,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.369432+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T14:52:06.369432+0000","last_peered":"2024-09-13T14:52:06.369432+0000","last_clean":"2024-09-13T14:52:06.369432+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T14:52:06.369432+0000","last_undegraded":"2024-09-13T14:52:06.369432+0000","last_fullsized":"2024-09-13T14:52:06.369432+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":405,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.370032+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:52:06.370032+0000","last_peered":"2024-09-13T14:52:06.370032+0000","last_clean":"2024-09-13T14:52:06.370032+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:52:06.370032+0000","last_undegraded":"2024-09-13T14:52:06.370032+0000","last_fullsized":"2024-09-13T14:52:06.370032+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"261'13360","reported_seq":13416,"reported_epoch":261,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:52:11.669510+0000","last_change":"2024-09-13T14:51:12.278325+0000","last_active":"2024-09-13T14:52:11.669510+0000","last_peered":"2024-09-13T14:52:11.669510+0000","last_clean":"2024-09-13T14:51:09.999835+0000","last_became_active":"2024-09-13T14:51:12.256335+0000","last_became_peered":"2024-09-13T14:51:12.256335+0000","last_unstale":"2024-09-13T14:52:11.669510+0000","last_undegraded":"2024-09-13T14:52:11.669510+0000","last_fullsized":"2024-09-13T14:52:11.669510+0000","mapping_epoch":245,"log_start":"234'5200","ondisk_log_start":"234'5200","created":223,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8160,"log_dups_size":0,"ondisk_log_size":8160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:40:16.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116129792,"num_objects":1772,"num_object_clones":0,"num_object_copies":5316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1772,"num_objects_unfound":0,"num_objects_dirty":1772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13360,"num_write_kb":484224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,1],"avail_no_missing":["6","0","1"],"object_location_counts":[{"shards":"0,1,6","objects":1772}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":161,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.332564+0000","last_change":"2024-09-13T14:40:26.896691+0000","last_active":"2024-09-13T14:52:05.332564+0000","last_peered":"2024-09-13T14:52:05.332564+0000","last_clean":"2024-09-13T14:52:05.332564+0000","last_became_active":"2024-09-13T14:40:22.949772+0000","last_became_peered":"2024-09-13T14:40:22.949772+0000","last_unstale":"2024-09-13T14:52:05.332564+0000","last_undegraded":"2024-09-13T14:52:05.332564+0000","last_fullsized":"2024-09-13T14:52:05.332564+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:26.896613+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:26.896613+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:01.615433+0000","scrub_duration":0.004120828,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":887,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.332757+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T14:52:05.332757+0000","last_peered":"2024-09-13T14:52:05.332757+0000","last_clean":"2024-09-13T14:52:05.332757+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T14:52:05.332757+0000","last_undegraded":"2024-09-13T14:52:05.332757+0000","last_fullsized":"2024-09-13T14:52:05.332757+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":246,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.316539+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T14:52:05.316539+0000","last_peered":"2024-09-13T14:52:05.316539+0000","last_clean":"2024-09-13T14:52:05.316539+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T14:52:05.316539+0000","last_undegraded":"2024-09-13T14:52:05.316539+0000","last_fullsized":"2024-09-13T14:52:05.316539+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":405,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.369401+0000","last_change":"2024-09-13T14:51:11.060994+0000","last_active":"2024-09-13T14:52:06.369401+0000","last_peered":"2024-09-13T14:52:06.369401+0000","last_clean":"2024-09-13T14:52:06.369401+0000","last_became_active":"2024-09-13T14:51:11.060666+0000","last_became_peered":"2024-09-13T14:51:11.060666+0000","last_unstale":"2024-09-13T14:52:06.369401+0000","last_undegraded":"2024-09-13T14:52:06.369401+0000","last_fullsized":"2024-09-13T14:52:06.369401+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:40:15.828766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"261'13154","reported_seq":18244,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:11.830752+0000","last_change":"2024-09-13T14:52:06.352163+0000","last_active":"2024-09-13T14:52:11.830752+0000","last_peered":"2024-09-13T14:52:11.830752+0000","last_clean":"2024-09-13T14:52:11.830752+0000","last_became_active":"2024-09-13T14:52:06.323005+0000","last_became_peered":"2024-09-13T14:52:06.323005+0000","last_unstale":"2024-09-13T14:52:11.830752+0000","last_undegraded":"2024-09-13T14:52:11.830752+0000","last_fullsized":"2024-09-13T14:52:11.830752+0000","mapping_epoch":260,"log_start":"233'4800","ondisk_log_start":"233'4800","created":223,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8354,"log_dups_size":0,"ondisk_log_size":8354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:30.406030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111673344,"num_objects":1704,"num_object_clones":0,"num_object_copies":5112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13154,"num_write_kb":475456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2504,"num_bytes_recovered":164102144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":65,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.319868+0000","last_change":"2024-09-13T14:51:11.061598+0000","last_active":"2024-09-13T14:52:05.319868+0000","last_peered":"2024-09-13T14:52:05.319868+0000","last_clean":"2024-09-13T14:52:05.319868+0000","last_became_active":"2024-09-13T14:51:11.061164+0000","last_became_peered":"2024-09-13T14:51:11.061164+0000","last_unstale":"2024-09-13T14:52:05.319868+0000","last_undegraded":"2024-09-13T14:52:05.319868+0000","last_fullsized":"2024-09-13T14:52:05.319868+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:48:34.095505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:48:34.095505+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:03.076025+0000","scrub_duration":0.001382681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":242,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.333109+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:52:05.333109+0000","last_peered":"2024-09-13T14:52:05.333109+0000","last_clean":"2024-09-13T14:52:05.333109+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:52:05.333109+0000","last_undegraded":"2024-09-13T14:52:05.333109+0000","last_fullsized":"2024-09-13T14:52:05.333109+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":890,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.369570+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:52:06.369570+0000","last_peered":"2024-09-13T14:52:06.369570+0000","last_clean":"2024-09-13T14:52:06.369570+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:52:06.369570+0000","last_undegraded":"2024-09-13T14:52:06.369570+0000","last_fullsized":"2024-09-13T14:52:06.369570+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":362,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.370216+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:52:06.370216+0000","last_peered":"2024-09-13T14:52:06.370216+0000","last_clean":"2024-09-13T14:52:06.370216+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:52:06.370216+0000","last_undegraded":"2024-09-13T14:52:06.370216+0000","last_fullsized":"2024-09-13T14:52:06.370216+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"261'13009","reported_seq":13055,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:11.816763+0000","last_change":"2024-09-13T14:48:30.192664+0000","last_active":"2024-09-13T14:52:11.816763+0000","last_peered":"2024-09-13T14:52:11.816763+0000","last_clean":"2024-09-13T14:52:11.816763+0000","last_became_active":"2024-09-13T14:48:30.192434+0000","last_became_peered":"2024-09-13T14:48:30.192434+0000","last_unstale":"2024-09-13T14:52:11.816763+0000","last_undegraded":"2024-09-13T14:52:11.816763+0000","last_fullsized":"2024-09-13T14:52:11.816763+0000","mapping_epoch":223,"log_start":"232'3300","ondisk_log_start":"232'3300","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":9709,"log_dups_size":0,"ondisk_log_size":9709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:58.920518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113967104,"num_objects":1739,"num_object_clones":0,"num_object_copies":5217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13009,"num_write_kb":471936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":55,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.319986+0000","last_change":"2024-09-13T14:51:11.060132+0000","last_active":"2024-09-13T14:52:05.319986+0000","last_peered":"2024-09-13T14:52:05.319986+0000","last_clean":"2024-09-13T14:52:05.319986+0000","last_became_active":"2024-09-13T14:51:11.059785+0000","last_became_peered":"2024-09-13T14:51:11.059785+0000","last_unstale":"2024-09-13T14:52:05.319986+0000","last_undegraded":"2024-09-13T14:52:05.319986+0000","last_fullsized":"2024-09-13T14:52:05.319986+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.334379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.334379+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:44.298431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":387,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.369481+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T14:52:06.369481+0000","last_peered":"2024-09-13T14:52:06.369481+0000","last_clean":"2024-09-13T14:52:06.369481+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T14:52:06.369481+0000","last_undegraded":"2024-09-13T14:52:06.369481+0000","last_fullsized":"2024-09-13T14:52:06.369481+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":474,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.317799+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:52:05.317799+0000","last_peered":"2024-09-13T14:52:05.317799+0000","last_clean":"2024-09-13T14:52:05.317799+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:52:05.317799+0000","last_undegraded":"2024-09-13T14:52:05.317799+0000","last_fullsized":"2024-09-13T14:52:05.317799+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":374,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.327717+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:52:06.327717+0000","last_peered":"2024-09-13T14:52:06.327717+0000","last_clean":"2024-09-13T14:52:06.327717+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:52:06.327717+0000","last_undegraded":"2024-09-13T14:52:06.327717+0000","last_fullsized":"2024-09-13T14:52:06.327717+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"261'13523","reported_seq":13611,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:13.188422+0000","last_change":"2024-09-13T14:48:30.197226+0000","last_active":"2024-09-13T14:52:13.188422+0000","last_peered":"2024-09-13T14:52:13.188422+0000","last_clean":"2024-09-13T14:52:13.188422+0000","last_became_active":"2024-09-13T14:48:30.195243+0000","last_became_peered":"2024-09-13T14:48:30.195243+0000","last_unstale":"2024-09-13T14:52:13.188422+0000","last_undegraded":"2024-09-13T14:52:13.188422+0000","last_fullsized":"2024-09-13T14:52:13.188422+0000","mapping_epoch":223,"log_start":"235'6517","ondisk_log_start":"235'6517","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7006,"log_dups_size":0,"ondisk_log_size":7006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:46:06.507375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110034944,"num_objects":1679,"num_object_clones":0,"num_object_copies":5037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13523,"num_write_kb":486464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":192,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.316188+0000","last_change":"2024-09-13T14:51:01.998101+0000","last_active":"2024-09-13T14:52:05.316188+0000","last_peered":"2024-09-13T14:52:05.316188+0000","last_clean":"2024-09-13T14:52:05.316188+0000","last_became_active":"2024-09-13T14:51:01.997688+0000","last_became_peered":"2024-09-13T14:51:01.997688+0000","last_unstale":"2024-09-13T14:52:05.316188+0000","last_undegraded":"2024-09-13T14:52:05.316188+0000","last_fullsized":"2024-09-13T14:52:05.316188+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:40:30.500046+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:02:51.444049+0000","scrub_duration":0.00248832,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":388,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.323165+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:52:06.323165+0000","last_peered":"2024-09-13T14:52:06.323165+0000","last_clean":"2024-09-13T14:52:06.323165+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:52:06.323165+0000","last_undegraded":"2024-09-13T14:52:06.323165+0000","last_fullsized":"2024-09-13T14:52:06.323165+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":744,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.369241+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T14:52:06.369241+0000","last_peered":"2024-09-13T14:52:06.369241+0000","last_clean":"2024-09-13T14:52:06.369241+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T14:52:06.369241+0000","last_undegraded":"2024-09-13T14:52:06.369241+0000","last_fullsized":"2024-09-13T14:52:06.369241+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":51,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.322105+0000","last_change":"2024-09-13T14:51:44.203186+0000","last_active":"2024-09-13T14:52:06.322105+0000","last_peered":"2024-09-13T14:52:06.322105+0000","last_clean":"2024-09-13T14:52:06.322105+0000","last_became_active":"2024-09-13T14:51:44.202851+0000","last_became_peered":"2024-09-13T14:51:44.202851+0000","last_unstale":"2024-09-13T14:52:06.322105+0000","last_undegraded":"2024-09-13T14:52:06.322105+0000","last_fullsized":"2024-09-13T14:52:06.322105+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:05:49.173340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"261'13307","reported_seq":13395,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:13.182395+0000","last_change":"2024-09-13T14:48:30.195027+0000","last_active":"2024-09-13T14:52:13.182395+0000","last_peered":"2024-09-13T14:52:13.182395+0000","last_clean":"2024-09-13T14:52:13.182395+0000","last_became_active":"2024-09-13T14:48:30.194672+0000","last_became_peered":"2024-09-13T14:48:30.194672+0000","last_unstale":"2024-09-13T14:52:13.182395+0000","last_undegraded":"2024-09-13T14:52:13.182395+0000","last_fullsized":"2024-09-13T14:52:13.182395+0000","mapping_epoch":223,"log_start":"235'6267","ondisk_log_start":"235'6267","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7040,"log_dups_size":0,"ondisk_log_size":7040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:38:50.182084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112394240,"num_objects":1715,"num_object_clones":0,"num_object_copies":5145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13307,"num_write_kb":480704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":318,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.322721+0000","last_change":"2024-09-13T14:40:24.124750+0000","last_active":"2024-09-13T14:52:06.322721+0000","last_peered":"2024-09-13T14:52:06.322721+0000","last_clean":"2024-09-13T14:52:06.322721+0000","last_became_active":"2024-09-13T14:40:22.958820+0000","last_became_peered":"2024-09-13T14:40:22.958820+0000","last_unstale":"2024-09-13T14:52:06.322721+0000","last_undegraded":"2024-09-13T14:52:06.322721+0000","last_fullsized":"2024-09-13T14:52:06.322721+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:24.124531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:24.124531+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:47:30.267886+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":396,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.328827+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:52:06.328827+0000","last_peered":"2024-09-13T14:52:06.328827+0000","last_clean":"2024-09-13T14:52:06.328827+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:52:06.328827+0000","last_undegraded":"2024-09-13T14:52:06.328827+0000","last_fullsized":"2024-09-13T14:52:06.328827+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1074,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.320090+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T14:52:05.320090+0000","last_peered":"2024-09-13T14:52:05.320090+0000","last_clean":"2024-09-13T14:52:05.320090+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T14:52:05.320090+0000","last_undegraded":"2024-09-13T14:52:05.320090+0000","last_fullsized":"2024-09-13T14:52:05.320090+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":228,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.316596+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T14:52:05.316596+0000","last_peered":"2024-09-13T14:52:05.316596+0000","last_clean":"2024-09-13T14:52:05.316596+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T14:52:05.316596+0000","last_undegraded":"2024-09-13T14:52:05.316596+0000","last_fullsized":"2024-09-13T14:52:05.316596+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":245,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.316523+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T14:52:05.316523+0000","last_peered":"2024-09-13T14:52:05.316523+0000","last_clean":"2024-09-13T14:52:05.316523+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T14:52:05.316523+0000","last_undegraded":"2024-09-13T14:52:05.316523+0000","last_fullsized":"2024-09-13T14:52:05.316523+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":403,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.327771+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T14:52:06.327771+0000","last_peered":"2024-09-13T14:52:06.327771+0000","last_clean":"2024-09-13T14:52:06.327771+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T14:52:06.327771+0000","last_undegraded":"2024-09-13T14:52:06.327771+0000","last_fullsized":"2024-09-13T14:52:06.327771+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":386,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.370078+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T14:52:06.370078+0000","last_peered":"2024-09-13T14:52:06.370078+0000","last_clean":"2024-09-13T14:52:06.370078+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T14:52:06.370078+0000","last_undegraded":"2024-09-13T14:52:06.370078+0000","last_fullsized":"2024-09-13T14:52:06.370078+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":241,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.317636+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:52:05.317636+0000","last_peered":"2024-09-13T14:52:05.317636+0000","last_clean":"2024-09-13T14:52:05.317636+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:52:05.317636+0000","last_undegraded":"2024-09-13T14:52:05.317636+0000","last_fullsized":"2024-09-13T14:52:05.317636+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":225,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.335620+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:52:05.335620+0000","last_peered":"2024-09-13T14:52:05.335620+0000","last_clean":"2024-09-13T14:52:05.335620+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:52:05.335620+0000","last_undegraded":"2024-09-13T14:52:05.335620+0000","last_fullsized":"2024-09-13T14:52:05.335620+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":356,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.322438+0000","last_change":"2024-09-13T14:48:32.200953+0000","last_active":"2024-09-13T14:52:06.322438+0000","last_peered":"2024-09-13T14:52:06.322438+0000","last_clean":"2024-09-13T14:52:06.322438+0000","last_became_active":"2024-09-13T14:48:32.200634+0000","last_became_peered":"2024-09-13T14:48:32.200634+0000","last_unstale":"2024-09-13T14:52:06.322438+0000","last_undegraded":"2024-09-13T14:52:06.322438+0000","last_fullsized":"2024-09-13T14:52:06.322438+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:18.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":383,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.327985+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:52:06.327985+0000","last_peered":"2024-09-13T14:52:06.327985+0000","last_clean":"2024-09-13T14:52:06.327985+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:52:06.327985+0000","last_undegraded":"2024-09-13T14:52:06.327985+0000","last_fullsized":"2024-09-13T14:52:06.327985+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":405,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.322342+0000","last_change":"2024-09-13T14:48:32.201812+0000","last_active":"2024-09-13T14:52:06.322342+0000","last_peered":"2024-09-13T14:52:06.322342+0000","last_clean":"2024-09-13T14:52:06.322342+0000","last_became_active":"2024-09-13T14:48:32.201486+0000","last_became_peered":"2024-09-13T14:48:32.201486+0000","last_unstale":"2024-09-13T14:52:06.322342+0000","last_undegraded":"2024-09-13T14:52:06.322342+0000","last_fullsized":"2024-09-13T14:52:06.322342+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:23:29.693906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":45,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.319940+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T14:52:05.319940+0000","last_peered":"2024-09-13T14:52:05.319940+0000","last_clean":"2024-09-13T14:52:05.319940+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T14:52:05.319940+0000","last_undegraded":"2024-09-13T14:52:05.319940+0000","last_fullsized":"2024-09-13T14:52:05.319940+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":408,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.322209+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:52:06.322209+0000","last_peered":"2024-09-13T14:52:06.322209+0000","last_clean":"2024-09-13T14:52:06.322209+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:52:06.322209+0000","last_undegraded":"2024-09-13T14:52:06.322209+0000","last_fullsized":"2024-09-13T14:52:06.322209+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":407,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.328891+0000","last_change":"2024-09-13T14:51:44.198771+0000","last_active":"2024-09-13T14:52:06.328891+0000","last_peered":"2024-09-13T14:52:06.328891+0000","last_clean":"2024-09-13T14:52:06.328891+0000","last_became_active":"2024-09-13T14:51:44.198466+0000","last_became_peered":"2024-09-13T14:51:44.198466+0000","last_unstale":"2024-09-13T14:52:06.328891+0000","last_undegraded":"2024-09-13T14:52:06.328891+0000","last_fullsized":"2024-09-13T14:52:06.328891+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:41:16.099954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":372,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.368996+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:52:06.368996+0000","last_peered":"2024-09-13T14:52:06.368996+0000","last_clean":"2024-09-13T14:52:06.368996+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:52:06.368996+0000","last_undegraded":"2024-09-13T14:52:06.368996+0000","last_fullsized":"2024-09-13T14:52:06.368996+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":404,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.322800+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:52:06.322800+0000","last_peered":"2024-09-13T14:52:06.322800+0000","last_clean":"2024-09-13T14:52:06.322800+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:52:06.322800+0000","last_undegraded":"2024-09-13T14:52:06.322800+0000","last_fullsized":"2024-09-13T14:52:06.322800+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":244,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.316212+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:52:05.316212+0000","last_peered":"2024-09-13T14:52:05.316212+0000","last_clean":"2024-09-13T14:52:05.316212+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:52:05.316212+0000","last_undegraded":"2024-09-13T14:52:05.316212+0000","last_fullsized":"2024-09-13T14:52:05.316212+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":48,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.359627+0000","last_change":"2024-09-13T14:51:02.029301+0000","last_active":"2024-09-13T14:52:06.359627+0000","last_peered":"2024-09-13T14:52:06.359627+0000","last_clean":"2024-09-13T14:52:06.359627+0000","last_became_active":"2024-09-13T14:51:02.029032+0000","last_became_peered":"2024-09-13T14:51:02.029032+0000","last_unstale":"2024-09-13T14:52:06.359627+0000","last_undegraded":"2024-09-13T14:52:06.359627+0000","last_fullsized":"2024-09-13T14:52:06.359627+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:40.150320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":376,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.327694+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:52:06.327694+0000","last_peered":"2024-09-13T14:52:06.327694+0000","last_clean":"2024-09-13T14:52:06.327694+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:52:06.327694+0000","last_undegraded":"2024-09-13T14:52:06.327694+0000","last_fullsized":"2024-09-13T14:52:06.327694+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":242,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.316460+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:52:05.316460+0000","last_peered":"2024-09-13T14:52:05.316460+0000","last_clean":"2024-09-13T14:52:05.316460+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:52:05.316460+0000","last_undegraded":"2024-09-13T14:52:05.316460+0000","last_fullsized":"2024-09-13T14:52:05.316460+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":227,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.332453+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:52:05.332453+0000","last_peered":"2024-09-13T14:52:05.332453+0000","last_clean":"2024-09-13T14:52:05.332453+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:52:05.332453+0000","last_undegraded":"2024-09-13T14:52:05.332453+0000","last_fullsized":"2024-09-13T14:52:05.332453+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":242,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.316286+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:52:05.316286+0000","last_peered":"2024-09-13T14:52:05.316286+0000","last_clean":"2024-09-13T14:52:05.316286+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:52:05.316286+0000","last_undegraded":"2024-09-13T14:52:05.316286+0000","last_fullsized":"2024-09-13T14:52:05.316286+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":266,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.332965+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T14:52:05.332965+0000","last_peered":"2024-09-13T14:52:05.332965+0000","last_clean":"2024-09-13T14:52:05.332965+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T14:52:05.332965+0000","last_undegraded":"2024-09-13T14:52:05.332965+0000","last_fullsized":"2024-09-13T14:52:05.332965+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":216,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.332561+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:52:05.332561+0000","last_peered":"2024-09-13T14:52:05.332561+0000","last_clean":"2024-09-13T14:52:05.332561+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:52:05.332561+0000","last_undegraded":"2024-09-13T14:52:05.332561+0000","last_fullsized":"2024-09-13T14:52:05.332561+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":412,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.323133+0000","last_change":"2024-09-13T14:48:32.202404+0000","last_active":"2024-09-13T14:52:06.323133+0000","last_peered":"2024-09-13T14:52:06.323133+0000","last_clean":"2024-09-13T14:52:06.323133+0000","last_became_active":"2024-09-13T14:48:32.202064+0000","last_became_peered":"2024-09-13T14:48:32.202064+0000","last_unstale":"2024-09-13T14:52:06.323133+0000","last_undegraded":"2024-09-13T14:52:06.323133+0000","last_fullsized":"2024-09-13T14:52:06.323133+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:25.674819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":248,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.315949+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T14:52:05.315949+0000","last_peered":"2024-09-13T14:52:05.315949+0000","last_clean":"2024-09-13T14:52:05.315949+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T14:52:05.315949+0000","last_undegraded":"2024-09-13T14:52:05.315949+0000","last_fullsized":"2024-09-13T14:52:05.315949+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":427,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.327983+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T14:52:06.327983+0000","last_peered":"2024-09-13T14:52:06.327983+0000","last_clean":"2024-09-13T14:52:06.327983+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T14:52:06.327983+0000","last_undegraded":"2024-09-13T14:52:06.327983+0000","last_fullsized":"2024-09-13T14:52:06.327983+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1781858328,"num_objects":27190,"num_object_clones":0,"num_object_copies":81570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2579,"num_objects_unfound":0,"num_objects_dirty":27190,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":213068,"num_write_kb":7688193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7098,"num_bytes_recovered":465174528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5408632832,"data_stored":5408620616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126531,"ondisk_log_size":126531,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":16172,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397819,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":871372,"kb_used_data":561540,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92873780,"statfs":{"total":95995035648,"available":95102750720,"internally_reserved":0,"allocated":575016960,"data_stored":573261706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594048,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":887176,"kb_used_data":573184,"kb_used_omap":1,"kb_used_meta":313982,"kb_avail":92857976,"statfs":{"total":95995035648,"available":95086567424,"internally_reserved":0,"allocated":586940416,"data_stored":585188854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":321517912},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790277,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1148160,"kb_used_data":562424,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":92596992,"statfs":{"total":95995035648,"available":94819319808,"internally_reserved":0,"allocated":575922176,"data_stored":574175223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986506,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1162944,"kb_used_data":568888,"kb_used_omap":1,"kb_used_meta":594046,"kb_avail":92582208,"statfs":{"total":95995035648,"available":94804180992,"internally_reserved":0,"allocated":582541312,"data_stored":580793546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":608303448},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598954,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1323444,"kb_used_data":816364,"kb_used_omap":6,"kb_used_meta":507065,"kb_avail":92421708,"statfs":{"total":95995035648,"available":94639828992,"internally_reserved":0,"allocated":835956736,"data_stored":834221440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":519235552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762481,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1225344,"kb_used_data":620984,"kb_used_omap":8,"kb_used_meta":604343,"kb_avail":92519808,"statfs":{"total":95995035648,"available":94740283392,"internally_reserved":0,"allocated":635887616,"data_stored":634140658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":618847690},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926007,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1341500,"kb_used_data":788340,"kb_used_omap":7,"kb_used_meta":553144,"kb_avail":92403652,"statfs":{"total":95995035648,"available":94621339648,"internally_reserved":0,"allocated":807260160,"data_stored":805516415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":566420169},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122247,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1389200,"kb_used_data":880840,"kb_used_omap":7,"kb_used_meta":508344,"kb_avail":92355952,"statfs":{"total":95995035648,"available":94572494848,"internally_reserved":0,"allocated":901980160,"data_stored":900221342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":520544847},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":882970624,"data_stored":882966552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":784076800,"data_stored":784072728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":629211136,"data_stored":629211136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":822542336,"data_stored":822542336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":574423040,"data_stored":574423040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":567083008,"data_stored":567083008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":577376256,"data_stored":577372184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":570949632,"data_stored":570949632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:52:16.911 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:52:16.937 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:16.962 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:17.064 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:17.088 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:17.176 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:17.190 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:17.214 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:17.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:17.316 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:17.371 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:17.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:17.473 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:17.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.500 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:17.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:17.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:17 smithi096 ceph-mon[98901]: pgmap v772: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 2579/81654 objects misplaced (3.158%); 7.9 MiB/s, 126 objects/s recovering 2024-09-13T14:52:17.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:17 smithi096 ceph-mon[98901]: from='client.55261 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:17.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:17 smithi096 ceph-mon[102928]: pgmap v772: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 2579/81654 objects misplaced (3.158%); 7.9 MiB/s, 126 objects/s recovering 2024-09-13T14:52:17.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:17 smithi096 ceph-mon[102928]: from='client.55261 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:17.601 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:17.626 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:17.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:17.642 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:52:17.676 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:17.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:17 smithi204 ceph-mon[87071]: pgmap v772: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 2579/81654 objects misplaced (3.158%); 7.9 MiB/s, 126 objects/s recovering 2024-09-13T14:52:17.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:17 smithi204 ceph-mon[87071]: from='client.55261 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:17.728 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:17.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.783 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:17.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:17.884 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:17.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.937 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:17.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:18.038 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:18.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.067 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:18.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:18.168 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:18.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.214 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:18.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:18.229 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:52:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:52:17] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.51.0" 2024-09-13T14:52:18.315 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:18.364 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:18.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:18.466 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:18.492 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:18.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:18.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:18.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.631 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:18.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:18.732 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:18.770 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:18.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:18.872 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:18.912 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:18.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:19.014 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:19.039 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:19.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:19.141 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:19.169 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:19.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:19.271 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:19.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.319 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:19.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:19.420 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:19.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.476 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:19.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:19.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:19 smithi096 ceph-mon[98901]: pgmap v773: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 2333/77445 objects misplaced (3.012%); 6.8 MiB/s, 109 objects/s recovering 2024-09-13T14:52:19.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:19 smithi096 ceph-mon[102928]: pgmap v773: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 2333/77445 objects misplaced (3.012%); 6.8 MiB/s, 109 objects/s recovering 2024-09-13T14:52:19.577 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:19.610 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:19.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:19.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:19.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:19 smithi204 ceph-mon[87071]: pgmap v773: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 2333/77445 objects misplaced (3.012%); 6.8 MiB/s, 109 objects/s recovering 2024-09-13T14:52:19.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.758 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:19.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:19.859 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:19.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.897 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:19.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:19.998 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:20.043 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:20.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:20.145 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:20.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.186 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:20.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:20.287 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:20.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.335 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:20.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:20.436 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:52:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:52:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:20.474 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:20.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:20.576 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:20.603 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:20.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:20.705 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:20.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.748 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:20.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:20.849 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:20.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.891 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:20.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:20.992 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:21.021 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:21.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:21.123 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:21.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.152 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:21.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:21.253 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:21.290 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:21.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:21.355 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:52:21.392 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:21.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.436 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:21.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:21.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:21 smithi204 ceph-mon[87071]: pgmap v774: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 1973/71796 objects misplaced (2.748%); 6.3 MiB/s, 100 objects/s recovering 2024-09-13T14:52:21.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:21 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3932075727' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:21.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:21 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:21.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:21.538 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:21.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:21 smithi096 ceph-mon[98901]: pgmap v774: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 1973/71796 objects misplaced (2.748%); 6.3 MiB/s, 100 objects/s recovering 2024-09-13T14:52:21.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:21 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3932075727' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:21.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:21 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:21.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:21 smithi096 ceph-mon[102928]: pgmap v774: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 1973/71796 objects misplaced (2.748%); 6.3 MiB/s, 100 objects/s recovering 2024-09-13T14:52:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:21 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3932075727' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:21 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:21.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.579 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:21.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:21.680 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:21.707 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:21.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:21.809 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:21.846 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:21.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:21.948 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:21.976 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:21.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:22.079 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:22.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.121 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:22.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:22.221 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:22.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.277 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:22.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:22.379 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:22.426 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:22.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:22.528 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:22.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.555 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:22.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:22.656 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:22.672 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:22 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:52:22.672 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:22 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:52:22.672 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:22 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:22.672 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:22 smithi096 ceph-mon[98901]: osdmap e262: 8 total, 8 up, 8 in 2024-09-13T14:52:22.672 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:22 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:52:22.673 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:22 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:52:22.673 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:22 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:22.673 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:22 smithi096 ceph-mon[102928]: osdmap e262: 8 total, 8 up, 8 in 2024-09-13T14:52:22.697 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:22.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:22.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:22 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:52:22.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:22 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:52:22.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:22 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:22.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:22 smithi204 ceph-mon[87071]: osdmap e262: 8 total, 8 up, 8 in 2024-09-13T14:52:22.798 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:22.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.840 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:22.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:22.941 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:22.985 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:22.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:23.087 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:23.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.129 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:23.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:23.151 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:23.162 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:23.177 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:52:23.178 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:52:23.218 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:52:23.218 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:52:23.230 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:23.268 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:52:23.268 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:52:23.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.277 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:23.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:23.309 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:52:23.309 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:52:23.341 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:52:23.342 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:52:23.369 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:52:23.370 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:52:23.379 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:23.402 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:52:23.403 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:52:23.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.428 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:23.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:23.439 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:52:23.439 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:52:23.529 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:23.570 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:23.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:23.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:23 smithi096 ceph-mon[98901]: pgmap v776: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 1738/66663 objects misplaced (2.607%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:52:23.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:23 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/188371593' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:23.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:23 smithi096 ceph-mon[102928]: pgmap v776: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 1738/66663 objects misplaced (2.607%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:52:23.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:23 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/188371593' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:23.672 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:23.699 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:23.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:23.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:23 smithi204 ceph-mon[87071]: pgmap v776: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 1738/66663 objects misplaced (2.607%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:52:23.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:23 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/188371593' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:23.801 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:23.827 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:23.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:23.929 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:23.955 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:23.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:24.057 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:24.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.097 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:24.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:24.198 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:24.227 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:24.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:24.329 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:24.358 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:24.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:24.460 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:24.503 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:24.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:24.605 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:52:24.606 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:24.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.642 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:24.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:24.743 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:24.769 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:24.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:24.871 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:24.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.914 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:24.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:25.015 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:25.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.052 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:25.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:25.153 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:25.186 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:25.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:25.288 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:25.311 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:25.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:25.413 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:25.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.456 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:25.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:25.558 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:25.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:25 smithi096 ceph-mon[98901]: pgmap v777: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 1392/59916 objects misplaced (2.323%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:52:25.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:25 smithi096 ceph-mon[102928]: pgmap v777: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 1392/59916 objects misplaced (2.323%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:52:25.583 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:25.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:25.685 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:25.707 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:25.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:25.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:25 smithi204 ceph-mon[87071]: pgmap v777: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 1392/59916 objects misplaced (2.323%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:52:25.809 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:25.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.832 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:25.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:25.934 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:25.956 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:25.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:26.058 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:26.080 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:26.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:26.182 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:26.206 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:26.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:26.308 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:26.329 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:26.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:26.431 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:26.452 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:26.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:26.554 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:26.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:26 smithi096 ceph-mon[102928]: osdmap e263: 8 total, 8 up, 8 in 2024-09-13T14:52:26.576 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:26 smithi096 ceph-mon[98901]: osdmap e263: 8 total, 8 up, 8 in 2024-09-13T14:52:26.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.588 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:26.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:26.689 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:26.715 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:26.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:26.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:26 smithi204 ceph-mon[87071]: osdmap e263: 8 total, 8 up, 8 in 2024-09-13T14:52:26.817 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:26.852 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:26.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:26.956 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:26.983 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:26.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:27.085 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:27.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.109 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:27.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:27.210 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:27.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.241 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:27.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:27.342 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:27.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.384 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:27.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:27.485 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:27.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.519 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:27.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:27.620 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:27.645 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:27.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:27.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:27 smithi204 ceph-mon[87071]: pgmap v779: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 1392/58497 objects misplaced (2.380%); 4.4 MiB/s, 70 objects/s recovering 2024-09-13T14:52:27.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:27 smithi204 ceph-mon[87071]: osdmap e264: 8 total, 8 up, 8 in 2024-09-13T14:52:27.736 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:52:27.746 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:27.770 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:27 smithi096 ceph-mon[98901]: pgmap v779: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 1392/58497 objects misplaced (2.380%); 4.4 MiB/s, 70 objects/s recovering 2024-09-13T14:52:27.771 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:27 smithi096 ceph-mon[98901]: osdmap e264: 8 total, 8 up, 8 in 2024-09-13T14:52:27.771 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:27 smithi096 ceph-mon[102928]: pgmap v779: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 1392/58497 objects misplaced (2.380%); 4.4 MiB/s, 70 objects/s recovering 2024-09-13T14:52:27.771 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:27 smithi096 ceph-mon[102928]: osdmap e264: 8 total, 8 up, 8 in 2024-09-13T14:52:27.772 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:27.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:27.878 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:27.902 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:27.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:28.004 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:28.030 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:28.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:28.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:52:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:52:27] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.51.0" 2024-09-13T14:52:28.131 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:28.174 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:28.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:28.276 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:28.318 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:28.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:28.420 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:28.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.468 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:28.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:28.569 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:28.582 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:52:28.582 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:52:28.583 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:52:28.583 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:52:28.583 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:52:28.620 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:28.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:28.721 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:28.760 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:28.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:28.862 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:28.888 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:28.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:28.989 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:29.015 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:29.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:29.116 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:29.116 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:29.141 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:29.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:29.243 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:29.287 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:29.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:29.389 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:29.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.431 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:29.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:29.532 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:29.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.558 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:29.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:29.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:29 smithi096 ceph-mon[98901]: pgmap v781: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 1.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 532/54882 objects misplaced (0.969%); 9.0 MiB/s, 144 objects/s recovering 2024-09-13T14:52:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:29 smithi096 ceph-mon[102928]: pgmap v781: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 1.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 532/54882 objects misplaced (0.969%); 9.0 MiB/s, 144 objects/s recovering 2024-09-13T14:52:29.659 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:29.693 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:29.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:29.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:29 smithi204 ceph-mon[87071]: pgmap v781: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 1.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 532/54882 objects misplaced (0.969%); 9.0 MiB/s, 144 objects/s recovering 2024-09-13T14:52:29.795 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:29.835 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:29.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:29.920 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:52:29.920 INFO:teuthology.orchestra.run.smithi096.stdout:{"fsid":"51e50ed6-71db-11ef-bceb-c7b262605968","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1091,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":264,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238292,"num_in_osds":8,"osd_in_since":1726238911,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":97},{"state_name":"active+remapped+backfilling","count":2}],"num_pgs":99,"num_pools":5,"num_objects":18294,"data_bytes":1201805285,"bytes_used":7862194176,"bytes_avail":760098091008,"bytes_total":767960285184,"misplaced_objects":532,"misplaced_total":54882,"misplaced_ratio":0.009693524288473452,"recovering_objects_per_sec":144,"recovering_bytes_per_sec":9458047,"recovering_keys_per_sec":0,"num_objects_recovered":948,"num_bytes_recovered":62128128,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":780},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi096.dqiqpz","status":"up:active","gid":24539}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.96:8443/","prometheus":"http://172.21.15.96:9283/"}},"servicemap":{"epoch":26,"modified":"2024-09-13T14:51:49.892973+0000","services":{}},"progress_events":{"c818c95b-7b46-4520-8d2d-806ec384bbc4":{"message":"Global Recovery Event (70s)\n [===========================.] (remaining: 1s)","progress":0.97979795932769775,"add_to_ceph_s":true},"e114218e-ac8a-4fa8-8ddb-d075f64e6f70":{"message":"Upgrade to 19.3.0-4697-gd58f7306 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-13T14:52:29.938 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:29.980 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:29.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:30.080 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:52:30.082 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:30.108 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:30.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:30.210 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:30.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.263 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:30.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:30.364 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:30.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.418 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:30.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:30.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:30 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2669124149' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:52:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:52:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:52:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:30.518 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:30.545 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:30.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:30.646 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:30.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.677 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:30.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:30.778 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:30.807 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:30.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:30.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:30 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2669124149' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:52:30.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:30 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2669124149' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:52:30.908 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:30.932 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:30.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:30.948 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:30.963 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:52:30.963 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:52:31.034 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:31.059 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:31.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:31.162 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:31.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.211 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:31.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:31.312 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:31.339 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:31.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:31.441 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:31.725 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:31 smithi204 ceph-mon[87071]: pgmap v782: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 431/49980 objects misplaced (0.862%); 8.2 MiB/s, 130 objects/s recovering 2024-09-13T14:52:31.726 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:31 smithi096 ceph-mon[98901]: pgmap v782: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 431/49980 objects misplaced (0.862%); 8.2 MiB/s, 130 objects/s recovering 2024-09-13T14:52:31.726 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:31 smithi096 ceph-mon[102928]: pgmap v782: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 431/49980 objects misplaced (0.862%); 8.2 MiB/s, 130 objects/s recovering 2024-09-13T14:52:31.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.753 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:31.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:31.854 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:31.875 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:31.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:31.977 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:31.998 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:31.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:32.100 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:32.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.122 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:32.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:32.225 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:32.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.248 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:32.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:32.349 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:32.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.371 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:32.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:32.472 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:32.496 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:32.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:32.598 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:32.625 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:32.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:32.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:32 smithi204 ceph-mon[87071]: osdmap e265: 8 total, 8 up, 8 in 2024-09-13T14:52:32.727 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:32.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.765 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:32.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:32.767 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:32 smithi096 ceph-mon[98901]: osdmap e265: 8 total, 8 up, 8 in 2024-09-13T14:52:32.768 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:32 smithi096 ceph-mon[102928]: osdmap e265: 8 total, 8 up, 8 in 2024-09-13T14:52:32.865 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:32.893 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:32.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:32.995 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:33.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.025 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:33.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:33.126 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:33.168 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:33.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:33.270 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:33.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.297 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:33.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:33.398 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:33.408 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:33.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.423 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:33.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:33.446 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:33.524 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:33.551 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:33.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:33.653 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:33.685 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:33 smithi096 ceph-mon[102928]: pgmap v784: 99 pgs: 1 peering, 98 active+clean; 943 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 11 MiB/s, 173 objects/s recovering 2024-09-13T14:52:33.685 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:33 smithi096 ceph-mon[102928]: osdmap e266: 8 total, 8 up, 8 in 2024-09-13T14:52:33.685 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:33 smithi096 ceph-mon[98901]: pgmap v784: 99 pgs: 1 peering, 98 active+clean; 943 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 11 MiB/s, 173 objects/s recovering 2024-09-13T14:52:33.685 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:33 smithi096 ceph-mon[98901]: osdmap e266: 8 total, 8 up, 8 in 2024-09-13T14:52:33.695 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:33.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:33.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:33 smithi204 ceph-mon[87071]: pgmap v784: 99 pgs: 1 peering, 98 active+clean; 943 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 11 MiB/s, 173 objects/s recovering 2024-09-13T14:52:33.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:33 smithi204 ceph-mon[87071]: osdmap e266: 8 total, 8 up, 8 in 2024-09-13T14:52:33.796 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:33.824 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:33.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:33.926 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:33.972 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:34.074 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:34.098 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:34.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:34.200 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:34.226 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:34.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:34.328 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:34.354 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:34.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:34.456 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:34.481 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:34.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:34.484 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:34 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:52:34.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:34 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3516239041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:34.485 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:34 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:52:34.485 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:34 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3516239041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:34.583 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:34.606 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:34.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:34.708 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:34.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:34 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:52:34.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:34 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3516239041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:34.732 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:34.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:34.834 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:34.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.865 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:34.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:34.966 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:34.990 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:34.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:35.038 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:52:35.038 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:52:35.039 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:52:35.039 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:52:35.039 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:52:35.039 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:52:35.039 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:52:35.039 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:52:35.039 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:52:35.039 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:52:35.039 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:52:35.039 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:52:35.039 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:52:35.039 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:52:35.040 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:52:35.040 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:52:35.092 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:35.116 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:35.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:35.218 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:35.244 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:35.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:35.347 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:35.378 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:35.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:35.480 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:35.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.507 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:35.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:35.587 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:35 smithi096 ceph-mon[98901]: pgmap v786: 99 pgs: 1 peering, 98 active+clean; 825 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 9.4 MiB/s, 150 objects/s recovering 2024-09-13T14:52:35.587 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:35 smithi096 ceph-mon[102928]: pgmap v786: 99 pgs: 1 peering, 98 active+clean; 825 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 9.4 MiB/s, 150 objects/s recovering 2024-09-13T14:52:35.608 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:35.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.650 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:35.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:35.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:35 smithi204 ceph-mon[87071]: pgmap v786: 99 pgs: 1 peering, 98 active+clean; 825 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 9.4 MiB/s, 150 objects/s recovering 2024-09-13T14:52:35.750 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:35.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.801 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:35.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:35.903 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:35.946 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:35.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:36.048 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:36.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.082 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:36.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:36.183 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:36.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.212 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:36.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:36.313 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:36.341 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:36.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:36.444 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:36.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.480 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:36.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:36.581 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:36.614 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:36.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:36.716 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:36.740 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:36.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:36.842 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:36.867 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:36.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:36.969 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:36.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.997 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:36.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:37.098 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:37.128 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:37.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:52:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:36 smithi204 ceph-mon[87071]: pgmap v787: 99 pgs: 1 peering, 98 active+clean; 799 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 3.8 MiB/s, 60 objects/s recovering 2024-09-13T14:52:37.230 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:37.243 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:52:37.243 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:37.243 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:36 smithi096 ceph-mon[98901]: pgmap v787: 99 pgs: 1 peering, 98 active+clean; 799 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 3.8 MiB/s, 60 objects/s recovering 2024-09-13T14:52:37.243 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:52:37.243 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:37.244 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:36 smithi096 ceph-mon[102928]: pgmap v787: 99 pgs: 1 peering, 98 active+clean; 799 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 3.8 MiB/s, 60 objects/s recovering 2024-09-13T14:52:37.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.286 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:37.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:37.387 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:37.414 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:37.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:37.516 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:37.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.580 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:37.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:37.681 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:37.735 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:37.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:37.812 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:52:37.837 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:37.861 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:37.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:37.932 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:52:37.963 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:37.986 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:37.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:38.089 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:38.103 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:37 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/159041686' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:38.103 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:52:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:52:37] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.51.0" 2024-09-13T14:52:38.103 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:37 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/159041686' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:38.122 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:38.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:38.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:37 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/159041686' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:38.224 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:38.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.274 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:38.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:38.376 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:38.411 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:38.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:38.513 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:38.564 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:38.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:38.666 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:38.707 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:38.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:38.808 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:38.834 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:38.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:38.936 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:38.985 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:38.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:39.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:38 smithi096 ceph-mon[98901]: pgmap v788: 99 pgs: 99 active+clean; 720 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 3.1 MiB/s, 48 objects/s recovering 2024-09-13T14:52:39.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:38 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:39.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:38 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/159041686' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:52:39.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:38 smithi096 ceph-mon[98901]: osdmap e267: 8 total, 8 up, 8 in 2024-09-13T14:52:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:38 smithi096 ceph-mon[102928]: pgmap v788: 99 pgs: 99 active+clean; 720 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 3.1 MiB/s, 48 objects/s recovering 2024-09-13T14:52:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:38 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:38 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/159041686' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:52:39.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:38 smithi096 ceph-mon[102928]: osdmap e267: 8 total, 8 up, 8 in 2024-09-13T14:52:39.087 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:39.102 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:52:39.102 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:52:39.119 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:39.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:38 smithi204 ceph-mon[87071]: pgmap v788: 99 pgs: 99 active+clean; 720 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 3.1 MiB/s, 48 objects/s recovering 2024-09-13T14:52:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:38 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:38 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/159041686' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:52:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:38 smithi204 ceph-mon[87071]: osdmap e267: 8 total, 8 up, 8 in 2024-09-13T14:52:39.221 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:39.253 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:39.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:39.355 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:39.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.395 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:39.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:39.496 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:39.523 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:39.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:39.626 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:39.660 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:39.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:39.762 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:39.786 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:39.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:39.798 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:52:39.887 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:39.932 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:39.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:40.033 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:40.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:39 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:52:40.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:39 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:52:40.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:39 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:52:40.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:39 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:52:40.086 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:40.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:40.188 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:40.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.219 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:40.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:40.320 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:40.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.362 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:40.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:40.463 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:40.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:39 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:52:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:39 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:52:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:52:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:52:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:40.498 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:40.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.518 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:40.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:40.619 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:40.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.678 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:40.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:40.779 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:40.819 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:52:40.821 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:52:40.823 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:40.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:40.925 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:40.961 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:40.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:41.003 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:52:41.064 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:41.087 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:41.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:41.190 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:41.215 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:41.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:41.317 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:41.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:41 smithi096 ceph-mon[98901]: pgmap v790: 99 pgs: 99 active+clean; 580 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-09-13T14:52:41.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:41 smithi096 ceph-mon[102928]: pgmap v790: 99 pgs: 99 active+clean; 580 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-09-13T14:52:41.361 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:41.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:41.463 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:41.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:41 smithi204 ceph-mon[87071]: pgmap v790: 99 pgs: 99 active+clean; 580 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-09-13T14:52:41.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.493 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:41.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:41.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:41.625 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:41.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:41.727 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:41.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.754 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:41.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:41.855 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:41.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.886 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:41.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:41.987 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:42.013 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:42.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:42.115 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:42.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.156 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:42.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:42.257 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:42.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.295 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:42.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:42.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:42 smithi096 ceph-mon[98901]: from='client.45674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:42.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:42 smithi096 ceph-mon[102928]: from='client.45674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:42.396 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:42.421 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:42.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:42.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:42 smithi204 ceph-mon[87071]: from='client.45674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:42.523 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:42.550 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:42.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:42.652 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:42.679 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:42.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:42.781 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:42.802 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:42.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:42.904 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:42.929 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:42.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:43.031 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:43.058 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:43.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:43.160 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:43.186 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:43.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:43.231 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:43.288 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:43.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.315 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:43.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:43.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:43 smithi096 ceph-mon[98901]: pgmap v791: 99 pgs: 99 active+clean; 503 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-09-13T14:52:43.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:43 smithi096 ceph-mon[102928]: pgmap v791: 99 pgs: 99 active+clean; 503 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-09-13T14:52:43.417 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:43.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.444 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:43.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:43.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:43 smithi204 ceph-mon[87071]: pgmap v791: 99 pgs: 99 active+clean; 503 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-09-13T14:52:43.545 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:43.570 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:43.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:43.671 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:43.699 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:43.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:43.801 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:43.838 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:43.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:43.940 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:43.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.978 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:43.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:44.080 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:44.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.124 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:44.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:44.225 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:44.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.264 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:44.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:44.365 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:44.379 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":790,"stamp":"2024-09-13T14:52:39.925179+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":608376805,"num_objects":9239,"num_object_clones":0,"num_object_copies":27717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9239,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":231954,"num_write_kb":7700816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8913,"num_bytes_recovered":584793938,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128942,"ondisk_log_size":128942,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5982652,"kb_used_data":1914164,"kb_used_omap":35,"kb_used_meta":4068124,"kb_avail":743978564,"statfs":{"total":767960285184,"available":761834049536,"internally_reserved":0,"allocated":1960103936,"data_stored":1945791968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36830,"internal_metadata":4165759010},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-380764160,"num_objects":-5810,"num_object_clones":0,"num_object_copies":-17430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5810,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002509"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":403,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.948537+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:52:37.948537+0000","last_peered":"2024-09-13T14:52:37.948537+0000","last_clean":"2024-09-13T14:52:37.948537+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:52:37.948537+0000","last_undegraded":"2024-09-13T14:52:37.948537+0000","last_fullsized":"2024-09-13T14:52:37.948537+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":232,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.447006+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:52:31.447006+0000","last_peered":"2024-09-13T14:52:31.447006+0000","last_clean":"2024-09-13T14:52:31.447006+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:52:31.447006+0000","last_undegraded":"2024-09-13T14:52:31.447006+0000","last_fullsized":"2024-09-13T14:52:31.447006+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":229,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.454723+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:52:31.454723+0000","last_peered":"2024-09-13T14:52:31.454723+0000","last_clean":"2024-09-13T14:52:31.454723+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:52:31.454723+0000","last_undegraded":"2024-09-13T14:52:31.454723+0000","last_fullsized":"2024-09-13T14:52:31.454723+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":417,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.941409+0000","last_change":"2024-09-13T14:51:44.196384+0000","last_active":"2024-09-13T14:52:37.941409+0000","last_peered":"2024-09-13T14:52:37.941409+0000","last_clean":"2024-09-13T14:52:37.941409+0000","last_became_active":"2024-09-13T14:51:44.196082+0000","last_became_peered":"2024-09-13T14:51:44.196082+0000","last_unstale":"2024-09-13T14:52:37.941409+0000","last_undegraded":"2024-09-13T14:52:37.941409+0000","last_fullsized":"2024-09-13T14:52:37.941409+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:11:03.757803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":410,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.943884+0000","last_change":"2024-09-13T14:51:44.199134+0000","last_active":"2024-09-13T14:52:37.943884+0000","last_peered":"2024-09-13T14:52:37.943884+0000","last_clean":"2024-09-13T14:52:37.943884+0000","last_became_active":"2024-09-13T14:51:44.198525+0000","last_became_peered":"2024-09-13T14:51:44.198525+0000","last_unstale":"2024-09-13T14:52:37.943884+0000","last_undegraded":"2024-09-13T14:52:37.943884+0000","last_fullsized":"2024-09-13T14:52:37.943884+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:44.009127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":233,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.446920+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T14:52:31.446920+0000","last_peered":"2024-09-13T14:52:31.446920+0000","last_clean":"2024-09-13T14:52:31.446920+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T14:52:31.446920+0000","last_undegraded":"2024-09-13T14:52:31.446920+0000","last_fullsized":"2024-09-13T14:52:31.446920+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":63,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.962504+0000","last_change":"2024-09-13T14:51:44.207246+0000","last_active":"2024-09-13T14:52:37.962504+0000","last_peered":"2024-09-13T14:52:37.962504+0000","last_clean":"2024-09-13T14:52:37.962504+0000","last_became_active":"2024-09-13T14:51:44.206637+0000","last_became_peered":"2024-09-13T14:51:44.206637+0000","last_unstale":"2024-09-13T14:52:37.962504+0000","last_undegraded":"2024-09-13T14:52:37.962504+0000","last_fullsized":"2024-09-13T14:52:37.962504+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:42.020301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":60,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.442252+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T14:52:31.442252+0000","last_peered":"2024-09-13T14:52:31.442252+0000","last_clean":"2024-09-13T14:52:31.442252+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T14:52:31.442252+0000","last_undegraded":"2024-09-13T14:52:31.442252+0000","last_fullsized":"2024-09-13T14:52:31.442252+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"267'14266","reported_seq":14372,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:38.104446+0000","last_change":"2024-09-13T14:48:32.204190+0000","last_active":"2024-09-13T14:52:38.104446+0000","last_peered":"2024-09-13T14:52:38.104446+0000","last_clean":"2024-09-13T14:52:38.104446+0000","last_became_active":"2024-09-13T14:48:32.203935+0000","last_became_peered":"2024-09-13T14:48:32.203935+0000","last_unstale":"2024-09-13T14:52:38.104446+0000","last_undegraded":"2024-09-13T14:52:38.104446+0000","last_fullsized":"2024-09-13T14:52:38.104446+0000","mapping_epoch":225,"log_start":"235'6100","ondisk_log_start":"235'6100","created":223,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8166,"log_dups_size":0,"ondisk_log_size":8166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:29:24.200434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36044800,"num_objects":550,"num_object_clones":0,"num_object_copies":1650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14266,"num_write_kb":474112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":387,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.952748+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:52:37.952748+0000","last_peered":"2024-09-13T14:52:37.952748+0000","last_clean":"2024-09-13T14:52:37.952748+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:52:37.952748+0000","last_undegraded":"2024-09-13T14:52:37.952748+0000","last_fullsized":"2024-09-13T14:52:37.952748+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":252,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.454652+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T14:52:31.454652+0000","last_peered":"2024-09-13T14:52:31.454652+0000","last_clean":"2024-09-13T14:52:31.454652+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T14:52:31.454652+0000","last_undegraded":"2024-09-13T14:52:31.454652+0000","last_fullsized":"2024-09-13T14:52:31.454652+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"267'14356","reported_seq":14456,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:38.175249+0000","last_change":"2024-09-13T14:48:30.196807+0000","last_active":"2024-09-13T14:52:38.175249+0000","last_peered":"2024-09-13T14:52:38.175249+0000","last_clean":"2024-09-13T14:52:38.175249+0000","last_became_active":"2024-09-13T14:48:30.195433+0000","last_became_peered":"2024-09-13T14:48:30.195433+0000","last_unstale":"2024-09-13T14:52:38.175249+0000","last_undegraded":"2024-09-13T14:52:38.175249+0000","last_fullsized":"2024-09-13T14:52:38.175249+0000","mapping_epoch":223,"log_start":"236'7368","ondisk_log_start":"236'7368","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":6988,"log_dups_size":0,"ondisk_log_size":6988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:31:16.623107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34340864,"num_objects":524,"num_object_clones":0,"num_object_copies":1572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14356,"num_write_kb":476160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":232,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.447307+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:52:31.447307+0000","last_peered":"2024-09-13T14:52:31.447307+0000","last_clean":"2024-09-13T14:52:31.447307+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:52:31.447307+0000","last_undegraded":"2024-09-13T14:52:31.447307+0000","last_fullsized":"2024-09-13T14:52:31.447307+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":287,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.465802+0000","last_change":"2024-09-13T14:51:11.059483+0000","last_active":"2024-09-13T14:52:31.465802+0000","last_peered":"2024-09-13T14:52:31.465802+0000","last_clean":"2024-09-13T14:52:31.465802+0000","last_became_active":"2024-09-13T14:51:11.059262+0000","last_became_peered":"2024-09-13T14:51:11.059262+0000","last_unstale":"2024-09-13T14:52:31.465802+0000","last_undegraded":"2024-09-13T14:52:31.465802+0000","last_fullsized":"2024-09-13T14:52:31.465802+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:45:35.152631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"267'14304","reported_seq":14404,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:38.438155+0000","last_change":"2024-09-13T14:48:30.196414+0000","last_active":"2024-09-13T14:52:38.438155+0000","last_peered":"2024-09-13T14:52:38.438155+0000","last_clean":"2024-09-13T14:52:38.438155+0000","last_became_active":"2024-09-13T14:48:30.195744+0000","last_became_peered":"2024-09-13T14:48:30.195744+0000","last_unstale":"2024-09-13T14:52:38.438155+0000","last_undegraded":"2024-09-13T14:52:38.438155+0000","last_fullsized":"2024-09-13T14:52:38.438155+0000","mapping_epoch":223,"log_start":"235'6900","ondisk_log_start":"235'6900","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7404,"log_dups_size":0,"ondisk_log_size":7404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.893543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36438016,"num_objects":556,"num_object_clones":0,"num_object_copies":1668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":556,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14304,"num_write_kb":475520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":242,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.447258+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T14:52:31.447258+0000","last_peered":"2024-09-13T14:52:31.447258+0000","last_clean":"2024-09-13T14:52:31.447258+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T14:52:31.447258+0000","last_undegraded":"2024-09-13T14:52:31.447258+0000","last_fullsized":"2024-09-13T14:52:31.447258+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":60,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.442337+0000","last_change":"2024-09-13T14:51:44.190879+0000","last_active":"2024-09-13T14:52:31.442337+0000","last_peered":"2024-09-13T14:52:31.442337+0000","last_clean":"2024-09-13T14:52:31.442337+0000","last_became_active":"2024-09-13T14:51:44.190617+0000","last_became_peered":"2024-09-13T14:51:44.190617+0000","last_unstale":"2024-09-13T14:52:31.442337+0000","last_undegraded":"2024-09-13T14:52:31.442337+0000","last_fullsized":"2024-09-13T14:52:31.442337+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:19.536298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"267'14637","reported_seq":14737,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:38.451094+0000","last_change":"2024-09-13T14:48:30.193569+0000","last_active":"2024-09-13T14:52:38.451094+0000","last_peered":"2024-09-13T14:52:38.451094+0000","last_clean":"2024-09-13T14:52:38.451094+0000","last_became_active":"2024-09-13T14:48:30.191205+0000","last_became_peered":"2024-09-13T14:48:30.191205+0000","last_unstale":"2024-09-13T14:52:38.451094+0000","last_undegraded":"2024-09-13T14:52:38.451094+0000","last_fullsized":"2024-09-13T14:52:38.451094+0000","mapping_epoch":223,"log_start":"234'5768","ondisk_log_start":"234'5768","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8869,"log_dups_size":0,"ondisk_log_size":8869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:27.050525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35848192,"num_objects":547,"num_object_clones":0,"num_object_copies":1641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14637,"num_write_kb":485888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":250,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.454937+0000","last_change":"2024-09-13T14:48:32.206148+0000","last_active":"2024-09-13T14:52:31.454937+0000","last_peered":"2024-09-13T14:52:31.454937+0000","last_clean":"2024-09-13T14:52:31.454937+0000","last_became_active":"2024-09-13T14:48:32.205832+0000","last_became_peered":"2024-09-13T14:48:32.205832+0000","last_unstale":"2024-09-13T14:52:31.454937+0000","last_undegraded":"2024-09-13T14:52:31.454937+0000","last_fullsized":"2024-09-13T14:52:31.454937+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:55.026094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":397,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.941861+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:52:37.941861+0000","last_peered":"2024-09-13T14:52:37.941861+0000","last_clean":"2024-09-13T14:52:37.941861+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:52:37.941861+0000","last_undegraded":"2024-09-13T14:52:37.941861+0000","last_fullsized":"2024-09-13T14:52:37.941861+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"267'14543","reported_seq":14643,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:38.443067+0000","last_change":"2024-09-13T14:48:30.198101+0000","last_active":"2024-09-13T14:52:38.443067+0000","last_peered":"2024-09-13T14:52:38.443067+0000","last_clean":"2024-09-13T14:52:38.443067+0000","last_became_active":"2024-09-13T14:48:30.197770+0000","last_became_peered":"2024-09-13T14:48:30.197770+0000","last_unstale":"2024-09-13T14:52:38.443067+0000","last_undegraded":"2024-09-13T14:52:38.443067+0000","last_fullsized":"2024-09-13T14:52:38.443067+0000","mapping_epoch":223,"log_start":"236'7200","ondisk_log_start":"236'7200","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7343,"log_dups_size":0,"ondisk_log_size":7343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:26:49.717826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36241408,"num_objects":553,"num_object_clones":0,"num_object_copies":1659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14543,"num_write_kb":483072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":413,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.446814+0000","last_change":"2024-09-13T14:51:01.996634+0000","last_active":"2024-09-13T14:52:31.446814+0000","last_peered":"2024-09-13T14:52:31.446814+0000","last_clean":"2024-09-13T14:52:31.446814+0000","last_became_active":"2024-09-13T14:51:01.996264+0000","last_became_peered":"2024-09-13T14:51:01.996264+0000","last_unstale":"2024-09-13T14:52:31.446814+0000","last_undegraded":"2024-09-13T14:52:31.446814+0000","last_fullsized":"2024-09-13T14:52:31.446814+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:30.083115+0000","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":400,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.947762+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:52:37.947762+0000","last_peered":"2024-09-13T14:52:37.947762+0000","last_clean":"2024-09-13T14:52:37.947762+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:52:37.947762+0000","last_undegraded":"2024-09-13T14:52:37.947762+0000","last_fullsized":"2024-09-13T14:52:37.947762+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":250,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.463491+0000","last_change":"2024-09-13T14:48:32.196993+0000","last_active":"2024-09-13T14:52:31.463491+0000","last_peered":"2024-09-13T14:52:31.463491+0000","last_clean":"2024-09-13T14:52:31.463491+0000","last_became_active":"2024-09-13T14:48:32.196681+0000","last_became_peered":"2024-09-13T14:48:32.196681+0000","last_unstale":"2024-09-13T14:52:31.463491+0000","last_undegraded":"2024-09-13T14:52:31.463491+0000","last_fullsized":"2024-09-13T14:52:31.463491+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:56.040494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"267'14440","reported_seq":14538,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:38.482413+0000","last_change":"2024-09-13T14:48:30.197998+0000","last_active":"2024-09-13T14:52:38.482413+0000","last_peered":"2024-09-13T14:52:38.482413+0000","last_clean":"2024-09-13T14:52:38.482413+0000","last_became_active":"2024-09-13T14:48:30.197697+0000","last_became_peered":"2024-09-13T14:48:30.197697+0000","last_unstale":"2024-09-13T14:52:38.482413+0000","last_undegraded":"2024-09-13T14:52:38.482413+0000","last_fullsized":"2024-09-13T14:52:38.482413+0000","mapping_epoch":223,"log_start":"234'5585","ondisk_log_start":"234'5585","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8855,"log_dups_size":0,"ondisk_log_size":8855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:17.417819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36962304,"num_objects":564,"num_object_clones":0,"num_object_copies":1692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14440,"num_write_kb":480128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":173,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.454428+0000","last_change":"2024-09-13T14:40:27.868402+0000","last_active":"2024-09-13T14:52:31.454428+0000","last_peered":"2024-09-13T14:52:31.454428+0000","last_clean":"2024-09-13T14:52:31.454428+0000","last_became_active":"2024-09-13T14:40:22.966301+0000","last_became_peered":"2024-09-13T14:40:22.966301+0000","last_unstale":"2024-09-13T14:52:31.454428+0000","last_undegraded":"2024-09-13T14:52:31.454428+0000","last_fullsized":"2024-09-13T14:52:31.454428+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:18.660186+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":50,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.442372+0000","last_change":"2024-09-13T14:48:32.200521+0000","last_active":"2024-09-13T14:52:31.442372+0000","last_peered":"2024-09-13T14:52:31.442372+0000","last_clean":"2024-09-13T14:52:31.442372+0000","last_became_active":"2024-09-13T14:48:32.200256+0000","last_became_peered":"2024-09-13T14:48:32.200256+0000","last_unstale":"2024-09-13T14:52:31.442372+0000","last_undegraded":"2024-09-13T14:52:31.442372+0000","last_fullsized":"2024-09-13T14:52:31.442372+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:38.899629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":247,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.463667+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:52:31.463667+0000","last_peered":"2024-09-13T14:52:31.463667+0000","last_clean":"2024-09-13T14:52:31.463667+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:52:31.463667+0000","last_undegraded":"2024-09-13T14:52:31.463667+0000","last_fullsized":"2024-09-13T14:52:31.463667+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"266'14399","reported_seq":14450,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:36.628175+0000","last_change":"2024-09-13T14:48:32.204769+0000","last_active":"2024-09-13T14:52:36.628175+0000","last_peered":"2024-09-13T14:52:36.628175+0000","last_clean":"2024-09-13T14:52:36.628175+0000","last_became_active":"2024-09-13T14:48:32.204464+0000","last_became_peered":"2024-09-13T14:48:32.204464+0000","last_unstale":"2024-09-13T14:52:36.628175+0000","last_undegraded":"2024-09-13T14:52:36.628175+0000","last_fullsized":"2024-09-13T14:52:36.628175+0000","mapping_epoch":225,"log_start":"235'6200","ondisk_log_start":"235'6200","created":223,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8199,"log_dups_size":0,"ondisk_log_size":8199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:46.636678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42205208,"num_objects":645,"num_object_clones":0,"num_object_copies":1935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":645,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14399,"num_write_kb":481345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":383,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.947789+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:52:37.947789+0000","last_peered":"2024-09-13T14:52:37.947789+0000","last_clean":"2024-09-13T14:52:37.947789+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:52:37.947789+0000","last_undegraded":"2024-09-13T14:52:37.947789+0000","last_fullsized":"2024-09-13T14:52:37.947789+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":250,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.463581+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T14:52:31.463581+0000","last_peered":"2024-09-13T14:52:31.463581+0000","last_clean":"2024-09-13T14:52:31.463581+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T14:52:31.463581+0000","last_undegraded":"2024-09-13T14:52:31.463581+0000","last_fullsized":"2024-09-13T14:52:31.463581+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"266'14297","reported_seq":21665,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:36.653969+0000","last_change":"2024-09-13T14:51:46.210926+0000","last_active":"2024-09-13T14:52:36.653969+0000","last_peered":"2024-09-13T14:52:36.653969+0000","last_clean":"2024-09-13T14:52:36.653969+0000","last_became_active":"2024-09-13T14:51:46.187150+0000","last_became_peered":"2024-09-13T14:51:46.187150+0000","last_unstale":"2024-09-13T14:52:36.653969+0000","last_undegraded":"2024-09-13T14:52:36.653969+0000","last_fullsized":"2024-09-13T14:52:36.653969+0000","mapping_epoch":254,"log_start":"235'6100","ondisk_log_start":"235'6100","created":223,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8197,"log_dups_size":0,"ondisk_log_size":8197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:35:28.550999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41484288,"num_objects":633,"num_object_clones":0,"num_object_copies":1899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14297,"num_write_kb":477760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3647,"num_bytes_recovered":239009792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":176,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.454519+0000","last_change":"2024-09-13T14:48:32.205288+0000","last_active":"2024-09-13T14:52:31.454519+0000","last_peered":"2024-09-13T14:52:31.454519+0000","last_clean":"2024-09-13T14:52:31.454519+0000","last_became_active":"2024-09-13T14:48:32.204657+0000","last_became_peered":"2024-09-13T14:48:32.204657+0000","last_unstale":"2024-09-13T14:52:31.454519+0000","last_undegraded":"2024-09-13T14:52:31.454519+0000","last_fullsized":"2024-09-13T14:52:31.454519+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:52.513329+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":56,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.936010+0000","last_change":"2024-09-13T14:51:11.059644+0000","last_active":"2024-09-13T14:52:37.936010+0000","last_peered":"2024-09-13T14:52:37.936010+0000","last_clean":"2024-09-13T14:52:37.936010+0000","last_became_active":"2024-09-13T14:51:11.058874+0000","last_became_peered":"2024-09-13T14:51:11.058874+0000","last_unstale":"2024-09-13T14:52:37.936010+0000","last_undegraded":"2024-09-13T14:52:37.936010+0000","last_fullsized":"2024-09-13T14:52:37.936010+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:15:14.736476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":384,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.950113+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:52:37.950113+0000","last_peered":"2024-09-13T14:52:37.950113+0000","last_clean":"2024-09-13T14:52:37.950113+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:52:37.950113+0000","last_undegraded":"2024-09-13T14:52:37.950113+0000","last_fullsized":"2024-09-13T14:52:37.950113+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"267'14655","reported_seq":17990,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:38.399176+0000","last_change":"2024-09-13T14:52:26.525258+0000","last_active":"2024-09-13T14:52:38.399176+0000","last_peered":"2024-09-13T14:52:38.399176+0000","last_clean":"2024-09-13T14:52:38.399176+0000","last_became_active":"2024-09-13T14:52:26.403535+0000","last_became_peered":"2024-09-13T14:52:26.403535+0000","last_unstale":"2024-09-13T14:52:38.399176+0000","last_undegraded":"2024-09-13T14:52:38.399176+0000","last_fullsized":"2024-09-13T14:52:38.399176+0000","mapping_epoch":263,"log_start":"236'7300","ondisk_log_start":"236'7300","created":223,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7355,"log_dups_size":0,"ondisk_log_size":7355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:07:51.400877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33226752,"num_objects":507,"num_object_clones":0,"num_object_copies":1521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14655,"num_write_kb":485184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1609,"num_bytes_recovered":105447424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":394,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.447008+0000","last_change":"2024-09-13T14:48:32.204164+0000","last_active":"2024-09-13T14:52:31.447008+0000","last_peered":"2024-09-13T14:52:31.447008+0000","last_clean":"2024-09-13T14:52:31.447008+0000","last_became_active":"2024-09-13T14:48:32.203815+0000","last_became_peered":"2024-09-13T14:48:32.203815+0000","last_unstale":"2024-09-13T14:52:31.447008+0000","last_undegraded":"2024-09-13T14:52:31.447008+0000","last_fullsized":"2024-09-13T14:52:31.447008+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:22.639760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:22.639760+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:21.985178+0000","scrub_duration":0.0051120610000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":792,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.454699+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:52:31.454699+0000","last_peered":"2024-09-13T14:52:31.454699+0000","last_clean":"2024-09-13T14:52:31.454699+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:52:31.454699+0000","last_undegraded":"2024-09-13T14:52:31.454699+0000","last_fullsized":"2024-09-13T14:52:31.454699+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":56,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.947266+0000","last_change":"2024-09-13T14:51:11.059953+0000","last_active":"2024-09-13T14:52:37.947266+0000","last_peered":"2024-09-13T14:52:37.947266+0000","last_clean":"2024-09-13T14:52:37.947266+0000","last_became_active":"2024-09-13T14:51:11.059396+0000","last_became_peered":"2024-09-13T14:51:11.059396+0000","last_unstale":"2024-09-13T14:52:37.947266+0000","last_undegraded":"2024-09-13T14:52:37.947266+0000","last_fullsized":"2024-09-13T14:52:37.947266+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:11:51.119142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":50,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.442687+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T14:52:31.442687+0000","last_peered":"2024-09-13T14:52:31.442687+0000","last_clean":"2024-09-13T14:52:31.442687+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T14:52:31.442687+0000","last_undegraded":"2024-09-13T14:52:31.442687+0000","last_fullsized":"2024-09-13T14:52:31.442687+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"266'14549","reported_seq":14600,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:36.857296+0000","last_change":"2024-09-13T14:48:30.197655+0000","last_active":"2024-09-13T14:52:36.857296+0000","last_peered":"2024-09-13T14:52:36.857296+0000","last_clean":"2024-09-13T14:52:36.857296+0000","last_became_active":"2024-09-13T14:48:30.197442+0000","last_became_peered":"2024-09-13T14:48:30.197442+0000","last_unstale":"2024-09-13T14:52:36.857296+0000","last_undegraded":"2024-09-13T14:52:36.857296+0000","last_fullsized":"2024-09-13T14:52:36.857296+0000","mapping_epoch":223,"log_start":"235'6832","ondisk_log_start":"235'6832","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7717,"log_dups_size":0,"ondisk_log_size":7717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:44.452905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39518208,"num_objects":603,"num_object_clones":0,"num_object_copies":1809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14549,"num_write_kb":484864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":523,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.951741+0000","last_change":"2024-09-13T14:48:32.204587+0000","last_active":"2024-09-13T14:52:37.951741+0000","last_peered":"2024-09-13T14:52:37.951741+0000","last_clean":"2024-09-13T14:52:37.951741+0000","last_became_active":"2024-09-13T14:48:32.204153+0000","last_became_peered":"2024-09-13T14:48:32.204153+0000","last_unstale":"2024-09-13T14:52:37.951741+0000","last_undegraded":"2024-09-13T14:52:37.951741+0000","last_fullsized":"2024-09-13T14:52:37.951741+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.784263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.784263+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:11:12.651038+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":477,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.446964+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:52:31.446964+0000","last_peered":"2024-09-13T14:52:31.446964+0000","last_clean":"2024-09-13T14:52:31.446964+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:52:31.446964+0000","last_undegraded":"2024-09-13T14:52:31.446964+0000","last_fullsized":"2024-09-13T14:52:31.446964+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":419,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.947891+0000","last_change":"2024-09-13T14:51:11.061677+0000","last_active":"2024-09-13T14:52:37.947891+0000","last_peered":"2024-09-13T14:52:37.947891+0000","last_clean":"2024-09-13T14:52:37.947891+0000","last_became_active":"2024-09-13T14:51:11.061205+0000","last_became_peered":"2024-09-13T14:51:11.061205+0000","last_unstale":"2024-09-13T14:52:37.947891+0000","last_undegraded":"2024-09-13T14:52:37.947891+0000","last_fullsized":"2024-09-13T14:52:37.947891+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:38.521784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":422,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.944049+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T14:52:37.944049+0000","last_peered":"2024-09-13T14:52:37.944049+0000","last_clean":"2024-09-13T14:52:37.944049+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T14:52:37.944049+0000","last_undegraded":"2024-09-13T14:52:37.944049+0000","last_fullsized":"2024-09-13T14:52:37.944049+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"267'14631","reported_seq":14731,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:38.387924+0000","last_change":"2024-09-13T14:48:30.196036+0000","last_active":"2024-09-13T14:52:38.387924+0000","last_peered":"2024-09-13T14:52:38.387924+0000","last_clean":"2024-09-13T14:52:38.387924+0000","last_became_active":"2024-09-13T14:48:30.195478+0000","last_became_peered":"2024-09-13T14:48:30.195478+0000","last_unstale":"2024-09-13T14:52:38.387924+0000","last_undegraded":"2024-09-13T14:52:38.387924+0000","last_fullsized":"2024-09-13T14:52:38.387924+0000","mapping_epoch":223,"log_start":"236'7300","ondisk_log_start":"236'7300","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7331,"log_dups_size":0,"ondisk_log_size":7331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:21:44.624753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35192832,"num_objects":537,"num_object_clones":0,"num_object_copies":1611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14631,"num_write_kb":485376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":23,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.447079+0000","last_change":"2024-09-13T14:51:44.190446+0000","last_active":"2024-09-13T14:52:31.447079+0000","last_peered":"2024-09-13T14:52:31.447079+0000","last_clean":"2024-09-13T14:52:31.447079+0000","last_became_active":"2024-09-13T14:51:44.190115+0000","last_became_peered":"2024-09-13T14:51:44.190115+0000","last_unstale":"2024-09-13T14:52:31.447079+0000","last_undegraded":"2024-09-13T14:52:31.447079+0000","last_fullsized":"2024-09-13T14:52:31.447079+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:45:49.425578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:45:49.425578+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:43.724060+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":871,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.952674+0000","last_change":"2024-09-13T14:51:11.072066+0000","last_active":"2024-09-13T14:52:37.952674+0000","last_peered":"2024-09-13T14:52:37.952674+0000","last_clean":"2024-09-13T14:52:37.952674+0000","last_became_active":"2024-09-13T14:51:11.061307+0000","last_became_peered":"2024-09-13T14:51:11.061307+0000","last_unstale":"2024-09-13T14:52:37.952674+0000","last_undegraded":"2024-09-13T14:52:37.952674+0000","last_fullsized":"2024-09-13T14:52:37.952674+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:06.774106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":399,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.963604+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T14:52:37.963604+0000","last_peered":"2024-09-13T14:52:37.963604+0000","last_clean":"2024-09-13T14:52:37.963604+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T14:52:37.963604+0000","last_undegraded":"2024-09-13T14:52:37.963604+0000","last_fullsized":"2024-09-13T14:52:37.963604+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":418,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.950037+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:52:37.950037+0000","last_peered":"2024-09-13T14:52:37.950037+0000","last_clean":"2024-09-13T14:52:37.950037+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:52:37.950037+0000","last_undegraded":"2024-09-13T14:52:37.950037+0000","last_fullsized":"2024-09-13T14:52:37.950037+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"266'14501","reported_seq":16645,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:36.653585+0000","last_change":"2024-09-13T14:52:32.477979+0000","last_active":"2024-09-13T14:52:36.653585+0000","last_peered":"2024-09-13T14:52:36.653585+0000","last_clean":"2024-09-13T14:52:36.653585+0000","last_became_active":"2024-09-13T14:52:32.448142+0000","last_became_peered":"2024-09-13T14:52:32.448142+0000","last_unstale":"2024-09-13T14:52:36.653585+0000","last_undegraded":"2024-09-13T14:52:36.653585+0000","last_fullsized":"2024-09-13T14:52:36.653585+0000","mapping_epoch":265,"log_start":"235'6300","ondisk_log_start":"235'6300","created":223,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8201,"log_dups_size":0,"ondisk_log_size":8201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:01:22.201541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41353216,"num_objects":631,"num_object_clones":0,"num_object_copies":1893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14501,"num_write_kb":484224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1035,"num_bytes_recovered":67829760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":166,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.454481+0000","last_change":"2024-09-13T14:40:26.896691+0000","last_active":"2024-09-13T14:52:31.454481+0000","last_peered":"2024-09-13T14:52:31.454481+0000","last_clean":"2024-09-13T14:52:31.454481+0000","last_became_active":"2024-09-13T14:40:22.949772+0000","last_became_peered":"2024-09-13T14:40:22.949772+0000","last_unstale":"2024-09-13T14:52:31.454481+0000","last_undegraded":"2024-09-13T14:52:31.454481+0000","last_fullsized":"2024-09-13T14:52:31.454481+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:26.896613+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:26.896613+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:01.615433+0000","scrub_duration":0.004120828,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":892,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.457174+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T14:52:31.457174+0000","last_peered":"2024-09-13T14:52:31.457174+0000","last_clean":"2024-09-13T14:52:31.457174+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T14:52:31.457174+0000","last_undegraded":"2024-09-13T14:52:31.457174+0000","last_fullsized":"2024-09-13T14:52:31.457174+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":251,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.463681+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T14:52:31.463681+0000","last_peered":"2024-09-13T14:52:31.463681+0000","last_clean":"2024-09-13T14:52:31.463681+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T14:52:31.463681+0000","last_undegraded":"2024-09-13T14:52:31.463681+0000","last_fullsized":"2024-09-13T14:52:31.463681+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":417,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.950478+0000","last_change":"2024-09-13T14:51:11.060994+0000","last_active":"2024-09-13T14:52:37.950478+0000","last_peered":"2024-09-13T14:52:37.950478+0000","last_clean":"2024-09-13T14:52:37.950478+0000","last_became_active":"2024-09-13T14:51:11.060666+0000","last_became_peered":"2024-09-13T14:51:11.060666+0000","last_unstale":"2024-09-13T14:52:37.950478+0000","last_undegraded":"2024-09-13T14:52:37.950478+0000","last_fullsized":"2024-09-13T14:52:37.950478+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:40:15.828766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"266'14211","reported_seq":19306,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:36.760953+0000","last_change":"2024-09-13T14:52:06.352163+0000","last_active":"2024-09-13T14:52:36.760953+0000","last_peered":"2024-09-13T14:52:36.760953+0000","last_clean":"2024-09-13T14:52:36.760953+0000","last_became_active":"2024-09-13T14:52:06.323005+0000","last_became_peered":"2024-09-13T14:52:06.323005+0000","last_unstale":"2024-09-13T14:52:36.760953+0000","last_undegraded":"2024-09-13T14:52:36.760953+0000","last_fullsized":"2024-09-13T14:52:36.760953+0000","mapping_epoch":260,"log_start":"234'5800","ondisk_log_start":"234'5800","created":223,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8411,"log_dups_size":0,"ondisk_log_size":8411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:30.406030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42401792,"num_objects":647,"num_object_clones":0,"num_object_copies":1941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14211,"num_write_kb":475456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2504,"num_bytes_recovered":164102144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":70,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.442666+0000","last_change":"2024-09-13T14:51:11.061598+0000","last_active":"2024-09-13T14:52:31.442666+0000","last_peered":"2024-09-13T14:52:31.442666+0000","last_clean":"2024-09-13T14:52:31.442666+0000","last_became_active":"2024-09-13T14:51:11.061164+0000","last_became_peered":"2024-09-13T14:51:11.061164+0000","last_unstale":"2024-09-13T14:52:31.442666+0000","last_undegraded":"2024-09-13T14:52:31.442666+0000","last_fullsized":"2024-09-13T14:52:31.442666+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:48:34.095505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:48:34.095505+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:03.076025+0000","scrub_duration":0.001382681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":247,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.454826+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:52:31.454826+0000","last_peered":"2024-09-13T14:52:31.454826+0000","last_clean":"2024-09-13T14:52:31.454826+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:52:31.454826+0000","last_undegraded":"2024-09-13T14:52:31.454826+0000","last_fullsized":"2024-09-13T14:52:31.454826+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":902,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.950644+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:52:37.950644+0000","last_peered":"2024-09-13T14:52:37.950644+0000","last_clean":"2024-09-13T14:52:37.950644+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:52:37.950644+0000","last_undegraded":"2024-09-13T14:52:37.950644+0000","last_fullsized":"2024-09-13T14:52:37.950644+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":374,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.951533+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:52:37.951533+0000","last_peered":"2024-09-13T14:52:37.951533+0000","last_clean":"2024-09-13T14:52:37.951533+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:52:37.951533+0000","last_undegraded":"2024-09-13T14:52:37.951533+0000","last_fullsized":"2024-09-13T14:52:37.951533+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"266'14154","reported_seq":14205,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:36.948038+0000","last_change":"2024-09-13T14:48:30.192664+0000","last_active":"2024-09-13T14:52:36.948038+0000","last_peered":"2024-09-13T14:52:36.948038+0000","last_clean":"2024-09-13T14:52:36.948038+0000","last_became_active":"2024-09-13T14:48:30.192434+0000","last_became_peered":"2024-09-13T14:48:30.192434+0000","last_unstale":"2024-09-13T14:52:36.948038+0000","last_undegraded":"2024-09-13T14:52:36.948038+0000","last_fullsized":"2024-09-13T14:52:36.948038+0000","mapping_epoch":223,"log_start":"233'4400","ondisk_log_start":"233'4400","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":9754,"log_dups_size":0,"ondisk_log_size":9754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:58.920518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38928384,"num_objects":594,"num_object_clones":0,"num_object_copies":1782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14154,"num_write_kb":471936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":60,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.442712+0000","last_change":"2024-09-13T14:51:11.060132+0000","last_active":"2024-09-13T14:52:31.442712+0000","last_peered":"2024-09-13T14:52:31.442712+0000","last_clean":"2024-09-13T14:52:31.442712+0000","last_became_active":"2024-09-13T14:51:11.059785+0000","last_became_peered":"2024-09-13T14:51:11.059785+0000","last_unstale":"2024-09-13T14:52:31.442712+0000","last_undegraded":"2024-09-13T14:52:31.442712+0000","last_fullsized":"2024-09-13T14:52:31.442712+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.334379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.334379+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:44.298431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":399,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.950563+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T14:52:37.950563+0000","last_peered":"2024-09-13T14:52:37.950563+0000","last_clean":"2024-09-13T14:52:37.950563+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T14:52:37.950563+0000","last_undegraded":"2024-09-13T14:52:37.950563+0000","last_fullsized":"2024-09-13T14:52:37.950563+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":479,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.447257+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:52:31.447257+0000","last_peered":"2024-09-13T14:52:31.447257+0000","last_clean":"2024-09-13T14:52:31.447257+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:52:31.447257+0000","last_undegraded":"2024-09-13T14:52:31.447257+0000","last_fullsized":"2024-09-13T14:52:31.447257+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":387,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.941788+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:52:37.941788+0000","last_peered":"2024-09-13T14:52:37.941788+0000","last_clean":"2024-09-13T14:52:37.941788+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:52:37.941788+0000","last_undegraded":"2024-09-13T14:52:37.941788+0000","last_fullsized":"2024-09-13T14:52:37.941788+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"267'14638","reported_seq":14738,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:38.262582+0000","last_change":"2024-09-13T14:48:30.197226+0000","last_active":"2024-09-13T14:52:38.262582+0000","last_peered":"2024-09-13T14:52:38.262582+0000","last_clean":"2024-09-13T14:52:38.262582+0000","last_became_active":"2024-09-13T14:48:30.195243+0000","last_became_peered":"2024-09-13T14:48:30.195243+0000","last_unstale":"2024-09-13T14:52:38.262582+0000","last_undegraded":"2024-09-13T14:52:38.262582+0000","last_fullsized":"2024-09-13T14:52:38.262582+0000","mapping_epoch":223,"log_start":"236'7617","ondisk_log_start":"236'7617","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7021,"log_dups_size":0,"ondisk_log_size":7021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:46:06.507375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36962304,"num_objects":564,"num_object_clones":0,"num_object_copies":1692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14638,"num_write_kb":486464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":197,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.463394+0000","last_change":"2024-09-13T14:51:01.998101+0000","last_active":"2024-09-13T14:52:31.463394+0000","last_peered":"2024-09-13T14:52:31.463394+0000","last_clean":"2024-09-13T14:52:31.463394+0000","last_became_active":"2024-09-13T14:51:01.997688+0000","last_became_peered":"2024-09-13T14:51:01.997688+0000","last_unstale":"2024-09-13T14:52:31.463394+0000","last_undegraded":"2024-09-13T14:52:31.463394+0000","last_fullsized":"2024-09-13T14:52:31.463394+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:40:30.500046+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:02:51.444049+0000","scrub_duration":0.00248832,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":400,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.962669+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:52:37.962669+0000","last_peered":"2024-09-13T14:52:37.962669+0000","last_clean":"2024-09-13T14:52:37.962669+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:52:37.962669+0000","last_undegraded":"2024-09-13T14:52:37.962669+0000","last_fullsized":"2024-09-13T14:52:37.962669+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":757,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.949933+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T14:52:37.949933+0000","last_peered":"2024-09-13T14:52:37.949933+0000","last_clean":"2024-09-13T14:52:37.949933+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T14:52:37.949933+0000","last_undegraded":"2024-09-13T14:52:37.949933+0000","last_fullsized":"2024-09-13T14:52:37.949933+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":63,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.936113+0000","last_change":"2024-09-13T14:51:44.203186+0000","last_active":"2024-09-13T14:52:37.936113+0000","last_peered":"2024-09-13T14:52:37.936113+0000","last_clean":"2024-09-13T14:52:37.936113+0000","last_became_active":"2024-09-13T14:51:44.202851+0000","last_became_peered":"2024-09-13T14:51:44.202851+0000","last_unstale":"2024-09-13T14:52:37.936113+0000","last_undegraded":"2024-09-13T14:52:37.936113+0000","last_fullsized":"2024-09-13T14:52:37.936113+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:05:49.173340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"267'14466","reported_seq":14566,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:38.236165+0000","last_change":"2024-09-13T14:48:30.195027+0000","last_active":"2024-09-13T14:52:38.236165+0000","last_peered":"2024-09-13T14:52:38.236165+0000","last_clean":"2024-09-13T14:52:38.236165+0000","last_became_active":"2024-09-13T14:48:30.194672+0000","last_became_peered":"2024-09-13T14:48:30.194672+0000","last_unstale":"2024-09-13T14:52:38.236165+0000","last_undegraded":"2024-09-13T14:52:38.236165+0000","last_fullsized":"2024-09-13T14:52:38.236165+0000","mapping_epoch":223,"log_start":"236'7467","ondisk_log_start":"236'7467","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":6999,"log_dups_size":0,"ondisk_log_size":6999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:38:50.182084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36438016,"num_objects":556,"num_object_clones":0,"num_object_copies":1668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":556,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14466,"num_write_kb":480704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":330,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.948262+0000","last_change":"2024-09-13T14:40:24.124750+0000","last_active":"2024-09-13T14:52:37.948262+0000","last_peered":"2024-09-13T14:52:37.948262+0000","last_clean":"2024-09-13T14:52:37.948262+0000","last_became_active":"2024-09-13T14:40:22.958820+0000","last_became_peered":"2024-09-13T14:40:22.958820+0000","last_unstale":"2024-09-13T14:52:37.948262+0000","last_undegraded":"2024-09-13T14:52:37.948262+0000","last_fullsized":"2024-09-13T14:52:37.948262+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:24.124531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:24.124531+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:47:30.267886+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":408,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.943788+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:52:37.943788+0000","last_peered":"2024-09-13T14:52:37.943788+0000","last_clean":"2024-09-13T14:52:37.943788+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:52:37.943788+0000","last_undegraded":"2024-09-13T14:52:37.943788+0000","last_fullsized":"2024-09-13T14:52:37.943788+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1079,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.443200+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T14:52:31.443200+0000","last_peered":"2024-09-13T14:52:31.443200+0000","last_clean":"2024-09-13T14:52:31.443200+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T14:52:31.443200+0000","last_undegraded":"2024-09-13T14:52:31.443200+0000","last_fullsized":"2024-09-13T14:52:31.443200+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":233,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.465737+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T14:52:31.465737+0000","last_peered":"2024-09-13T14:52:31.465737+0000","last_clean":"2024-09-13T14:52:31.465737+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T14:52:31.465737+0000","last_undegraded":"2024-09-13T14:52:31.465737+0000","last_fullsized":"2024-09-13T14:52:31.465737+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":250,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.463583+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T14:52:31.463583+0000","last_peered":"2024-09-13T14:52:31.463583+0000","last_clean":"2024-09-13T14:52:31.463583+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T14:52:31.463583+0000","last_undegraded":"2024-09-13T14:52:31.463583+0000","last_fullsized":"2024-09-13T14:52:31.463583+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":416,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.941541+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T14:52:37.941541+0000","last_peered":"2024-09-13T14:52:37.941541+0000","last_clean":"2024-09-13T14:52:37.941541+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T14:52:37.941541+0000","last_undegraded":"2024-09-13T14:52:37.941541+0000","last_fullsized":"2024-09-13T14:52:37.941541+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":398,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.951092+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T14:52:37.951092+0000","last_peered":"2024-09-13T14:52:37.951092+0000","last_clean":"2024-09-13T14:52:37.951092+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T14:52:37.951092+0000","last_undegraded":"2024-09-13T14:52:37.951092+0000","last_fullsized":"2024-09-13T14:52:37.951092+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":246,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.447160+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:52:31.447160+0000","last_peered":"2024-09-13T14:52:31.447160+0000","last_clean":"2024-09-13T14:52:31.447160+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:52:31.447160+0000","last_undegraded":"2024-09-13T14:52:31.447160+0000","last_fullsized":"2024-09-13T14:52:31.447160+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":230,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.454983+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:52:31.454983+0000","last_peered":"2024-09-13T14:52:31.454983+0000","last_clean":"2024-09-13T14:52:31.454983+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:52:31.454983+0000","last_undegraded":"2024-09-13T14:52:31.454983+0000","last_fullsized":"2024-09-13T14:52:31.454983+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":368,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.946833+0000","last_change":"2024-09-13T14:48:32.200953+0000","last_active":"2024-09-13T14:52:37.946833+0000","last_peered":"2024-09-13T14:52:37.946833+0000","last_clean":"2024-09-13T14:52:37.946833+0000","last_became_active":"2024-09-13T14:48:32.200634+0000","last_became_peered":"2024-09-13T14:48:32.200634+0000","last_unstale":"2024-09-13T14:52:37.946833+0000","last_undegraded":"2024-09-13T14:52:37.946833+0000","last_fullsized":"2024-09-13T14:52:37.946833+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:18.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":395,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.944847+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:52:37.944847+0000","last_peered":"2024-09-13T14:52:37.944847+0000","last_clean":"2024-09-13T14:52:37.944847+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:52:37.944847+0000","last_undegraded":"2024-09-13T14:52:37.944847+0000","last_fullsized":"2024-09-13T14:52:37.944847+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":417,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.936766+0000","last_change":"2024-09-13T14:48:32.201812+0000","last_active":"2024-09-13T14:52:37.936766+0000","last_peered":"2024-09-13T14:52:37.936766+0000","last_clean":"2024-09-13T14:52:37.936766+0000","last_became_active":"2024-09-13T14:48:32.201486+0000","last_became_peered":"2024-09-13T14:48:32.201486+0000","last_unstale":"2024-09-13T14:52:37.936766+0000","last_undegraded":"2024-09-13T14:52:37.936766+0000","last_fullsized":"2024-09-13T14:52:37.936766+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:23:29.693906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":50,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.442740+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T14:52:31.442740+0000","last_peered":"2024-09-13T14:52:31.442740+0000","last_clean":"2024-09-13T14:52:31.442740+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T14:52:31.442740+0000","last_undegraded":"2024-09-13T14:52:31.442740+0000","last_fullsized":"2024-09-13T14:52:31.442740+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":420,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.936616+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:52:37.936616+0000","last_peered":"2024-09-13T14:52:37.936616+0000","last_clean":"2024-09-13T14:52:37.936616+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:52:37.936616+0000","last_undegraded":"2024-09-13T14:52:37.936616+0000","last_fullsized":"2024-09-13T14:52:37.936616+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":419,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.943967+0000","last_change":"2024-09-13T14:51:44.198771+0000","last_active":"2024-09-13T14:52:37.943967+0000","last_peered":"2024-09-13T14:52:37.943967+0000","last_clean":"2024-09-13T14:52:37.943967+0000","last_became_active":"2024-09-13T14:51:44.198466+0000","last_became_peered":"2024-09-13T14:51:44.198466+0000","last_unstale":"2024-09-13T14:52:37.943967+0000","last_undegraded":"2024-09-13T14:52:37.943967+0000","last_fullsized":"2024-09-13T14:52:37.943967+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:41:16.099954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":384,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.964224+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:52:37.964224+0000","last_peered":"2024-09-13T14:52:37.964224+0000","last_clean":"2024-09-13T14:52:37.964224+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:52:37.964224+0000","last_undegraded":"2024-09-13T14:52:37.964224+0000","last_fullsized":"2024-09-13T14:52:37.964224+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":416,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.946758+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:52:37.946758+0000","last_peered":"2024-09-13T14:52:37.946758+0000","last_clean":"2024-09-13T14:52:37.946758+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:52:37.946758+0000","last_undegraded":"2024-09-13T14:52:37.946758+0000","last_fullsized":"2024-09-13T14:52:37.946758+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":249,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.463378+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:52:31.463378+0000","last_peered":"2024-09-13T14:52:31.463378+0000","last_clean":"2024-09-13T14:52:31.463378+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:52:31.463378+0000","last_undegraded":"2024-09-13T14:52:31.463378+0000","last_fullsized":"2024-09-13T14:52:31.463378+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":60,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.952304+0000","last_change":"2024-09-13T14:51:02.029301+0000","last_active":"2024-09-13T14:52:37.952304+0000","last_peered":"2024-09-13T14:52:37.952304+0000","last_clean":"2024-09-13T14:52:37.952304+0000","last_became_active":"2024-09-13T14:51:02.029032+0000","last_became_peered":"2024-09-13T14:51:02.029032+0000","last_unstale":"2024-09-13T14:52:37.952304+0000","last_undegraded":"2024-09-13T14:52:37.952304+0000","last_fullsized":"2024-09-13T14:52:37.952304+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:40.150320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":388,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.941703+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:52:37.941703+0000","last_peered":"2024-09-13T14:52:37.941703+0000","last_clean":"2024-09-13T14:52:37.941703+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:52:37.941703+0000","last_undegraded":"2024-09-13T14:52:37.941703+0000","last_fullsized":"2024-09-13T14:52:37.941703+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":247,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.463582+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:52:31.463582+0000","last_peered":"2024-09-13T14:52:31.463582+0000","last_clean":"2024-09-13T14:52:31.463582+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:52:31.463582+0000","last_undegraded":"2024-09-13T14:52:31.463582+0000","last_fullsized":"2024-09-13T14:52:31.463582+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":232,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.454845+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:52:31.454845+0000","last_peered":"2024-09-13T14:52:31.454845+0000","last_clean":"2024-09-13T14:52:31.454845+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:52:31.454845+0000","last_undegraded":"2024-09-13T14:52:31.454845+0000","last_fullsized":"2024-09-13T14:52:31.454845+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":247,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.463351+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:52:31.463351+0000","last_peered":"2024-09-13T14:52:31.463351+0000","last_clean":"2024-09-13T14:52:31.463351+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:52:31.463351+0000","last_undegraded":"2024-09-13T14:52:31.463351+0000","last_fullsized":"2024-09-13T14:52:31.463351+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":271,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.454827+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T14:52:31.454827+0000","last_peered":"2024-09-13T14:52:31.454827+0000","last_clean":"2024-09-13T14:52:31.454827+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T14:52:31.454827+0000","last_undegraded":"2024-09-13T14:52:31.454827+0000","last_fullsized":"2024-09-13T14:52:31.454827+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":221,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.454608+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:52:31.454608+0000","last_peered":"2024-09-13T14:52:31.454608+0000","last_clean":"2024-09-13T14:52:31.454608+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:52:31.454608+0000","last_undegraded":"2024-09-13T14:52:31.454608+0000","last_fullsized":"2024-09-13T14:52:31.454608+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":424,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.947138+0000","last_change":"2024-09-13T14:48:32.202404+0000","last_active":"2024-09-13T14:52:37.947138+0000","last_peered":"2024-09-13T14:52:37.947138+0000","last_clean":"2024-09-13T14:52:37.947138+0000","last_became_active":"2024-09-13T14:48:32.202064+0000","last_became_peered":"2024-09-13T14:48:32.202064+0000","last_unstale":"2024-09-13T14:52:37.947138+0000","last_undegraded":"2024-09-13T14:52:37.947138+0000","last_fullsized":"2024-09-13T14:52:37.947138+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:25.674819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":253,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.463177+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T14:52:31.463177+0000","last_peered":"2024-09-13T14:52:31.463177+0000","last_clean":"2024-09-13T14:52:31.463177+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T14:52:31.463177+0000","last_undegraded":"2024-09-13T14:52:31.463177+0000","last_fullsized":"2024-09-13T14:52:31.463177+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":440,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.944758+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T14:52:37.944758+0000","last_peered":"2024-09-13T14:52:37.944758+0000","last_clean":"2024-09-13T14:52:37.944758+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T14:52:37.944758+0000","last_undegraded":"2024-09-13T14:52:37.944758+0000","last_fullsized":"2024-09-13T14:52:37.944758+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":603586584,"num_objects":9211,"num_object_clones":0,"num_object_copies":27633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9211,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":231047,"num_write_kb":7688193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8795,"num_bytes_recovered":576389120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1860382720,"data_stored":1860370504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126810,"ondisk_log_size":126810,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":16172,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397824,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":503900,"kb_used_data":202388,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93241252,"statfs":{"total":95995035648,"available":95479042048,"internally_reserved":0,"allocated":207245312,"data_stored":205451304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594053,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":513816,"kb_used_data":212304,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93231336,"statfs":{"total":95995035648,"available":95468888064,"internally_reserved":0,"allocated":217399296,"data_stored":215608980,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790282,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":799312,"kb_used_data":209352,"kb_used_omap":1,"kb_used_meta":589886,"kb_avail":92945840,"statfs":{"total":95995035648,"available":95176540160,"internally_reserved":0,"allocated":214376448,"data_stored":212590741,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":604043608},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986511,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":807696,"kb_used_data":209480,"kb_used_omap":1,"kb_used_meta":598206,"kb_avail":92937456,"statfs":{"total":95995035648,"available":95167954944,"internally_reserved":0,"allocated":214507520,"data_stored":212721000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612563288},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598959,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":919832,"kb_used_data":328076,"kb_used_omap":6,"kb_used_meta":591737,"kb_avail":92825320,"statfs":{"total":95995035648,"available":95053127680,"internally_reserved":0,"allocated":335949824,"data_stored":334167982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":605939680},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762486,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":836196,"kb_used_data":222104,"kb_used_omap":8,"kb_used_meta":614071,"kb_avail":92908956,"statfs":{"total":95995035648,"available":95138770944,"internally_reserved":0,"allocated":227434496,"data_stored":225640992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":628809162},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926012,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":783308,"kb_used_data":234180,"kb_used_omap":7,"kb_used_meta":548984,"kb_avail":92961844,"statfs":{"total":95995035648,"available":95192928256,"internally_reserved":0,"allocated":239800320,"data_stored":238017821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":562160329},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122252,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":818592,"kb_used_data":296280,"kb_used_omap":7,"kb_used_meta":522232,"kb_avail":92926560,"statfs":{"total":95995035648,"available":95156797440,"internally_reserved":0,"allocated":303390720,"data_stored":301593148,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":534766159},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":284954624,"data_stored":284950552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":219156480,"data_stored":219152408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":213778432,"data_stored":213778432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":323420160,"data_stored":323420160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":206241792,"data_stored":206241792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":204079104,"data_stored":204079104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":208605184,"data_stored":208601112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":200146944,"data_stored":200146944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:52:44.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.411 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:44.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:44.512 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:44.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.537 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:44.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:44.638 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:44.672 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:44.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:44.774 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:44.823 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:44.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:44.925 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:44.968 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:44.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:45.070 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:45.095 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:45.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:45.107 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:45 smithi096 ceph-mon[98901]: pgmap v792: 99 pgs: 99 active+clean; 329 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2024-09-13T14:52:45.107 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:45 smithi096 ceph-mon[102928]: pgmap v792: 99 pgs: 99 active+clean; 329 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2024-09-13T14:52:45.197 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:45.222 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:45.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:45.256 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:45.324 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:45.346 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:52:45.346 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:52:45.348 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:45.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:45.387 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:52:45.387 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:52:45.424 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:52:45.425 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:52:45.450 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:45.458 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:52:45.458 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:52:45.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:45 smithi204 ceph-mon[87071]: pgmap v792: 99 pgs: 99 active+clean; 329 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2024-09-13T14:52:45.475 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:45.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:45.493 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:52:45.493 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:52:45.526 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:52:45.527 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:52:45.562 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:52:45.563 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:52:45.577 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:45.599 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:52:45.599 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:52:45.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.615 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:45.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:45.716 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:45.739 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:45.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:45.841 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:45.868 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:45.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:45.969 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:46.004 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:46.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:46.106 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:46.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.132 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:46.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:46.234 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:46.271 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:46.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:46.318 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:46 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3422481214' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:46.318 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:46 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3422481214' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:46.373 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:46.412 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:46.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:46.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:46 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3422481214' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:46.514 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:46.549 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:46.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:46.651 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:46.693 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:46.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:46.795 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:46.822 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:46.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:46.924 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:46.951 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:46.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:47.053 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:47.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.083 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:47.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:47.184 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:47.213 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:47.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:47.315 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:47.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:47 smithi096 ceph-mon[98901]: pgmap v793: 99 pgs: 99 active+clean; 329 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-09-13T14:52:47.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:47 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:52:47.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:47 smithi096 ceph-mon[102928]: pgmap v793: 99 pgs: 99 active+clean; 329 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-09-13T14:52:47.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:47 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:52:47.357 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:47.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:47.385 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:52:47.459 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:47.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:47 smithi204 ceph-mon[87071]: pgmap v793: 99 pgs: 99 active+clean; 329 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-09-13T14:52:47.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:47 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:52:47.512 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:47.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:47.614 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:47.637 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:47.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:47.739 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:47.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.764 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:47.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:47.865 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:47.907 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:47.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:47.979 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:52:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:52:47] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.51.0" 2024-09-13T14:52:48.009 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:48.052 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:48.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:48.154 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:48.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.184 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:48.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:48.285 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:48.311 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:48.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:48.413 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:48.438 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:48.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:48.540 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:48.566 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:48.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:48.668 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:48.698 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:48.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:48.800 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:48.826 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:48.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:48.928 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:48.949 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:48.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:49.051 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:49.055 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Removed 120128 objects 2024-09-13T14:52:49.055 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Clean up completed and total clean up time :159.307 2024-09-13T14:52:49.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.073 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:49.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:49.175 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:49.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.199 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:49.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:49.301 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:49.322 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:49.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:49.424 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:49.446 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:49.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:49 smithi204 ceph-mon[87071]: pgmap v794: 99 pgs: 99 active+clean; 231 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2024-09-13T14:52:49.548 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:49.571 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:49.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:49.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:49 smithi096 ceph-mon[98901]: pgmap v794: 99 pgs: 99 active+clean; 231 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2024-09-13T14:52:49.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:49 smithi096 ceph-mon[102928]: pgmap v794: 99 pgs: 99 active+clean; 231 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2024-09-13T14:52:49.606 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:52:49.674 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:49.696 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:49.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:49.798 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:49.819 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:49.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:49.921 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:49.943 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:49.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:50.029 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-09-13T14:52:50.030 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-09-13T14:52:50.045 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:50.098 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:50.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:50.199 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:50.223 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:50.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:50.325 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:50.346 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:50.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:50.448 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:52:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:52:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:50.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.471 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:50.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:50.572 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:50.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.596 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:50.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:50.697 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:50.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.721 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:50.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:50.822 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:50.851 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:50.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:50.952 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:50.975 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:50.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:51.077 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:51.097 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:51.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:51.200 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:51.226 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:51.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:51.328 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:51.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.352 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:51.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:51.379 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:51 smithi096 ceph-mon[98901]: pgmap v795: 99 pgs: 99 active+clean; 70 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-09-13T14:52:51.379 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:52:51.379 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:51.379 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:51 smithi096 ceph-mon[102928]: pgmap v795: 99 pgs: 99 active+clean; 70 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-09-13T14:52:51.379 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:52:51.379 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:51.452 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:51.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:51 smithi204 ceph-mon[87071]: pgmap v795: 99 pgs: 99 active+clean; 70 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-09-13T14:52:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:52:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:51.473 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:51.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:51.537 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:52:51.537 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:52:51.537 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 0 2024-09-13T14:52:51.537 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:52:51.575 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:51.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.598 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:51.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:51.699 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:51.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.721 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:51.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:51.822 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:51.844 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:51.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:51.945 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:51.968 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:51.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:52.070 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:52.093 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:52.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:52.195 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:52.217 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:52.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:52.318 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:52.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.342 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:52.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:52.443 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:52.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.465 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:52.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:52.560 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:52.567 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:52.588 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:52.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:52.690 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:52.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.713 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:52.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:52.814 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:52.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.835 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:52.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:52.936 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:52.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.959 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:52.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:53.060 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:53.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.083 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:53.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:53.086 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:52:53.150 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:52:53.184 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:53.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.206 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:53.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:53.307 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:53.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.330 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:53.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:53.429 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:53 smithi096 ceph-mon[98901]: pgmap v796: 99 pgs: 99 active+clean; 33 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-09-13T14:52:53.429 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:53 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/985218145' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:53.429 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:53 smithi096 ceph-mon[102928]: pgmap v796: 99 pgs: 99 active+clean; 33 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-09-13T14:52:53.430 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:53 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/985218145' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:53.431 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:53.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.453 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:53.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:53 smithi204 ceph-mon[87071]: pgmap v796: 99 pgs: 99 active+clean; 33 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-09-13T14:52:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:53 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/985218145' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:53.555 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:53.576 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:53.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:53.678 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:53.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.701 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:53.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:53.802 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:53.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.825 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:53.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:53.926 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:53.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.948 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:53.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:54.049 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:54.071 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:54.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:54.173 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:54.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.196 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:54.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:54.297 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:54.320 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:54.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:54.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:54 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:54.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:54 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/985218145' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:54.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:54 smithi096 ceph-mon[98901]: osdmap e268: 8 total, 8 up, 8 in 2024-09-13T14:52:54.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:54 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:54.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:54 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/985218145' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:54.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:54 smithi096 ceph-mon[102928]: osdmap e268: 8 total, 8 up, 8 in 2024-09-13T14:52:54.422 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:54.445 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:54.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:54 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:54 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/985218145' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:54 smithi204 ceph-mon[87071]: osdmap e268: 8 total, 8 up, 8 in 2024-09-13T14:52:54.548 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:54.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.570 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:54.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:54.671 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:54.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.695 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:54.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:54.796 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:54.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.819 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:54.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:54.905 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:54.920 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:54.943 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:54.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:55.045 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:55.068 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:55.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:55.170 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:55.193 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:55.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:55.295 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:55.317 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:55.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:55.397 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:55 smithi096 ceph-mon[98901]: pgmap v798: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 519 op/s 2024-09-13T14:52:55.398 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:55 smithi096 ceph-mon[102928]: pgmap v798: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 519 op/s 2024-09-13T14:52:55.418 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:55.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.441 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:55.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:55.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:55 smithi204 ceph-mon[87071]: pgmap v798: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 519 op/s 2024-09-13T14:52:55.542 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:55.552 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:52:55.553 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:52:55.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.566 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:55.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:55.667 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:55.689 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:55.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:55.718 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:52:55.791 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:55.812 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:55.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:55.914 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:55.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.937 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:55.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:56.038 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:56.059 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:56.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:56.160 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:56.182 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:56.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:56.284 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:56.309 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:56.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:56.411 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:56.429 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:56 smithi096 ceph-mon[102928]: from='client.35877 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:56.430 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:56 smithi096 ceph-mon[98901]: from='client.35877 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:56.435 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:56.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:56.456 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":798,"stamp":"2024-09-13T14:52:53.932127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":241165,"num_write_kb":7700816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8913,"num_bytes_recovered":584793938,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128853,"ondisk_log_size":128853,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4133000,"kb_used_data":60616,"kb_used_omap":35,"kb_used_meta":4072284,"kb_avail":745828216,"statfs":{"total":767960285184,"available":763728093184,"internally_reserved":0,"allocated":62070784,"data_stored":47724280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36830,"internal_metadata":4170018850},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,2,5,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-340328472,"num_objects":-5194,"num_object_clones":0,"num_object_copies":-15582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5194,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005887"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":403,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.948537+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:52:37.948537+0000","last_peered":"2024-09-13T14:52:37.948537+0000","last_clean":"2024-09-13T14:52:37.948537+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:52:37.948537+0000","last_undegraded":"2024-09-13T14:52:37.948537+0000","last_fullsized":"2024-09-13T14:52:37.948537+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":233,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.444840+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:52:32.444840+0000","last_peered":"2024-09-13T14:52:32.444840+0000","last_clean":"2024-09-13T14:52:32.444840+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:52:32.444840+0000","last_undegraded":"2024-09-13T14:52:32.444840+0000","last_fullsized":"2024-09-13T14:52:32.444840+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":230,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.479785+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:52:32.479785+0000","last_peered":"2024-09-13T14:52:32.479785+0000","last_clean":"2024-09-13T14:52:32.479785+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:52:32.479785+0000","last_undegraded":"2024-09-13T14:52:32.479785+0000","last_fullsized":"2024-09-13T14:52:32.479785+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":419,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.400837+0000","last_change":"2024-09-13T14:51:44.196384+0000","last_active":"2024-09-13T14:52:53.400837+0000","last_peered":"2024-09-13T14:52:53.400837+0000","last_clean":"2024-09-13T14:52:53.400837+0000","last_became_active":"2024-09-13T14:51:44.196082+0000","last_became_peered":"2024-09-13T14:51:44.196082+0000","last_unstale":"2024-09-13T14:52:53.400837+0000","last_undegraded":"2024-09-13T14:52:53.400837+0000","last_fullsized":"2024-09-13T14:52:53.400837+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:11:03.757803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":413,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.401443+0000","last_change":"2024-09-13T14:51:44.199134+0000","last_active":"2024-09-13T14:52:53.401443+0000","last_peered":"2024-09-13T14:52:53.401443+0000","last_clean":"2024-09-13T14:52:53.401443+0000","last_became_active":"2024-09-13T14:51:44.198525+0000","last_became_peered":"2024-09-13T14:51:44.198525+0000","last_unstale":"2024-09-13T14:52:53.401443+0000","last_undegraded":"2024-09-13T14:52:53.401443+0000","last_fullsized":"2024-09-13T14:52:53.401443+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:44.009127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":234,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.444715+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T14:52:32.444715+0000","last_peered":"2024-09-13T14:52:32.444715+0000","last_clean":"2024-09-13T14:52:32.444715+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T14:52:32.444715+0000","last_undegraded":"2024-09-13T14:52:32.444715+0000","last_fullsized":"2024-09-13T14:52:32.444715+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":63,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.962504+0000","last_change":"2024-09-13T14:51:44.207246+0000","last_active":"2024-09-13T14:52:37.962504+0000","last_peered":"2024-09-13T14:52:37.962504+0000","last_clean":"2024-09-13T14:52:37.962504+0000","last_became_active":"2024-09-13T14:51:44.206637+0000","last_became_peered":"2024-09-13T14:51:44.206637+0000","last_unstale":"2024-09-13T14:52:37.962504+0000","last_undegraded":"2024-09-13T14:52:37.962504+0000","last_fullsized":"2024-09-13T14:52:37.962504+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:42.020301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":61,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.444515+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T14:52:32.444515+0000","last_peered":"2024-09-13T14:52:32.444515+0000","last_clean":"2024-09-13T14:52:32.444515+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T14:52:32.444515+0000","last_undegraded":"2024-09-13T14:52:32.444515+0000","last_fullsized":"2024-09-13T14:52:32.444515+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"267'14816","reported_seq":14922,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:49.010222+0000","last_change":"2024-09-13T14:48:32.204190+0000","last_active":"2024-09-13T14:52:49.010222+0000","last_peered":"2024-09-13T14:52:49.010222+0000","last_clean":"2024-09-13T14:52:49.010222+0000","last_became_active":"2024-09-13T14:48:32.203935+0000","last_became_peered":"2024-09-13T14:48:32.203935+0000","last_unstale":"2024-09-13T14:52:49.010222+0000","last_undegraded":"2024-09-13T14:52:49.010222+0000","last_fullsized":"2024-09-13T14:52:49.010222+0000","mapping_epoch":225,"log_start":"235'6700","ondisk_log_start":"235'6700","created":223,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8116,"log_dups_size":0,"ondisk_log_size":8116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:29:24.200434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14816,"num_write_kb":474112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":389,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.115202+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:52:53.115202+0000","last_peered":"2024-09-13T14:52:53.115202+0000","last_clean":"2024-09-13T14:52:53.115202+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:52:53.115202+0000","last_undegraded":"2024-09-13T14:52:53.115202+0000","last_fullsized":"2024-09-13T14:52:53.115202+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":253,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.479481+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T14:52:32.479481+0000","last_peered":"2024-09-13T14:52:32.479481+0000","last_clean":"2024-09-13T14:52:32.479481+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T14:52:32.479481+0000","last_undegraded":"2024-09-13T14:52:32.479481+0000","last_fullsized":"2024-09-13T14:52:32.479481+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"267'14880","reported_seq":14981,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:49.049148+0000","last_change":"2024-09-13T14:48:30.196807+0000","last_active":"2024-09-13T14:52:49.049148+0000","last_peered":"2024-09-13T14:52:49.049148+0000","last_clean":"2024-09-13T14:52:49.049148+0000","last_became_active":"2024-09-13T14:48:30.195433+0000","last_became_peered":"2024-09-13T14:48:30.195433+0000","last_unstale":"2024-09-13T14:52:49.049148+0000","last_undegraded":"2024-09-13T14:52:49.049148+0000","last_fullsized":"2024-09-13T14:52:49.049148+0000","mapping_epoch":223,"log_start":"236'7868","ondisk_log_start":"236'7868","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7012,"log_dups_size":0,"ondisk_log_size":7012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:31:16.623107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14880,"num_write_kb":476160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":233,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.445358+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:52:32.445358+0000","last_peered":"2024-09-13T14:52:32.445358+0000","last_clean":"2024-09-13T14:52:32.445358+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:52:32.445358+0000","last_undegraded":"2024-09-13T14:52:32.445358+0000","last_fullsized":"2024-09-13T14:52:32.445358+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":288,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.493955+0000","last_change":"2024-09-13T14:51:11.059483+0000","last_active":"2024-09-13T14:52:32.493955+0000","last_peered":"2024-09-13T14:52:32.493955+0000","last_clean":"2024-09-13T14:52:32.493955+0000","last_became_active":"2024-09-13T14:51:11.059262+0000","last_became_peered":"2024-09-13T14:51:11.059262+0000","last_unstale":"2024-09-13T14:52:32.493955+0000","last_undegraded":"2024-09-13T14:52:32.493955+0000","last_fullsized":"2024-09-13T14:52:32.493955+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:45:35.152631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"267'14860","reported_seq":14963,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.400642+0000","last_change":"2024-09-13T14:48:30.196414+0000","last_active":"2024-09-13T14:52:53.400642+0000","last_peered":"2024-09-13T14:52:53.400642+0000","last_clean":"2024-09-13T14:52:53.400642+0000","last_became_active":"2024-09-13T14:48:30.195744+0000","last_became_peered":"2024-09-13T14:48:30.195744+0000","last_unstale":"2024-09-13T14:52:53.400642+0000","last_undegraded":"2024-09-13T14:52:53.400642+0000","last_fullsized":"2024-09-13T14:52:53.400642+0000","mapping_epoch":223,"log_start":"236'7500","ondisk_log_start":"236'7500","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7360,"log_dups_size":0,"ondisk_log_size":7360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.893543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14860,"num_write_kb":475520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":243,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.445335+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T14:52:32.445335+0000","last_peered":"2024-09-13T14:52:32.445335+0000","last_clean":"2024-09-13T14:52:32.445335+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T14:52:32.445335+0000","last_undegraded":"2024-09-13T14:52:32.445335+0000","last_fullsized":"2024-09-13T14:52:32.445335+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":61,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.444626+0000","last_change":"2024-09-13T14:51:44.190879+0000","last_active":"2024-09-13T14:52:32.444626+0000","last_peered":"2024-09-13T14:52:32.444626+0000","last_clean":"2024-09-13T14:52:32.444626+0000","last_became_active":"2024-09-13T14:51:44.190617+0000","last_became_peered":"2024-09-13T14:51:44.190617+0000","last_unstale":"2024-09-13T14:52:32.444626+0000","last_undegraded":"2024-09-13T14:52:32.444626+0000","last_fullsized":"2024-09-13T14:52:32.444626+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:19.536298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"267'15184","reported_seq":15287,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.115198+0000","last_change":"2024-09-13T14:48:30.193569+0000","last_active":"2024-09-13T14:52:53.115198+0000","last_peered":"2024-09-13T14:52:53.115198+0000","last_clean":"2024-09-13T14:52:53.115198+0000","last_became_active":"2024-09-13T14:48:30.191205+0000","last_became_peered":"2024-09-13T14:48:30.191205+0000","last_unstale":"2024-09-13T14:52:53.115198+0000","last_undegraded":"2024-09-13T14:52:53.115198+0000","last_fullsized":"2024-09-13T14:52:53.115198+0000","mapping_epoch":223,"log_start":"235'6268","ondisk_log_start":"235'6268","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8916,"log_dups_size":0,"ondisk_log_size":8916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:27.050525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15184,"num_write_kb":485888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":251,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.479714+0000","last_change":"2024-09-13T14:48:32.206148+0000","last_active":"2024-09-13T14:52:32.479714+0000","last_peered":"2024-09-13T14:52:32.479714+0000","last_clean":"2024-09-13T14:52:32.479714+0000","last_became_active":"2024-09-13T14:48:32.205832+0000","last_became_peered":"2024-09-13T14:48:32.205832+0000","last_unstale":"2024-09-13T14:52:32.479714+0000","last_undegraded":"2024-09-13T14:52:32.479714+0000","last_fullsized":"2024-09-13T14:52:32.479714+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:55.026094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":399,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.401004+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:52:53.401004+0000","last_peered":"2024-09-13T14:52:53.401004+0000","last_clean":"2024-09-13T14:52:53.401004+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:52:53.401004+0000","last_undegraded":"2024-09-13T14:52:53.401004+0000","last_fullsized":"2024-09-13T14:52:53.401004+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"267'15096","reported_seq":15198,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.401385+0000","last_change":"2024-09-13T14:48:30.198101+0000","last_active":"2024-09-13T14:52:53.401385+0000","last_peered":"2024-09-13T14:52:53.401385+0000","last_clean":"2024-09-13T14:52:53.401385+0000","last_became_active":"2024-09-13T14:48:30.197770+0000","last_became_peered":"2024-09-13T14:48:30.197770+0000","last_unstale":"2024-09-13T14:52:53.401385+0000","last_undegraded":"2024-09-13T14:52:53.401385+0000","last_fullsized":"2024-09-13T14:52:53.401385+0000","mapping_epoch":223,"log_start":"236'7700","ondisk_log_start":"236'7700","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7396,"log_dups_size":0,"ondisk_log_size":7396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:26:49.717826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15096,"num_write_kb":483072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":414,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.444612+0000","last_change":"2024-09-13T14:51:01.996634+0000","last_active":"2024-09-13T14:52:32.444612+0000","last_peered":"2024-09-13T14:52:32.444612+0000","last_clean":"2024-09-13T14:52:32.444612+0000","last_became_active":"2024-09-13T14:51:01.996264+0000","last_became_peered":"2024-09-13T14:51:01.996264+0000","last_unstale":"2024-09-13T14:52:32.444612+0000","last_undegraded":"2024-09-13T14:52:32.444612+0000","last_fullsized":"2024-09-13T14:52:32.444612+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:30.083115+0000","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":400,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.947762+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:52:37.947762+0000","last_peered":"2024-09-13T14:52:37.947762+0000","last_clean":"2024-09-13T14:52:37.947762+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:52:37.947762+0000","last_undegraded":"2024-09-13T14:52:37.947762+0000","last_fullsized":"2024-09-13T14:52:37.947762+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":251,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.493993+0000","last_change":"2024-09-13T14:48:32.196993+0000","last_active":"2024-09-13T14:52:32.493993+0000","last_peered":"2024-09-13T14:52:32.493993+0000","last_clean":"2024-09-13T14:52:32.493993+0000","last_became_active":"2024-09-13T14:48:32.196681+0000","last_became_peered":"2024-09-13T14:48:32.196681+0000","last_unstale":"2024-09-13T14:52:32.493993+0000","last_undegraded":"2024-09-13T14:52:32.493993+0000","last_fullsized":"2024-09-13T14:52:32.493993+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:56.040494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"267'15004","reported_seq":15106,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.115653+0000","last_change":"2024-09-13T14:48:30.197998+0000","last_active":"2024-09-13T14:52:53.115653+0000","last_peered":"2024-09-13T14:52:53.115653+0000","last_clean":"2024-09-13T14:52:53.115653+0000","last_became_active":"2024-09-13T14:48:30.197697+0000","last_became_peered":"2024-09-13T14:48:30.197697+0000","last_unstale":"2024-09-13T14:52:53.115653+0000","last_undegraded":"2024-09-13T14:52:53.115653+0000","last_fullsized":"2024-09-13T14:52:53.115653+0000","mapping_epoch":223,"log_start":"235'6085","ondisk_log_start":"235'6085","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8919,"log_dups_size":0,"ondisk_log_size":8919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:17.417819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15004,"num_write_kb":480128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":174,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.479329+0000","last_change":"2024-09-13T14:40:27.868402+0000","last_active":"2024-09-13T14:52:32.479329+0000","last_peered":"2024-09-13T14:52:32.479329+0000","last_clean":"2024-09-13T14:52:32.479329+0000","last_became_active":"2024-09-13T14:40:22.966301+0000","last_became_peered":"2024-09-13T14:40:22.966301+0000","last_unstale":"2024-09-13T14:52:32.479329+0000","last_undegraded":"2024-09-13T14:52:32.479329+0000","last_fullsized":"2024-09-13T14:52:32.479329+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:18.660186+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":51,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.444713+0000","last_change":"2024-09-13T14:48:32.200521+0000","last_active":"2024-09-13T14:52:32.444713+0000","last_peered":"2024-09-13T14:52:32.444713+0000","last_clean":"2024-09-13T14:52:32.444713+0000","last_became_active":"2024-09-13T14:48:32.200256+0000","last_became_peered":"2024-09-13T14:48:32.200256+0000","last_unstale":"2024-09-13T14:52:32.444713+0000","last_undegraded":"2024-09-13T14:52:32.444713+0000","last_fullsized":"2024-09-13T14:52:32.444713+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:38.899629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":248,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.494190+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:52:32.494190+0000","last_peered":"2024-09-13T14:52:32.494190+0000","last_clean":"2024-09-13T14:52:32.494190+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:52:32.494190+0000","last_undegraded":"2024-09-13T14:52:32.494190+0000","last_fullsized":"2024-09-13T14:52:32.494190+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"267'15044","reported_seq":15096,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:49.014805+0000","last_change":"2024-09-13T14:48:32.204769+0000","last_active":"2024-09-13T14:52:49.014805+0000","last_peered":"2024-09-13T14:52:49.014805+0000","last_clean":"2024-09-13T14:52:49.014805+0000","last_became_active":"2024-09-13T14:48:32.204464+0000","last_became_peered":"2024-09-13T14:48:32.204464+0000","last_unstale":"2024-09-13T14:52:49.014805+0000","last_undegraded":"2024-09-13T14:52:49.014805+0000","last_fullsized":"2024-09-13T14:52:49.014805+0000","mapping_epoch":225,"log_start":"235'6900","ondisk_log_start":"235'6900","created":223,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8144,"log_dups_size":0,"ondisk_log_size":8144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:46.636678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":15044,"num_write_kb":481345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":385,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.399828+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:52:53.399828+0000","last_peered":"2024-09-13T14:52:53.399828+0000","last_clean":"2024-09-13T14:52:53.399828+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:52:53.399828+0000","last_undegraded":"2024-09-13T14:52:53.399828+0000","last_fullsized":"2024-09-13T14:52:53.399828+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":251,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.494362+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T14:52:32.494362+0000","last_peered":"2024-09-13T14:52:32.494362+0000","last_clean":"2024-09-13T14:52:32.494362+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T14:52:32.494362+0000","last_undegraded":"2024-09-13T14:52:32.494362+0000","last_fullsized":"2024-09-13T14:52:32.494362+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"267'14930","reported_seq":22299,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:49.040832+0000","last_change":"2024-09-13T14:51:46.210926+0000","last_active":"2024-09-13T14:52:49.040832+0000","last_peered":"2024-09-13T14:52:49.040832+0000","last_clean":"2024-09-13T14:52:49.040832+0000","last_became_active":"2024-09-13T14:51:46.187150+0000","last_became_peered":"2024-09-13T14:51:46.187150+0000","last_unstale":"2024-09-13T14:52:49.040832+0000","last_undegraded":"2024-09-13T14:52:49.040832+0000","last_fullsized":"2024-09-13T14:52:49.040832+0000","mapping_epoch":254,"log_start":"235'6800","ondisk_log_start":"235'6800","created":223,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8130,"log_dups_size":0,"ondisk_log_size":8130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:35:28.550999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14930,"num_write_kb":477760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3647,"num_bytes_recovered":239009792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":177,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.479586+0000","last_change":"2024-09-13T14:48:32.205288+0000","last_active":"2024-09-13T14:52:32.479586+0000","last_peered":"2024-09-13T14:52:32.479586+0000","last_clean":"2024-09-13T14:52:32.479586+0000","last_became_active":"2024-09-13T14:48:32.204657+0000","last_became_peered":"2024-09-13T14:48:32.204657+0000","last_unstale":"2024-09-13T14:52:32.479586+0000","last_undegraded":"2024-09-13T14:52:32.479586+0000","last_fullsized":"2024-09-13T14:52:32.479586+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:52.513329+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":56,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.936010+0000","last_change":"2024-09-13T14:51:11.059644+0000","last_active":"2024-09-13T14:52:37.936010+0000","last_peered":"2024-09-13T14:52:37.936010+0000","last_clean":"2024-09-13T14:52:37.936010+0000","last_became_active":"2024-09-13T14:51:11.058874+0000","last_became_peered":"2024-09-13T14:51:11.058874+0000","last_unstale":"2024-09-13T14:52:37.936010+0000","last_undegraded":"2024-09-13T14:52:37.936010+0000","last_fullsized":"2024-09-13T14:52:37.936010+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:15:14.736476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":385,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.950144+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:52:37.950144+0000","last_peered":"2024-09-13T14:52:37.950144+0000","last_clean":"2024-09-13T14:52:37.950144+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:52:37.950144+0000","last_undegraded":"2024-09-13T14:52:37.950144+0000","last_fullsized":"2024-09-13T14:52:37.950144+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"267'15162","reported_seq":18500,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.401382+0000","last_change":"2024-09-13T14:52:26.525258+0000","last_active":"2024-09-13T14:52:53.401382+0000","last_peered":"2024-09-13T14:52:53.401382+0000","last_clean":"2024-09-13T14:52:53.401382+0000","last_became_active":"2024-09-13T14:52:26.403535+0000","last_became_peered":"2024-09-13T14:52:26.403535+0000","last_unstale":"2024-09-13T14:52:53.401382+0000","last_undegraded":"2024-09-13T14:52:53.401382+0000","last_fullsized":"2024-09-13T14:52:53.401382+0000","mapping_epoch":263,"log_start":"236'7800","ondisk_log_start":"236'7800","created":223,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7362,"log_dups_size":0,"ondisk_log_size":7362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:07:51.400877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15162,"num_write_kb":485184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1609,"num_bytes_recovered":105447424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":395,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.444603+0000","last_change":"2024-09-13T14:48:32.204164+0000","last_active":"2024-09-13T14:52:32.444603+0000","last_peered":"2024-09-13T14:52:32.444603+0000","last_clean":"2024-09-13T14:52:32.444603+0000","last_became_active":"2024-09-13T14:48:32.203815+0000","last_became_peered":"2024-09-13T14:48:32.203815+0000","last_unstale":"2024-09-13T14:52:32.444603+0000","last_undegraded":"2024-09-13T14:52:32.444603+0000","last_fullsized":"2024-09-13T14:52:32.444603+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:22.639760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:22.639760+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:21.985178+0000","scrub_duration":0.0051120610000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":793,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.479686+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:52:32.479686+0000","last_peered":"2024-09-13T14:52:32.479686+0000","last_clean":"2024-09-13T14:52:32.479686+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:52:32.479686+0000","last_undegraded":"2024-09-13T14:52:32.479686+0000","last_fullsized":"2024-09-13T14:52:32.479686+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":56,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.947266+0000","last_change":"2024-09-13T14:51:11.059953+0000","last_active":"2024-09-13T14:52:37.947266+0000","last_peered":"2024-09-13T14:52:37.947266+0000","last_clean":"2024-09-13T14:52:37.947266+0000","last_became_active":"2024-09-13T14:51:11.059396+0000","last_became_peered":"2024-09-13T14:51:11.059396+0000","last_unstale":"2024-09-13T14:52:37.947266+0000","last_undegraded":"2024-09-13T14:52:37.947266+0000","last_fullsized":"2024-09-13T14:52:37.947266+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:11:51.119142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":51,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.445235+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T14:52:32.445235+0000","last_peered":"2024-09-13T14:52:32.445235+0000","last_clean":"2024-09-13T14:52:32.445235+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T14:52:32.445235+0000","last_undegraded":"2024-09-13T14:52:32.445235+0000","last_fullsized":"2024-09-13T14:52:32.445235+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"267'15152","reported_seq":15204,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:49.044306+0000","last_change":"2024-09-13T14:48:30.197655+0000","last_active":"2024-09-13T14:52:49.044306+0000","last_peered":"2024-09-13T14:52:49.044306+0000","last_clean":"2024-09-13T14:52:49.044306+0000","last_became_active":"2024-09-13T14:48:30.197442+0000","last_became_peered":"2024-09-13T14:48:30.197442+0000","last_unstale":"2024-09-13T14:52:49.044306+0000","last_undegraded":"2024-09-13T14:52:49.044306+0000","last_fullsized":"2024-09-13T14:52:49.044306+0000","mapping_epoch":223,"log_start":"236'7432","ondisk_log_start":"236'7432","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7720,"log_dups_size":0,"ondisk_log_size":7720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:44.452905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15152,"num_write_kb":484864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":523,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.951741+0000","last_change":"2024-09-13T14:48:32.204587+0000","last_active":"2024-09-13T14:52:37.951741+0000","last_peered":"2024-09-13T14:52:37.951741+0000","last_clean":"2024-09-13T14:52:37.951741+0000","last_became_active":"2024-09-13T14:48:32.204153+0000","last_became_peered":"2024-09-13T14:48:32.204153+0000","last_unstale":"2024-09-13T14:52:37.951741+0000","last_undegraded":"2024-09-13T14:52:37.951741+0000","last_fullsized":"2024-09-13T14:52:37.951741+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.784263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.784263+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:11:12.651038+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":478,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.444618+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:52:32.444618+0000","last_peered":"2024-09-13T14:52:32.444618+0000","last_clean":"2024-09-13T14:52:32.444618+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:52:32.444618+0000","last_undegraded":"2024-09-13T14:52:32.444618+0000","last_fullsized":"2024-09-13T14:52:32.444618+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":421,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.400661+0000","last_change":"2024-09-13T14:51:11.061677+0000","last_active":"2024-09-13T14:52:53.400661+0000","last_peered":"2024-09-13T14:52:53.400661+0000","last_clean":"2024-09-13T14:52:53.400661+0000","last_became_active":"2024-09-13T14:51:11.061205+0000","last_became_peered":"2024-09-13T14:51:11.061205+0000","last_unstale":"2024-09-13T14:52:53.400661+0000","last_undegraded":"2024-09-13T14:52:53.400661+0000","last_fullsized":"2024-09-13T14:52:53.400661+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:38.521784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":424,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.401536+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T14:52:53.401536+0000","last_peered":"2024-09-13T14:52:53.401536+0000","last_clean":"2024-09-13T14:52:53.401536+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T14:52:53.401536+0000","last_undegraded":"2024-09-13T14:52:53.401536+0000","last_fullsized":"2024-09-13T14:52:53.401536+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"267'15168","reported_seq":15271,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.400857+0000","last_change":"2024-09-13T14:48:30.196036+0000","last_active":"2024-09-13T14:52:53.400857+0000","last_peered":"2024-09-13T14:52:53.400857+0000","last_clean":"2024-09-13T14:52:53.400857+0000","last_became_active":"2024-09-13T14:48:30.195478+0000","last_became_peered":"2024-09-13T14:48:30.195478+0000","last_unstale":"2024-09-13T14:52:53.400857+0000","last_undegraded":"2024-09-13T14:52:53.400857+0000","last_fullsized":"2024-09-13T14:52:53.400857+0000","mapping_epoch":223,"log_start":"236'7800","ondisk_log_start":"236'7800","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7368,"log_dups_size":0,"ondisk_log_size":7368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:21:44.624753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15168,"num_write_kb":485376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":24,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.444791+0000","last_change":"2024-09-13T14:51:44.190446+0000","last_active":"2024-09-13T14:52:32.444791+0000","last_peered":"2024-09-13T14:52:32.444791+0000","last_clean":"2024-09-13T14:52:32.444791+0000","last_became_active":"2024-09-13T14:51:44.190115+0000","last_became_peered":"2024-09-13T14:51:44.190115+0000","last_unstale":"2024-09-13T14:52:32.444791+0000","last_undegraded":"2024-09-13T14:52:32.444791+0000","last_fullsized":"2024-09-13T14:52:32.444791+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:45:49.425578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:45:49.425578+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:43.724060+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":873,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.115635+0000","last_change":"2024-09-13T14:51:11.072066+0000","last_active":"2024-09-13T14:52:53.115635+0000","last_peered":"2024-09-13T14:52:53.115635+0000","last_clean":"2024-09-13T14:52:53.115635+0000","last_became_active":"2024-09-13T14:51:11.061307+0000","last_became_peered":"2024-09-13T14:51:11.061307+0000","last_unstale":"2024-09-13T14:52:53.115635+0000","last_undegraded":"2024-09-13T14:52:53.115635+0000","last_fullsized":"2024-09-13T14:52:53.115635+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:06.774106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":399,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.963604+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T14:52:37.963604+0000","last_peered":"2024-09-13T14:52:37.963604+0000","last_clean":"2024-09-13T14:52:37.963604+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T14:52:37.963604+0000","last_undegraded":"2024-09-13T14:52:37.963604+0000","last_fullsized":"2024-09-13T14:52:37.963604+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":418,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.950037+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:52:37.950037+0000","last_peered":"2024-09-13T14:52:37.950037+0000","last_clean":"2024-09-13T14:52:37.950037+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:52:37.950037+0000","last_undegraded":"2024-09-13T14:52:37.950037+0000","last_fullsized":"2024-09-13T14:52:37.950037+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"267'15132","reported_seq":17277,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:49.054030+0000","last_change":"2024-09-13T14:52:32.477979+0000","last_active":"2024-09-13T14:52:49.054030+0000","last_peered":"2024-09-13T14:52:49.054030+0000","last_clean":"2024-09-13T14:52:49.054030+0000","last_became_active":"2024-09-13T14:52:32.448142+0000","last_became_peered":"2024-09-13T14:52:32.448142+0000","last_unstale":"2024-09-13T14:52:49.054030+0000","last_undegraded":"2024-09-13T14:52:49.054030+0000","last_fullsized":"2024-09-13T14:52:49.054030+0000","mapping_epoch":265,"log_start":"235'7000","ondisk_log_start":"235'7000","created":223,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8132,"log_dups_size":0,"ondisk_log_size":8132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:01:22.201541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15132,"num_write_kb":484224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1035,"num_bytes_recovered":67829760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":167,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.479366+0000","last_change":"2024-09-13T14:40:26.896691+0000","last_active":"2024-09-13T14:52:32.479366+0000","last_peered":"2024-09-13T14:52:32.479366+0000","last_clean":"2024-09-13T14:52:32.479366+0000","last_became_active":"2024-09-13T14:40:22.949772+0000","last_became_peered":"2024-09-13T14:40:22.949772+0000","last_unstale":"2024-09-13T14:52:32.479366+0000","last_undegraded":"2024-09-13T14:52:32.479366+0000","last_fullsized":"2024-09-13T14:52:32.479366+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:26.896613+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:26.896613+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:01.615433+0000","scrub_duration":0.004120828,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":893,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.481693+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T14:52:32.481693+0000","last_peered":"2024-09-13T14:52:32.481693+0000","last_clean":"2024-09-13T14:52:32.481693+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T14:52:32.481693+0000","last_undegraded":"2024-09-13T14:52:32.481693+0000","last_fullsized":"2024-09-13T14:52:32.481693+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":252,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.494198+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T14:52:32.494198+0000","last_peered":"2024-09-13T14:52:32.494198+0000","last_clean":"2024-09-13T14:52:32.494198+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T14:52:32.494198+0000","last_undegraded":"2024-09-13T14:52:32.494198+0000","last_fullsized":"2024-09-13T14:52:32.494198+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":417,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.950478+0000","last_change":"2024-09-13T14:51:11.060994+0000","last_active":"2024-09-13T14:52:37.950478+0000","last_peered":"2024-09-13T14:52:37.950478+0000","last_clean":"2024-09-13T14:52:37.950478+0000","last_became_active":"2024-09-13T14:51:11.060666+0000","last_became_peered":"2024-09-13T14:51:11.060666+0000","last_unstale":"2024-09-13T14:52:37.950478+0000","last_undegraded":"2024-09-13T14:52:37.950478+0000","last_fullsized":"2024-09-13T14:52:37.950478+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:40:15.828766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"267'14858","reported_seq":19954,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:48.991098+0000","last_change":"2024-09-13T14:52:06.352163+0000","last_active":"2024-09-13T14:52:48.991098+0000","last_peered":"2024-09-13T14:52:48.991098+0000","last_clean":"2024-09-13T14:52:48.991098+0000","last_became_active":"2024-09-13T14:52:06.323005+0000","last_became_peered":"2024-09-13T14:52:06.323005+0000","last_unstale":"2024-09-13T14:52:48.991098+0000","last_undegraded":"2024-09-13T14:52:48.991098+0000","last_fullsized":"2024-09-13T14:52:48.991098+0000","mapping_epoch":260,"log_start":"235'6500","ondisk_log_start":"235'6500","created":223,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":8358,"log_dups_size":0,"ondisk_log_size":8358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:30.406030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14858,"num_write_kb":475456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2504,"num_bytes_recovered":164102144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":71,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.445179+0000","last_change":"2024-09-13T14:51:11.061598+0000","last_active":"2024-09-13T14:52:32.445179+0000","last_peered":"2024-09-13T14:52:32.445179+0000","last_clean":"2024-09-13T14:52:32.445179+0000","last_became_active":"2024-09-13T14:51:11.061164+0000","last_became_peered":"2024-09-13T14:51:11.061164+0000","last_unstale":"2024-09-13T14:52:32.445179+0000","last_undegraded":"2024-09-13T14:52:32.445179+0000","last_fullsized":"2024-09-13T14:52:32.445179+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:48:34.095505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:48:34.095505+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:03.076025+0000","scrub_duration":0.001382681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":248,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.479855+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:52:32.479855+0000","last_peered":"2024-09-13T14:52:32.479855+0000","last_clean":"2024-09-13T14:52:32.479855+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:52:32.479855+0000","last_undegraded":"2024-09-13T14:52:32.479855+0000","last_fullsized":"2024-09-13T14:52:32.479855+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":902,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.950644+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:52:37.950644+0000","last_peered":"2024-09-13T14:52:37.950644+0000","last_clean":"2024-09-13T14:52:37.950644+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:52:37.950644+0000","last_undegraded":"2024-09-13T14:52:37.950644+0000","last_fullsized":"2024-09-13T14:52:37.950644+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":374,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.951533+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:52:37.951533+0000","last_peered":"2024-09-13T14:52:37.951533+0000","last_clean":"2024-09-13T14:52:37.951533+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:52:37.951533+0000","last_undegraded":"2024-09-13T14:52:37.951533+0000","last_fullsized":"2024-09-13T14:52:37.951533+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"267'14748","reported_seq":14800,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:49.043635+0000","last_change":"2024-09-13T14:48:30.192664+0000","last_active":"2024-09-13T14:52:49.043635+0000","last_peered":"2024-09-13T14:52:49.043635+0000","last_clean":"2024-09-13T14:52:49.043635+0000","last_became_active":"2024-09-13T14:48:30.192434+0000","last_became_peered":"2024-09-13T14:48:30.192434+0000","last_unstale":"2024-09-13T14:52:49.043635+0000","last_undegraded":"2024-09-13T14:52:49.043635+0000","last_fullsized":"2024-09-13T14:52:49.043635+0000","mapping_epoch":223,"log_start":"234'5000","ondisk_log_start":"234'5000","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":9748,"log_dups_size":0,"ondisk_log_size":9748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:58.920518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14748,"num_write_kb":471936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":61,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.445713+0000","last_change":"2024-09-13T14:51:11.060132+0000","last_active":"2024-09-13T14:52:32.445713+0000","last_peered":"2024-09-13T14:52:32.445713+0000","last_clean":"2024-09-13T14:52:32.445713+0000","last_became_active":"2024-09-13T14:51:11.059785+0000","last_became_peered":"2024-09-13T14:51:11.059785+0000","last_unstale":"2024-09-13T14:52:32.445713+0000","last_undegraded":"2024-09-13T14:52:32.445713+0000","last_fullsized":"2024-09-13T14:52:32.445713+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.334379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.334379+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:44.298431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":399,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.950563+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T14:52:37.950563+0000","last_peered":"2024-09-13T14:52:37.950563+0000","last_clean":"2024-09-13T14:52:37.950563+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T14:52:37.950563+0000","last_undegraded":"2024-09-13T14:52:37.950563+0000","last_fullsized":"2024-09-13T14:52:37.950563+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":480,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.445396+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:52:32.445396+0000","last_peered":"2024-09-13T14:52:32.445396+0000","last_clean":"2024-09-13T14:52:32.445396+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:52:32.445396+0000","last_undegraded":"2024-09-13T14:52:32.445396+0000","last_fullsized":"2024-09-13T14:52:32.445396+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":389,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.400972+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:52:53.400972+0000","last_peered":"2024-09-13T14:52:53.400972+0000","last_clean":"2024-09-13T14:52:53.400972+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:52:53.400972+0000","last_undegraded":"2024-09-13T14:52:53.400972+0000","last_fullsized":"2024-09-13T14:52:53.400972+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"267'15202","reported_seq":15303,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:49.028230+0000","last_change":"2024-09-13T14:48:30.197226+0000","last_active":"2024-09-13T14:52:49.028230+0000","last_peered":"2024-09-13T14:52:49.028230+0000","last_clean":"2024-09-13T14:52:49.028230+0000","last_became_active":"2024-09-13T14:48:30.195243+0000","last_became_peered":"2024-09-13T14:48:30.195243+0000","last_unstale":"2024-09-13T14:52:49.028230+0000","last_undegraded":"2024-09-13T14:52:49.028230+0000","last_fullsized":"2024-09-13T14:52:49.028230+0000","mapping_epoch":223,"log_start":"237'8217","ondisk_log_start":"237'8217","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":6985,"log_dups_size":0,"ondisk_log_size":6985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:46:06.507375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15202,"num_write_kb":486464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":198,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.494016+0000","last_change":"2024-09-13T14:51:01.998101+0000","last_active":"2024-09-13T14:52:32.494016+0000","last_peered":"2024-09-13T14:52:32.494016+0000","last_clean":"2024-09-13T14:52:32.494016+0000","last_became_active":"2024-09-13T14:51:01.997688+0000","last_became_peered":"2024-09-13T14:51:01.997688+0000","last_unstale":"2024-09-13T14:52:32.494016+0000","last_undegraded":"2024-09-13T14:52:32.494016+0000","last_fullsized":"2024-09-13T14:52:32.494016+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:40:30.500046+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:02:51.444049+0000","scrub_duration":0.00248832,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":400,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.962669+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:52:37.962669+0000","last_peered":"2024-09-13T14:52:37.962669+0000","last_clean":"2024-09-13T14:52:37.962669+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:52:37.962669+0000","last_undegraded":"2024-09-13T14:52:37.962669+0000","last_fullsized":"2024-09-13T14:52:37.962669+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":757,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.949933+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T14:52:37.949933+0000","last_peered":"2024-09-13T14:52:37.949933+0000","last_clean":"2024-09-13T14:52:37.949933+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T14:52:37.949933+0000","last_undegraded":"2024-09-13T14:52:37.949933+0000","last_fullsized":"2024-09-13T14:52:37.949933+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":63,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.936113+0000","last_change":"2024-09-13T14:51:44.203186+0000","last_active":"2024-09-13T14:52:37.936113+0000","last_peered":"2024-09-13T14:52:37.936113+0000","last_clean":"2024-09-13T14:52:37.936113+0000","last_became_active":"2024-09-13T14:51:44.202851+0000","last_became_peered":"2024-09-13T14:51:44.202851+0000","last_unstale":"2024-09-13T14:52:37.936113+0000","last_undegraded":"2024-09-13T14:52:37.936113+0000","last_fullsized":"2024-09-13T14:52:37.936113+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:05:49.173340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"267'15022","reported_seq":15123,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:49.022257+0000","last_change":"2024-09-13T14:48:30.195027+0000","last_active":"2024-09-13T14:52:49.022257+0000","last_peered":"2024-09-13T14:52:49.022257+0000","last_clean":"2024-09-13T14:52:49.022257+0000","last_became_active":"2024-09-13T14:48:30.194672+0000","last_became_peered":"2024-09-13T14:48:30.194672+0000","last_unstale":"2024-09-13T14:52:49.022257+0000","last_undegraded":"2024-09-13T14:52:49.022257+0000","last_fullsized":"2024-09-13T14:52:49.022257+0000","mapping_epoch":223,"log_start":"236'7967","ondisk_log_start":"236'7967","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:48:29.166540+0000","last_clean_scrub_stamp":"2024-09-13T14:48:29.166540+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:38:50.182084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15022,"num_write_kb":480704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":330,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.948262+0000","last_change":"2024-09-13T14:40:24.124750+0000","last_active":"2024-09-13T14:52:37.948262+0000","last_peered":"2024-09-13T14:52:37.948262+0000","last_clean":"2024-09-13T14:52:37.948262+0000","last_became_active":"2024-09-13T14:40:22.958820+0000","last_became_peered":"2024-09-13T14:40:22.958820+0000","last_unstale":"2024-09-13T14:52:37.948262+0000","last_undegraded":"2024-09-13T14:52:37.948262+0000","last_fullsized":"2024-09-13T14:52:37.948262+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:24.124531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:24.124531+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:47:30.267886+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":410,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.399894+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:52:53.399894+0000","last_peered":"2024-09-13T14:52:53.399894+0000","last_clean":"2024-09-13T14:52:53.399894+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:52:53.399894+0000","last_undegraded":"2024-09-13T14:52:53.399894+0000","last_fullsized":"2024-09-13T14:52:53.399894+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1080,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.445478+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T14:52:32.445478+0000","last_peered":"2024-09-13T14:52:32.445478+0000","last_clean":"2024-09-13T14:52:32.445478+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T14:52:32.445478+0000","last_undegraded":"2024-09-13T14:52:32.445478+0000","last_fullsized":"2024-09-13T14:52:32.445478+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":234,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.493824+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T14:52:32.493824+0000","last_peered":"2024-09-13T14:52:32.493824+0000","last_clean":"2024-09-13T14:52:32.493824+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T14:52:32.493824+0000","last_undegraded":"2024-09-13T14:52:32.493824+0000","last_fullsized":"2024-09-13T14:52:32.493824+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":251,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.494185+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T14:52:32.494185+0000","last_peered":"2024-09-13T14:52:32.494185+0000","last_clean":"2024-09-13T14:52:32.494185+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T14:52:32.494185+0000","last_undegraded":"2024-09-13T14:52:32.494185+0000","last_fullsized":"2024-09-13T14:52:32.494185+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":418,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.400984+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T14:52:53.400984+0000","last_peered":"2024-09-13T14:52:53.400984+0000","last_clean":"2024-09-13T14:52:53.400984+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T14:52:53.400984+0000","last_undegraded":"2024-09-13T14:52:53.400984+0000","last_fullsized":"2024-09-13T14:52:53.400984+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":398,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.951092+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T14:52:37.951092+0000","last_peered":"2024-09-13T14:52:37.951092+0000","last_clean":"2024-09-13T14:52:37.951092+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T14:52:37.951092+0000","last_undegraded":"2024-09-13T14:52:37.951092+0000","last_fullsized":"2024-09-13T14:52:37.951092+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":247,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.445195+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:52:32.445195+0000","last_peered":"2024-09-13T14:52:32.445195+0000","last_clean":"2024-09-13T14:52:32.445195+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:52:32.445195+0000","last_undegraded":"2024-09-13T14:52:32.445195+0000","last_fullsized":"2024-09-13T14:52:32.445195+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":231,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.480015+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:52:32.480015+0000","last_peered":"2024-09-13T14:52:32.480015+0000","last_clean":"2024-09-13T14:52:32.480015+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:52:32.480015+0000","last_undegraded":"2024-09-13T14:52:32.480015+0000","last_fullsized":"2024-09-13T14:52:32.480015+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":368,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.946833+0000","last_change":"2024-09-13T14:48:32.200953+0000","last_active":"2024-09-13T14:52:37.946833+0000","last_peered":"2024-09-13T14:52:37.946833+0000","last_clean":"2024-09-13T14:52:37.946833+0000","last_became_active":"2024-09-13T14:48:32.200634+0000","last_became_peered":"2024-09-13T14:48:32.200634+0000","last_unstale":"2024-09-13T14:52:37.946833+0000","last_undegraded":"2024-09-13T14:52:37.946833+0000","last_fullsized":"2024-09-13T14:52:37.946833+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:18.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":398,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.401453+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:52:53.401453+0000","last_peered":"2024-09-13T14:52:53.401453+0000","last_clean":"2024-09-13T14:52:53.401453+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:52:53.401453+0000","last_undegraded":"2024-09-13T14:52:53.401453+0000","last_fullsized":"2024-09-13T14:52:53.401453+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":417,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.936766+0000","last_change":"2024-09-13T14:48:32.201812+0000","last_active":"2024-09-13T14:52:37.936766+0000","last_peered":"2024-09-13T14:52:37.936766+0000","last_clean":"2024-09-13T14:52:37.936766+0000","last_became_active":"2024-09-13T14:48:32.201486+0000","last_became_peered":"2024-09-13T14:48:32.201486+0000","last_unstale":"2024-09-13T14:52:37.936766+0000","last_undegraded":"2024-09-13T14:52:37.936766+0000","last_fullsized":"2024-09-13T14:52:37.936766+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:23:29.693906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":51,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.445254+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T14:52:32.445254+0000","last_peered":"2024-09-13T14:52:32.445254+0000","last_clean":"2024-09-13T14:52:32.445254+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T14:52:32.445254+0000","last_undegraded":"2024-09-13T14:52:32.445254+0000","last_fullsized":"2024-09-13T14:52:32.445254+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":421,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.936662+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:52:37.936662+0000","last_peered":"2024-09-13T14:52:37.936662+0000","last_clean":"2024-09-13T14:52:37.936662+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:52:37.936662+0000","last_undegraded":"2024-09-13T14:52:37.936662+0000","last_fullsized":"2024-09-13T14:52:37.936662+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":421,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.401447+0000","last_change":"2024-09-13T14:51:44.198771+0000","last_active":"2024-09-13T14:52:53.401447+0000","last_peered":"2024-09-13T14:52:53.401447+0000","last_clean":"2024-09-13T14:52:53.401447+0000","last_became_active":"2024-09-13T14:51:44.198466+0000","last_became_peered":"2024-09-13T14:51:44.198466+0000","last_unstale":"2024-09-13T14:52:53.401447+0000","last_undegraded":"2024-09-13T14:52:53.401447+0000","last_fullsized":"2024-09-13T14:52:53.401447+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:41:16.099954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":384,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.964224+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:52:37.964224+0000","last_peered":"2024-09-13T14:52:37.964224+0000","last_clean":"2024-09-13T14:52:37.964224+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:52:37.964224+0000","last_undegraded":"2024-09-13T14:52:37.964224+0000","last_fullsized":"2024-09-13T14:52:37.964224+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":416,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.946758+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:52:37.946758+0000","last_peered":"2024-09-13T14:52:37.946758+0000","last_clean":"2024-09-13T14:52:37.946758+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:52:37.946758+0000","last_undegraded":"2024-09-13T14:52:37.946758+0000","last_fullsized":"2024-09-13T14:52:37.946758+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":250,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.493892+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:52:32.493892+0000","last_peered":"2024-09-13T14:52:32.493892+0000","last_clean":"2024-09-13T14:52:32.493892+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:52:32.493892+0000","last_undegraded":"2024-09-13T14:52:32.493892+0000","last_fullsized":"2024-09-13T14:52:32.493892+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":62,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.115508+0000","last_change":"2024-09-13T14:51:02.029301+0000","last_active":"2024-09-13T14:52:53.115508+0000","last_peered":"2024-09-13T14:52:53.115508+0000","last_clean":"2024-09-13T14:52:53.115508+0000","last_became_active":"2024-09-13T14:51:02.029032+0000","last_became_peered":"2024-09-13T14:51:02.029032+0000","last_unstale":"2024-09-13T14:52:53.115508+0000","last_undegraded":"2024-09-13T14:52:53.115508+0000","last_fullsized":"2024-09-13T14:52:53.115508+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:40.150320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":390,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.400915+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:52:53.400915+0000","last_peered":"2024-09-13T14:52:53.400915+0000","last_clean":"2024-09-13T14:52:53.400915+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:52:53.400915+0000","last_undegraded":"2024-09-13T14:52:53.400915+0000","last_fullsized":"2024-09-13T14:52:53.400915+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":248,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.494125+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:52:32.494125+0000","last_peered":"2024-09-13T14:52:32.494125+0000","last_clean":"2024-09-13T14:52:32.494125+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:52:32.494125+0000","last_undegraded":"2024-09-13T14:52:32.494125+0000","last_fullsized":"2024-09-13T14:52:32.494125+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":233,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.479642+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:52:32.479642+0000","last_peered":"2024-09-13T14:52:32.479642+0000","last_clean":"2024-09-13T14:52:32.479642+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:52:32.479642+0000","last_undegraded":"2024-09-13T14:52:32.479642+0000","last_fullsized":"2024-09-13T14:52:32.479642+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":248,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.493842+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:52:32.493842+0000","last_peered":"2024-09-13T14:52:32.493842+0000","last_clean":"2024-09-13T14:52:32.493842+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:52:32.493842+0000","last_undegraded":"2024-09-13T14:52:32.493842+0000","last_fullsized":"2024-09-13T14:52:32.493842+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":272,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.479843+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T14:52:32.479843+0000","last_peered":"2024-09-13T14:52:32.479843+0000","last_clean":"2024-09-13T14:52:32.479843+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T14:52:32.479843+0000","last_undegraded":"2024-09-13T14:52:32.479843+0000","last_fullsized":"2024-09-13T14:52:32.479843+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":222,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.479441+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:52:32.479441+0000","last_peered":"2024-09-13T14:52:32.479441+0000","last_clean":"2024-09-13T14:52:32.479441+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:52:32.479441+0000","last_undegraded":"2024-09-13T14:52:32.479441+0000","last_fullsized":"2024-09-13T14:52:32.479441+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":424,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:52:37.947138+0000","last_change":"2024-09-13T14:48:32.202404+0000","last_active":"2024-09-13T14:52:37.947138+0000","last_peered":"2024-09-13T14:52:37.947138+0000","last_clean":"2024-09-13T14:52:37.947138+0000","last_became_active":"2024-09-13T14:48:32.202064+0000","last_became_peered":"2024-09-13T14:48:32.202064+0000","last_unstale":"2024-09-13T14:52:37.947138+0000","last_undegraded":"2024-09-13T14:52:37.947138+0000","last_fullsized":"2024-09-13T14:52:37.947138+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:25.674819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":254,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:52:32.497350+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T14:52:32.497350+0000","last_peered":"2024-09-13T14:52:32.497350+0000","last_clean":"2024-09-13T14:52:32.497350+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T14:52:32.497350+0000","last_undegraded":"2024-09-13T14:52:32.497350+0000","last_fullsized":"2024-09-13T14:52:32.497350+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":442,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-09-13T14:52:53.401452+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T14:52:53.401452+0000","last_peered":"2024-09-13T14:52:53.401452+0000","last_clean":"2024-09-13T14:52:53.401452+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T14:52:53.401452+0000","last_undegraded":"2024-09-13T14:52:53.401452+0000","last_fullsized":"2024-09-13T14:52:53.401452+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":240258,"num_write_kb":7688193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8795,"num_bytes_recovered":576389120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17104896,"data_stored":17104896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126721,"ondisk_log_size":126721,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":16172,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397827,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":318060,"kb_used_data":4068,"kb_used_omap":1,"kb_used_meta":313982,"kb_avail":93427092,"statfs":{"total":95995035648,"available":95669342208,"internally_reserved":0,"allocated":4165632,"data_stored":2363832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":321517912},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594056,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":305188,"kb_used_data":7836,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93439964,"statfs":{"total":95995035648,"available":95682523136,"internally_reserved":0,"allocated":8024064,"data_stored":6230028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790285,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":623328,"kb_used_data":25112,"kb_used_omap":1,"kb_used_meta":598206,"kb_avail":93121824,"statfs":{"total":95995035648,"available":95356747776,"internally_reserved":0,"allocated":25714688,"data_stored":23921189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612563288},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986514,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":597792,"kb_used_data":3736,"kb_used_omap":1,"kb_used_meta":594046,"kb_avail":93147360,"statfs":{"total":95995035648,"available":95382896640,"internally_reserved":0,"allocated":3825664,"data_stored":2031352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":608303448},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598962,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":596052,"kb_used_data":4300,"kb_used_omap":6,"kb_used_meta":591737,"kb_avail":93149100,"statfs":{"total":95995035648,"available":95384678400,"internally_reserved":0,"allocated":4403200,"data_stored":2621358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":605939680},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762489,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":613920,"kb_used_data":3992,"kb_used_omap":8,"kb_used_meta":609911,"kb_avail":93131232,"statfs":{"total":95995035648,"available":95366381568,"internally_reserved":0,"allocated":4087808,"data_stored":2294304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":624549322},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926015,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":552664,"kb_used_data":7824,"kb_used_omap":7,"kb_used_meta":544824,"kb_avail":93192488,"statfs":{"total":95995035648,"available":95429107712,"internally_reserved":0,"allocated":8011776,"data_stored":6225557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":557900489},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122255,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":525996,"kb_used_data":3748,"kb_used_omap":7,"kb_used_meta":522232,"kb_avail":93219156,"statfs":{"total":95995035648,"available":95456415744,"internally_reserved":0,"allocated":3837952,"data_stored":2036660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":534766159},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":17104896,"data_stored":17104896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:52:56.460 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T14:52:56.460 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:52:56.460 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-09-13T14:52:56.461 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.0 'trim stale osdmaps' 2024-09-13T14:52:56.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:56 smithi204 ceph-mon[87071]: from='client.35877 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:56.537 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:56.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.559 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:56.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:56.660 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:56.687 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:56.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:56.789 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:56.792 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:56.814 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:56.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:56.916 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:56.937 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:56.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:57.039 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:57.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.066 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:57.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:57.127 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:57.166 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:57.192 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:57.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:57.294 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:57.316 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:57.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:57.371 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:57 smithi096 ceph-mon[98901]: pgmap v799: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 519 op/s 2024-09-13T14:52:57.371 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:57 smithi096 ceph-mon[102928]: pgmap v799: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 519 op/s 2024-09-13T14:52:57.417 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:57.443 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:57.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:57.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:57 smithi204 ceph-mon[87071]: pgmap v799: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 519 op/s 2024-09-13T14:52:57.545 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:57.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.567 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:57.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:57.668 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:57.698 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:57.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:57.800 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:57.822 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:57.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:57.924 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:57.945 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:57.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:58.046 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:58.072 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:52:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:52:57] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.51.0" 2024-09-13T14:52:58.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:58.174 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:58.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.196 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:58.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:58.297 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:58.323 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:58.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:58.425 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:58.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.447 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:58.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:58.548 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:58.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.570 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:58.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:58.671 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:58.696 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:58.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:58.798 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:58.819 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:58.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:58.921 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.943 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:58.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:59.044 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:59.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.066 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:59.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:59.167 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:59.193 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:59.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:59.295 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:59.317 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:59.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:59.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:52:59 smithi096 ceph-mon[102928]: pgmap v800: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 362 op/s 2024-09-13T14:52:59.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:52:59 smithi096 ceph-mon[98901]: pgmap v800: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 362 op/s 2024-09-13T14:52:59.419 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:59.445 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:59.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:59.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:52:59 smithi204 ceph-mon[87071]: pgmap v800: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 362 op/s 2024-09-13T14:52:59.548 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:59.574 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:52:59.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:59.675 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:59.698 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:59.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:59.800 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:59.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.822 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:59.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:59.923 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:59.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.945 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:52:59.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:00.046 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:00.068 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:00.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:00.169 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:00.191 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:00.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:00.293 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:00.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.316 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:00.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:00.417 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:00.443 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:00.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:53:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:53:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:00.546 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:00.571 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:00.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:00.673 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:00.699 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:00.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:00.801 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:00.823 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:00.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:00.925 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:00.951 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:00.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:01.053 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:01.079 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:01.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:01.083 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:01.181 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:01.205 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:01.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:01.307 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:01.330 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:01.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:01.397 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:01 smithi096 ceph-mon[98901]: pgmap v801: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 105 op/s 2024-09-13T14:53:01.397 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:01 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:01.397 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:01 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:01.398 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:01 smithi096 ceph-mon[102928]: pgmap v801: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 105 op/s 2024-09-13T14:53:01.398 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:01 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:01.398 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:01 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:01.432 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:01.454 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:01.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:01 smithi204 ceph-mon[87071]: pgmap v801: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 105 op/s 2024-09-13T14:53:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:01 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:01 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:01.556 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:01.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.581 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:01.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:01.682 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:01.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.704 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:01.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:01.805 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:01.827 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:01.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:01.929 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:01.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.950 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:01.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:02.051 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:02.072 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:02.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:02.175 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:02.196 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:02.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:02.244 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:02 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:53:02.245 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:02 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:53:02.245 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:02 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:02.245 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:02 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:53:02.245 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:02 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:53:02.245 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:02 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:02.298 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:02.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.321 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:02.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:02.422 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:02.692 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:53:02.692 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:53:02.693 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:53:02.693 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:53:02.694 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:53:02.694 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:53:02.694 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:53:02.695 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:53:02.695 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:02 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:53:02.696 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:02 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:53:02.696 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:02 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:02.700 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:53:02.701 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:53:02.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.717 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:02.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:02.718 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:53:02.718 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:53:02.733 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:53:02.734 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:53:02.750 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:53:02.751 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:53:02.818 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:02.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.842 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:02.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:02.943 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:02.965 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:02.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:03.068 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:03.089 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:03.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:03.191 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:03.214 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:03.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:03.315 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:03.338 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:03.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:03.440 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:03.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.462 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:03.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:03.521 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'unique_pool_2' removed 2024-09-13T14:53:03.538 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:03 smithi096 ceph-mon[98901]: pgmap v802: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 44 op/s 2024-09-13T14:53:03.538 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:03 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4118010825' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:03.538 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:03 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2036459934' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:53:03.538 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:03 smithi096 ceph-mon[102928]: pgmap v802: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 44 op/s 2024-09-13T14:53:03.538 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:03 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4118010825' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:03.539 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:03 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2036459934' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:53:03.563 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:03.587 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:03.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:03.689 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:03.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.712 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:03.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:03.813 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:03.836 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:03.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:03.938 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:03.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.961 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:03.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:03.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:03 smithi204 ceph-mon[87071]: pgmap v802: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 44 op/s 2024-09-13T14:53:03.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:03 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4118010825' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:03.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:03 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2036459934' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:53:04.059 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:53:04.059 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:53:04.062 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:04.086 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:04.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:04.187 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:04.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.210 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:04.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:04.311 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:04.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.333 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:04.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:04.434 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:04.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.456 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:04.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:04.557 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:04.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.592 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:04.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:04.693 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:04.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.716 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:04.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:04.817 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:04.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:04 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2036459934' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:53:04.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:04 smithi096 ceph-mon[98901]: osdmap e269: 8 total, 8 up, 8 in 2024-09-13T14:53:04.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:04 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2036459934' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:53:04.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:04 smithi096 ceph-mon[102928]: osdmap e269: 8 total, 8 up, 8 in 2024-09-13T14:53:04.838 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:04.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:04.940 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:04.946 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:53:04.947 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:53:04.947 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-09-13T14:53:04.948 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_3 16 2024-09-13T14:53:04.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.964 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:04.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:04.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:04 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2036459934' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:53:04.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:04 smithi204 ceph-mon[87071]: osdmap e269: 8 total, 8 up, 8 in 2024-09-13T14:53:05.065 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:05.087 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:05.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:05.189 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:05.210 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:05.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:05.233 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:05.312 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:05.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.336 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:05.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:05.437 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:05.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.459 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:05.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:05.560 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:05.579 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:05 smithi096 ceph-mon[98901]: pgmap v804: 83 pgs: 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:05.579 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:05 smithi096 ceph-mon[102928]: pgmap v804: 83 pgs: 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:05.585 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:05.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:05.601 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:53:05.686 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:05.710 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:05.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:05.812 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:05.836 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:05.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:05.937 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:05.962 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:05.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:05.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:05 smithi204 ceph-mon[87071]: pgmap v804: 83 pgs: 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:06.064 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:06.088 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:06.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:06.191 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:06.215 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:06.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:06.317 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:06.340 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:06.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:06.407 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:53:06.443 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:06.466 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:06.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:06.568 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:06.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.591 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:06.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:06.674 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:06.674 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:06.693 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:06.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.714 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:06.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:06.816 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:06.842 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:06.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:06.944 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:06.966 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:06.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:06.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:07.068 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:07.089 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:07.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:07.170 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:07.170 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:07.191 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:07.214 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:07.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:07.316 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:07.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.339 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:07.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:07.441 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:07.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.463 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:07.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:07.565 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:07.591 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:07.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:07.693 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:07.719 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:07.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:07.747 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:07 smithi096 ceph-mon[98901]: pgmap v805: 83 pgs: 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:07.748 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:07 smithi096 ceph-mon[102928]: pgmap v805: 83 pgs: 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:07.820 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:07.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.843 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:07.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:07.944 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:07.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.966 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:07.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:07.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:07 smithi204 ceph-mon[87071]: pgmap v805: 83 pgs: 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:08.067 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:08.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:53:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:53:07] "GET /metrics HTTP/1.1" 200 34147 "" "Prometheus/2.51.0" 2024-09-13T14:53:08.089 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:08.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:08.096 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T14:53:08.097 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T14:53:08.097 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T14:53:08.097 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T14:53:08.097 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T14:53:08.097 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T14:53:08.097 INFO:teuthology.orchestra.run.smithi096.stderr:bench [] [] [] [] 2024-09-13T14:53:08.097 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:53:08.097 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T14:53:08.097 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T14:53:08.097 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore allocator dump block 2024-09-13T14:53:08.098 INFO:teuthology.orchestra.run.smithi096.stderr:Error EINVAL: invalid command 2024-09-13T14:53:08.191 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:08.217 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:08.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:08.319 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:08.344 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:08.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:08.446 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:08.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.469 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:08.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:08.570 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:08.596 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:08.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:08.699 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:08.722 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:08.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:08.824 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:08.850 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:08.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:08.951 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:08.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.976 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:08.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:09.044 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:53:09.044 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:53:09.044 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.1 'trim stale osdmaps' 2024-09-13T14:53:09.077 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:09.109 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:09.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:09.211 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:09.237 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:09.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:09.339 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:09.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.365 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:09.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:09.436 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:09.466 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:09.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.493 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:09.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:09.595 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:09.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.622 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:09.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:09.723 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:09.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.749 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:09.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:09.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:09 smithi096 ceph-mon[98901]: pgmap v806: 83 pgs: 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:09.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:09 smithi096 ceph-mon[102928]: pgmap v806: 83 pgs: 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:09.850 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:09.875 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:09.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:09.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:09 smithi204 ceph-mon[87071]: pgmap v806: 83 pgs: 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:09.978 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:10.005 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:10.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:10.106 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:10.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.129 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:10.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:10.230 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:10.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.255 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:10.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:10.356 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:10.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.378 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:10.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:53:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:53:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:10.478 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:10.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.501 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:10.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:10.602 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:10.628 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:10.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:10.730 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:10.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.753 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:10.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:10.854 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:10.877 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:10.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:10.979 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:11.005 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:11.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:11.107 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:11.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.130 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:11.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:11.231 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:11.252 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:11.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:11.354 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:11.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.377 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:11.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:11.482 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:11.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.504 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:11.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:11.605 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:11.627 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:11.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:11.728 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:11.750 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:11.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:11.804 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:11 smithi096 ceph-mon[98901]: pgmap v807: 83 pgs: 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:11.805 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:11 smithi096 ceph-mon[102928]: pgmap v807: 83 pgs: 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:11.852 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:11.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.874 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:11.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:11 smithi204 ceph-mon[87071]: pgmap v807: 83 pgs: 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:11.975 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:12.001 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:12.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:12.103 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:12.129 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:12.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:12.230 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:12.256 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:12.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:12.357 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:12.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.380 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:12.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:12.481 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:12.502 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:12.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:12.556 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'unique_pool_3' created 2024-09-13T14:53:12.604 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:12.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.627 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:12.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:12.729 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:12.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.751 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:12.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:12.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:12 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2194027082' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T14:53:12.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:12 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2194027082' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T14:53:12.852 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:12.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.875 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:12.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:12.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:12 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2194027082' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T14:53:12.975 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:13.001 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:13.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:13.104 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:13.126 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:13.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:13.227 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:13.254 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:13.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:13.356 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:13.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.379 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:13.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:13.480 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:13.501 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:13.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:13.603 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:13.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.625 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:13.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:13.726 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:13.731 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-09-13T14:53:13.752 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:13.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:13.816 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:53:13.816 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:53:13.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:13 smithi096 ceph-mon[102928]: pgmap v808: 83 pgs: 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:13.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:13 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2194027082' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-13T14:53:13.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:13 smithi096 ceph-mon[102928]: osdmap e270: 8 total, 8 up, 8 in 2024-09-13T14:53:13.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:13 smithi096 ceph-mon[98901]: pgmap v808: 83 pgs: 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:13.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:13 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2194027082' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-13T14:53:13.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:13 smithi096 ceph-mon[98901]: osdmap e270: 8 total, 8 up, 8 in 2024-09-13T14:53:13.854 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:13.881 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:13.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:13.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:13 smithi204 ceph-mon[87071]: pgmap v808: 83 pgs: 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:13.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:13 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2194027082' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-13T14:53:13.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:13 smithi204 ceph-mon[87071]: osdmap e270: 8 total, 8 up, 8 in 2024-09-13T14:53:13.982 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:14.005 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:14.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:14.107 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:14.129 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:14.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:14.231 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:14.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.254 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:14.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:14.355 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:14.375 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:14.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:14.477 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:14.489 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:53:14.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.501 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:14.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:14.603 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:14.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.630 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:14.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:14.731 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:14.752 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:14.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:14.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:14 smithi096 ceph-mon[98901]: osdmap e271: 8 total, 8 up, 8 in 2024-09-13T14:53:14.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:14 smithi096 ceph-mon[102928]: osdmap e271: 8 total, 8 up, 8 in 2024-09-13T14:53:14.854 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:14.880 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:14.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:14.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:14 smithi204 ceph-mon[87071]: osdmap e271: 8 total, 8 up, 8 in 2024-09-13T14:53:14.982 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:15.003 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:15.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:15.105 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:15.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.127 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:15.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:15.228 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:15.254 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:15.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:15.356 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:15.381 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:15.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:15.415 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:53:15.415 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:53:15.432 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:53:15.432 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:53:15.447 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:53:15.448 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:53:15.464 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:53:15.464 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:53:15.480 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:53:15.480 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:53:15.483 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:15.496 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:53:15.497 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:53:15.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.505 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:15.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:15.512 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:53:15.512 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:53:15.528 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:53:15.528 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:53:15.607 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:15.627 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:15.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:15.729 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:15.754 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:15.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:15.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:15 smithi096 ceph-mon[98901]: pgmap v811: 99 pgs: 10 creating+peering, 6 unknown, 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:15.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:15 smithi096 ceph-mon[98901]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:53:15.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:15 smithi096 ceph-mon[98901]: osdmap e272: 8 total, 8 up, 8 in 2024-09-13T14:53:15.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:15 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3243875566' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:15.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:15 smithi096 ceph-mon[102928]: pgmap v811: 99 pgs: 10 creating+peering, 6 unknown, 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:15.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:15 smithi096 ceph-mon[102928]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:53:15.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:15 smithi096 ceph-mon[102928]: osdmap e272: 8 total, 8 up, 8 in 2024-09-13T14:53:15.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:15 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3243875566' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:15.857 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:15.878 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:15.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:15.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:15 smithi204 ceph-mon[87071]: pgmap v811: 99 pgs: 10 creating+peering, 6 unknown, 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:15.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:15 smithi204 ceph-mon[87071]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:53:15.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:15 smithi204 ceph-mon[87071]: osdmap e272: 8 total, 8 up, 8 in 2024-09-13T14:53:15.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:15 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3243875566' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:15.979 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:16.005 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:16.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:16.107 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:16.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.129 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:16.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:16.230 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:16.251 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:16.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:16.353 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:16.379 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:16.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:16.481 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:16.505 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:16.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:16.507 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:16.511 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:16.512 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:16.588 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:53:16.607 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:16.633 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:16.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:16.685 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:16 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2318685226' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:16.686 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:16 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2318685226' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:16.735 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:16.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.758 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:16.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:16.859 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:16.881 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:16.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:16.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:16 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2318685226' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:16.982 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:17.008 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:17.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:17.110 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:17.133 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:17.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:17.234 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:17.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.261 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:17.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:17.362 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:17.386 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:17.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:17.488 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:17.511 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:17.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:17.613 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:17.635 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:17.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:17.737 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:17.763 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:17.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:17.865 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:17.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:17.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:17 smithi096 ceph-mon[102928]: pgmap v813: 99 pgs: 10 creating+peering, 6 unknown, 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:17.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:17 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:17.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:17 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2318685226' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:17.948 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:17 smithi096 ceph-mon[102928]: osdmap e273: 8 total, 8 up, 8 in 2024-09-13T14:53:17.948 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:17 smithi096 ceph-mon[98901]: pgmap v813: 99 pgs: 10 creating+peering, 6 unknown, 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:17.948 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:17 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:17.948 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:17 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2318685226' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:17.948 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:17 smithi096 ceph-mon[98901]: osdmap e273: 8 total, 8 up, 8 in 2024-09-13T14:53:17.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:17 smithi204 ceph-mon[87071]: pgmap v813: 99 pgs: 10 creating+peering, 6 unknown, 83 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:17.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:17 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:17.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:17 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2318685226' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:17.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:17 smithi204 ceph-mon[87071]: osdmap e273: 8 total, 8 up, 8 in 2024-09-13T14:53:17.989 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:18.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.011 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:18.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:18.112 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:18.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.140 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:18.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:18.241 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:18.262 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:18.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:18.292 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:53:18.310 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:53:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:53:17] "GET /metrics HTTP/1.1" 200 34147 "" "Prometheus/2.51.0" 2024-09-13T14:53:18.365 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:18.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.393 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:18.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:18.494 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:18.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.520 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:18.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:18.621 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:18.643 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:18.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:18.745 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:18.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.773 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:18.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:18.874 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:18.900 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:18.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:19.002 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:19.027 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:19.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:19.129 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:19.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.154 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:19.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:19.258 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:19.284 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:19.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:19.386 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:19.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.408 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:19.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:19.452 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:19.508 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:19.530 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:19.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:19.632 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:19.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.655 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:19.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:19.756 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:19.783 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:19.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:19 smithi096 ceph-mon[98901]: pgmap v815: 99 pgs: 10 creating+peering, 89 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:19.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:19 smithi096 ceph-mon[102928]: pgmap v815: 99 pgs: 10 creating+peering, 89 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:19.885 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:19.907 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:19.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:19.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:19 smithi204 ceph-mon[87071]: pgmap v815: 99 pgs: 10 creating+peering, 89 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:20.008 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:20.030 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:20.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:20.131 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:20.153 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:20.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:20.255 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:20.281 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:20.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:20.312 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:53:20.383 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:20.406 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:20.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:53:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:53:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:20.508 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:20.530 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:20.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:20.602 INFO:teuthology.orchestra.run.smithi096.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-09-13T14:53:20.632 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:20.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.655 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:20.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:20.741 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:20 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1643544170' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:20.741 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:20 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:20.756 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:20.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.781 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:20.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:20.882 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:20.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.903 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:20.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:20.959 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:20 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1643544170' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:20.959 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:20 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:20.959 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:20 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1643544170' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:20.959 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:20 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:21.005 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:21.026 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:21.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:21.128 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:21.150 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:21.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:21.252 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:21.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.275 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:21.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:21.376 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:21.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.398 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:21.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:21.499 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:21.524 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:21.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:21.626 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:21.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.648 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:21.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:21.749 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:21.772 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:21.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:21.788 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:21 smithi096 ceph-mon[98901]: pgmap v816: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:21.789 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:21 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:53:21.789 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:21 smithi096 ceph-mon[98901]: osdmap e274: 8 total, 8 up, 8 in 2024-09-13T14:53:21.789 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:21.789 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:21.789 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:21 smithi096 ceph-mon[102928]: pgmap v816: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:21.789 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:21 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:53:21.789 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:21 smithi096 ceph-mon[102928]: osdmap e274: 8 total, 8 up, 8 in 2024-09-13T14:53:21.789 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:21.789 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:21.873 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:21.895 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:21.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:21.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:21 smithi204 ceph-mon[87071]: pgmap v816: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:21.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:21 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:53:21.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:21 smithi204 ceph-mon[87071]: osdmap e274: 8 total, 8 up, 8 in 2024-09-13T14:53:21.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:21.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:21.997 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:22.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.019 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:22.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:22.120 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:22.141 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:22.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:22.243 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:22.265 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:22.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:22.367 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:22.389 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:22.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:22.491 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:22.513 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:22.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:22.616 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:22.637 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:22.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:22.739 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:22.760 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:22.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:22.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:22 smithi096 ceph-mon[98901]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:53:22.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:22 smithi096 ceph-mon[102928]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:53:22.862 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:22.883 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:22.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:22.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:22 smithi204 ceph-mon[87071]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:53:22.985 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:23.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.009 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:23.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:23.111 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:23.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.134 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:23.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:23.237 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:23.260 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:23.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:23.362 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:23.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.385 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:23.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:23.437 DEBUG:teuthology.orchestra.run.smithi096:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-09-13T14:53:23.440 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:53:23.440 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:53:23.487 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:23.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.509 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:23.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:23.604 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:hints = 1 2024-09-13T14:53:23.604 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:53:23.604 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Object prefix: benchmark_data_smithi096_227542 2024-09-13T14:53:23.604 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:53:23.605 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:53:23.610 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:23.648 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:23.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:23.655 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:23 smithi096 ceph-mon[98901]: pgmap v818: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:23.750 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:23.782 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:23.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:23.883 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:23.922 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:23.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:23.926 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:23 smithi096 ceph-mon[102928]: pgmap v818: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:23.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:23 smithi204 ceph-mon[87071]: pgmap v818: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:23.972 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T14:53:23.973 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T14:53:23.973 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T14:53:23.973 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T14:53:23.973 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T14:53:23.973 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T14:53:23.973 INFO:teuthology.orchestra.run.smithi096.stderr:bench [] [] [] [] 2024-09-13T14:53:23.973 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:53:23.973 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T14:53:23.973 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T14:53:23.973 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore allocator dump block 2024-09-13T14:53:23.973 INFO:teuthology.orchestra.run.smithi096.stderr:Error EINVAL: invalid command 2024-09-13T14:53:24.025 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:24.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.058 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:24.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:24.159 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:24.176 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:24.188 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:24.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:24.290 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:24.321 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:24.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:24.423 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:24.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.461 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:24.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:24.562 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:24.596 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:24.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:24.604 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 1 16 3946 3930 245.612 245.625 0.00278791 0.00405754 2024-09-13T14:53:24.698 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:24.740 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:24.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:24.761 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:53:24.762 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:53:24.762 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.2 'trim stale osdmaps' 2024-09-13T14:53:24.842 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:24.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.874 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:24.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:24.975 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:25.032 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:25.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:25.134 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:25.177 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:25.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:25.281 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:25.313 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:25.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:25.415 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:25.446 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:25.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:25.548 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:25.606 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 2 16 7130 7114 222.121 199 0.00748666 0.00449203 2024-09-13T14:53:25.646 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:25.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:25.748 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:25.776 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:25.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:25.878 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:25.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.915 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:25.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:25.917 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:25 smithi096 ceph-mon[98901]: pgmap v819: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:25.917 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:25 smithi096 ceph-mon[102928]: pgmap v819: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:25.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:25 smithi204 ceph-mon[87071]: pgmap v819: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:26.017 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:26.052 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:26.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:26.154 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:26.186 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:26.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:26.279 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:53:26.279 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:53:26.279 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.279 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.45749.0:8077 8.1 e274/270)", 2024-09-13T14:53:26.279 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:53:25.897740+0000", 2024-09-13T14:53:26.280 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 6.2850999999999998e-05, 2024-09-13T14:53:26.280 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3281000000000004e-05, 2024-09-13T14:53:26.280 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:26.280 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:26.280 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "queued for pg", 2024-09-13T14:53:26.280 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:26.280 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.280 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:26.280 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897740+0000", 2024-09-13T14:53:26.280 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:26.280 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:26.280 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.281 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:26.281 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897740+0000", 2024-09-13T14:53:26.281 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:26.281 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:26.281 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.281 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:26.281 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897741+0000", 2024-09-13T14:53:26.281 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8599999999999997e-07 2024-09-13T14:53:26.281 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:26.281 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.281 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:26.281 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897763+0000", 2024-09-13T14:53:26.281 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.2269999999999999e-05 2024-09-13T14:53:26.282 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:26.282 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.282 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:26.282 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897777+0000", 2024-09-13T14:53:26.282 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4544e-05 2024-09-13T14:53:26.282 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:26.282 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.282 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:26.282 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897785+0000", 2024-09-13T14:53:26.282 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1019999999999999e-06 2024-09-13T14:53:26.282 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:26.282 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.282 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:26.282 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897838+0000", 2024-09-13T14:53:26.283 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3560000000000002e-05 2024-09-13T14:53:26.283 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:26.283 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:26.283 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:26.283 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:26.283 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.283 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.45749.0:8075 8.d e274/270 8:b0d2a760:::benchmark_data_smithi096_227542_object8074:head v 274'494, mlcod=274'491)", 2024-09-13T14:53:26.283 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:53:25.897005+0000", 2024-09-13T14:53:26.283 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.00079745200000000003, 2024-09-13T14:53:26.283 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00088980000000000005, 2024-09-13T14:53:26.283 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:26.283 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:26.283 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:26.283 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:26.283 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.284 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:26.284 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897005+0000", 2024-09-13T14:53:26.284 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:26.284 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:26.284 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.284 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:26.284 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897005+0000", 2024-09-13T14:53:26.284 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:26.284 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:26.284 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.284 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:26.284 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897007+0000", 2024-09-13T14:53:26.285 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.409e-06 2024-09-13T14:53:26.285 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:26.285 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.285 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:26.285 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897043+0000", 2024-09-13T14:53:26.285 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6773000000000002e-05 2024-09-13T14:53:26.285 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:26.285 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.285 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:26.285 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897056+0000", 2024-09-13T14:53:26.285 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2984e-05 2024-09-13T14:53:26.285 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:26.286 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.286 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:26.286 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897063+0000", 2024-09-13T14:53:26.286 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1219999999999997e-06 2024-09-13T14:53:26.286 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:26.286 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.286 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:26.286 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897124+0000", 2024-09-13T14:53:26.286 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0779000000000002e-05 2024-09-13T14:53:26.286 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:26.286 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.286 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:26.286 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897143+0000", 2024-09-13T14:53:26.287 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8349999999999999e-05 2024-09-13T14:53:26.287 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:26.287 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:26.287 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:26.287 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:53:25.897877+0000", 2024-09-13T14:53:26.287 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00073482799999999998 2024-09-13T14:53:26.287 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:26.287 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:26.287 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:26.287 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:26.287 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2024-09-13T14:53:26.287 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 2 2024-09-13T14:53:26.287 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:53:26.289 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:26.320 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:26.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:26.422 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:26.456 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:26.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:26.558 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:26.597 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:26.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:26.606 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 3 16 10394 10378 216.078 204 0.0031248 0.00462176 2024-09-13T14:53:26.699 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:26.735 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:26.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:26.837 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:26.873 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:26.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:26.975 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:27.010 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:27.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:27.112 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:27.150 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:27.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:27.252 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:27.292 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:27.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:27.320 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:53:27.320 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:53:27.354 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:53:27.354 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:53:27.355 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:27 smithi096 ceph-mon[98901]: pgmap v820: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:27.355 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:27 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:27.355 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:27 smithi096 ceph-mon[102928]: pgmap v820: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:27.355 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:27 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:27.389 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:53:27.389 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:53:27.394 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:27.425 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:27.426 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:53:27.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.426 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:53:27.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:27.458 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:53:27.459 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:53:27.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:27 smithi204 ceph-mon[87071]: pgmap v820: 99 pgs: 99 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:53:27.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:27 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:27.493 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:53:27.494 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:53:27.529 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:27.530 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:53:27.531 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:53:27.556 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:53:27.557 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:53:27.567 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:27.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:27.606 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 4 16 13483 13467 210.322 193.062 0.00436446 0.00474807 2024-09-13T14:53:27.670 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:27.703 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:27.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:27.805 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:27.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.842 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:27.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:27.943 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:27.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.991 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:27.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:28.092 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:28.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.131 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:28.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:28.232 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:28.266 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:28.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:28.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:28 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1796113322' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:28.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:53:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:53:27] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-09-13T14:53:28.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:28 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1796113322' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:28.369 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:28.404 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:28.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:28.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:28 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1796113322' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:28.506 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:28.538 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:28.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:28.606 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 5 16 16536 16520 206.418 190.812 0.00619576 0.00483869 2024-09-13T14:53:28.640 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:28.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.675 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:28.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:28.769 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:53:28.776 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:28.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.807 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:28.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:28.908 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:28.935 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:28.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:29.037 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:29.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.069 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:29.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:29.170 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:29.207 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:29.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:29.309 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:29.314 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:29.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.345 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:29.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:29.447 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:29.448 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:29 smithi096 ceph-mon[98901]: pgmap v821: 99 pgs: 99 active+clean; 265 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 26 MiB/s wr, 415 op/s 2024-09-13T14:53:29.448 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:29 smithi096 ceph-mon[102928]: pgmap v821: 99 pgs: 99 active+clean; 265 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 26 MiB/s wr, 415 op/s 2024-09-13T14:53:29.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:29 smithi204 ceph-mon[87071]: pgmap v821: 99 pgs: 99 active+clean; 265 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 26 MiB/s wr, 415 op/s 2024-09-13T14:53:29.478 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:29.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:29.580 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:29.606 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 6 16 19023 19007 197.921 155.438 0.0065574 0.00504586 2024-09-13T14:53:29.609 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:29.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:29.711 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:29.751 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:29.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:29.853 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:29.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.886 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:29.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:29.987 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:30.019 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:30.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:30.113 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:53:30.120 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:30.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.160 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:30.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:30.261 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:30.295 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:30.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:30.316 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:30 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1320481456' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:30.316 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:30 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1320481456' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:30.397 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:30.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.432 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:30.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:30.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:30 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1320481456' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:53:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:53:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:30.533 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:30.535 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:30.564 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:30.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:30.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 7 16 21635 21619 192.955 163.25 0.00607766 0.00517723 2024-09-13T14:53:30.652 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:53:30.666 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:30.740 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:30.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:30.842 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:30.881 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:30.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:30.983 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:31.024 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:31.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:31.126 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:31.173 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:31.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:31.275 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:31.307 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:31.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:31.389 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:31 smithi096 ceph-mon[98901]: pgmap v822: 99 pgs: 99 active+clean; 883 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-13T14:53:31.389 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:31 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:53:31.389 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:31 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:53:31.389 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:31 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1320481456' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:31.389 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:31 smithi096 ceph-mon[98901]: osdmap e275: 8 total, 8 up, 8 in 2024-09-13T14:53:31.390 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:31 smithi096 ceph-mon[102928]: pgmap v822: 99 pgs: 99 active+clean; 883 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-13T14:53:31.390 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:31 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:53:31.390 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:31 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:53:31.390 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:31 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1320481456' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:31.390 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:31 smithi096 ceph-mon[102928]: osdmap e275: 8 total, 8 up, 8 in 2024-09-13T14:53:31.408 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:31.446 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:31.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:31.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:31 smithi204 ceph-mon[87071]: pgmap v822: 99 pgs: 99 active+clean; 883 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-13T14:53:31.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:31 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:53:31.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:31 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:53:31.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:31 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1320481456' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:31.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:31 smithi204 ceph-mon[87071]: osdmap e275: 8 total, 8 up, 8 in 2024-09-13T14:53:31.549 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:31.588 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:31.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:31.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 8 16 24145 24129 188.445 156.875 0.00366171 0.00530088 2024-09-13T14:53:31.691 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:31.727 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:31.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:31.829 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:31.868 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:31.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:31.970 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:32.003 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:32.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:32.105 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:32.145 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:32.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:32.247 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:32.278 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:32.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:32.380 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:32.414 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:32.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:32.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:32.541 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:32.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:32.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 9 16 26681 26665 185.117 158.5 0.0090705 0.00539693 2024-09-13T14:53:32.643 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:32.674 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:32.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:32.776 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:32.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.815 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:32.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:32.916 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:32.950 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:32.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:33.052 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:33.087 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:33.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:33.190 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:33.221 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:33.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:33.260 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:53:33.323 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:33.356 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:33.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:33.420 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:33 smithi096 ceph-mon[98901]: pgmap v824: 99 pgs: 99 active+clean; 1.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:53:33.421 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:33 smithi096 ceph-mon[102928]: pgmap v824: 99 pgs: 99 active+clean; 1.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:53:33.458 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:33.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:33 smithi204 ceph-mon[87071]: pgmap v824: 99 pgs: 99 active+clean; 1.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:53:33.485 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:33.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:33.514 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:33.587 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:33.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 10 16 28808 28792 179.899 132.938 0.00328089 0.00554218 2024-09-13T14:53:33.608 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:33.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:33.710 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:33.746 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:33.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:33.849 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:33.875 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:33.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:33.977 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:34.008 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:34.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:34.110 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:34.143 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:34.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:34.245 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:34.275 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:34.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:34.377 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:34.415 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:34.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:34.517 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:34.552 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:34.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:34.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 11 15 30862 30847 175.22 128.438 0.0029814 0.00570257 2024-09-13T14:53:34.654 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:34.690 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:34.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:34.792 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:34.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.814 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:34.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:34.915 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:34.955 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:34.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:35.057 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:35.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.092 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:35.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:35.193 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:35.225 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:35.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:35.327 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:35.339 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:35 smithi096 ceph-mon[98901]: pgmap v825: 99 pgs: 99 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 169 MiB/s wr, 2.70k op/s 2024-09-13T14:53:35.339 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:35 smithi096 ceph-mon[102928]: pgmap v825: 99 pgs: 99 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 169 MiB/s wr, 2.70k op/s 2024-09-13T14:53:35.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.363 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:35.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:35.464 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:35.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:35 smithi204 ceph-mon[87071]: pgmap v825: 99 pgs: 99 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 169 MiB/s wr, 2.70k op/s 2024-09-13T14:53:35.498 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:35.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:35.600 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:35.608 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 12 16 32733 32717 170.342 116.875 0.00531399 0.0058653 2024-09-13T14:53:35.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.639 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:35.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:35.741 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:35.772 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:35.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:35.874 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:35.911 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:35.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:36.013 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:36.039 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:36.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:36.141 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:36.176 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:36.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:36.279 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:36.307 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:36.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:36.409 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:36.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.440 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:36.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:36.541 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:36.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.566 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:36.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:36.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 13 16 34538 34522 165.917 112.812 0.00517784 0.00602179 2024-09-13T14:53:36.668 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:36.702 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:36.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:36.804 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:36.839 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:36.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:36.941 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:36.965 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:36.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:37.067 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:37.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:37.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:37.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:36 smithi096 ceph-mon[98901]: pgmap v826: 99 pgs: 99 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 169 MiB/s wr, 2.70k op/s 2024-09-13T14:53:37.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:37.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:37.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:36 smithi096 ceph-mon[102928]: pgmap v826: 99 pgs: 99 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 169 MiB/s wr, 2.70k op/s 2024-09-13T14:53:37.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.107 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:37.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:37.208 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:37.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:53:37.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:36 smithi204 ceph-mon[87071]: pgmap v826: 99 pgs: 99 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 169 MiB/s wr, 2.70k op/s 2024-09-13T14:53:37.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.243 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:37.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:37.344 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:37.376 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:37.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:37.478 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:37.513 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:37.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:37.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 14 16 36262 36246 161.762 107.75 0.0436637 0.00617224 2024-09-13T14:53:37.615 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:37.648 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:37.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:37.752 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:37.780 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:37.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:37.882 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:37.912 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:37.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:38.014 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:38.045 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:38.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:38.146 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:38.174 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:38.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:38.276 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:38.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.298 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:38.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:38.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:53:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:53:37] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.51.0" 2024-09-13T14:53:38.399 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:38.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.434 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:38.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:38.535 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:38.557 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:38.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:38.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 15 16 37614 37598 156.612 84.5 0.00866386 0.0063803 2024-09-13T14:53:38.658 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:38.699 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:38.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:38.801 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:38.826 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:38.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:38.928 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:38.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.966 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:38.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:39.067 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:39.107 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:39.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:39.209 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:39.248 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:39.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:39.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:39 smithi096 ceph-mon[98901]: pgmap v827: 99 pgs: 99 active+clean; 1.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 166 MiB/s wr, 2.66k op/s 2024-09-13T14:53:39.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:39 smithi096 ceph-mon[102928]: pgmap v827: 99 pgs: 99 active+clean; 1.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 166 MiB/s wr, 2.66k op/s 2024-09-13T14:53:39.350 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:39.383 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:39.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:39.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:39 smithi204 ceph-mon[87071]: pgmap v827: 99 pgs: 99 active+clean; 1.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 166 MiB/s wr, 2.66k op/s 2024-09-13T14:53:39.485 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:39.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.519 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:39.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:39.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 16 16 39531 39515 154.312 119.812 0.0427644 0.00646198 2024-09-13T14:53:39.621 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:39.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.649 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:39.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:39.750 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:39.782 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:39.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:39.883 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:39.917 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:39.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:40.019 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:40.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.054 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:40.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:40.155 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:40.189 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:40.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:40.291 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:40.324 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:40.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:40.426 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:40.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.463 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:40.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:53:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:53:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:40.564 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:40.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.601 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:40.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:40.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 17 16 41572 41556 152.738 127.562 0.00438175 0.00654205 2024-09-13T14:53:40.702 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:40.731 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:40.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:40.833 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:40.863 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:40.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:40.965 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:40.995 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:40.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:41.096 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:41.125 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:41.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:41.165 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:41 smithi096 ceph-mon[98901]: pgmap v828: 99 pgs: 99 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-09-13T14:53:41.165 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:41 smithi096 ceph-mon[102928]: pgmap v828: 99 pgs: 99 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-09-13T14:53:41.227 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:41.263 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:41.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:41.364 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:41.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.411 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:41.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:41.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:41 smithi204 ceph-mon[87071]: pgmap v828: 99 pgs: 99 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-09-13T14:53:41.512 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:41.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.544 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:41.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:41.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 18 16 43208 43192 149.933 102.25 0.00244666 0.00663679 2024-09-13T14:53:41.645 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:41.667 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:41.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:41.769 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:41.799 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:41.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:41.901 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:41.941 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:41.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:42.043 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:42.074 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:42.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:42.176 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:42.198 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:42.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:42.300 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:42.321 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:42.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:42.423 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:42.458 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:42.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:42.560 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:42.590 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:42.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:42.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 19 16 44423 44407 146.039 75.9375 0.00825144 0.00684197 2024-09-13T14:53:42.693 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:42.730 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:42.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:42.831 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:42.865 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:42.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:42.967 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:42.997 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:42.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:43.098 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:43.131 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:43.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:43.233 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:43.267 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:43.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:43.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:43 smithi096 ceph-mon[98901]: pgmap v829: 99 pgs: 99 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-09-13T14:53:43.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:43 smithi096 ceph-mon[102928]: pgmap v829: 99 pgs: 99 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-09-13T14:53:43.369 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:43.400 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:43.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:43.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:43 smithi204 ceph-mon[87071]: pgmap v829: 99 pgs: 99 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-09-13T14:53:43.502 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:43.539 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:43.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:43.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:53:43.609107+0000 min lat: 0.00118538 max lat: 0.273309 avg lat: 0.00691574 2024-09-13T14:53:43.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:53:43.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 20 16 46263 46247 144.486 115 0.00393887 0.00691574 2024-09-13T14:53:43.641 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:43.688 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:43.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:43.790 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:43.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.841 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:43.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:43.942 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:43.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.984 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:43.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:44.044 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T14:53:44.045 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T14:53:44.045 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T14:53:44.045 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T14:53:44.045 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T14:53:44.045 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T14:53:44.045 INFO:teuthology.orchestra.run.smithi096.stderr:bench [] [] [] [] 2024-09-13T14:53:44.045 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:53:44.045 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T14:53:44.045 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T14:53:44.046 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore allocator dump block 2024-09-13T14:53:44.046 INFO:teuthology.orchestra.run.smithi096.stderr:Error EINVAL: invalid command 2024-09-13T14:53:44.086 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:44.113 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:44.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:44.217 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:44.256 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:44.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:44.358 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:44.393 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:44.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:44.495 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:44.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.525 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:44.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:44.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 21 16 47975 47959 142.701 107 0.0109096 0.00700271 2024-09-13T14:53:44.626 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:44.665 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:44.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:44.693 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:53:44.694 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:53:44.767 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:44.803 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:44.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:44.905 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:44.940 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:44.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:45.042 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:45.075 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:45.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:45.164 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:53:45.165 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:53:45.165 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.3 'trim stale osdmaps' 2024-09-13T14:53:45.177 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:45.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.253 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:45.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:45.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:45 smithi096 ceph-mon[98901]: pgmap v830: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 147 MiB/s wr, 2.36k op/s 2024-09-13T14:53:45.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:45 smithi096 ceph-mon[102928]: pgmap v830: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 147 MiB/s wr, 2.36k op/s 2024-09-13T14:53:45.355 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:45.396 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:45.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:45 smithi204 ceph-mon[87071]: pgmap v830: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 147 MiB/s wr, 2.36k op/s 2024-09-13T14:53:45.498 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:45.530 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:45.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:45.610 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 22 16 49717 49701 141.163 108.875 0.00578394 0.00707699 2024-09-13T14:53:45.614 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:53:45.632 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:45.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.699 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:45.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:45.800 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:45.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.830 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:45.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:45.931 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:45.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.963 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:45.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:46.064 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:46.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.096 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:46.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:46.144 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:46.197 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:46.219 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:46.250 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:53:46.251 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:53:46.270 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:46.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:46.286 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:53:46.287 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:53:46.321 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:53:46.321 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:53:46.359 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:53:46.359 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:53:46.372 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:46.394 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:53:46.394 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:53:46.412 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:46.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:46.418 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:53:46.418 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:53:46.434 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:53:46.434 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:53:46.449 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:53:46.450 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:53:46.514 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:46.536 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:46.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:46.610 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 23 16 50931 50915 138.325 75.875 0.0132318 0.00722469 2024-09-13T14:53:46.637 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:46.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.673 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:46.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:46.774 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:46.805 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:46.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:46.907 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:46.944 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:46.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:47.046 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:47.071 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:47.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:47.173 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:47.200 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:47.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:47.302 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:47.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:47 smithi096 ceph-mon[98901]: pgmap v831: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T14:53:47.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:47 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1179045228' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:47.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:47 smithi096 ceph-mon[102928]: pgmap v831: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T14:53:47.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:47 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1179045228' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:47.330 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:47.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:47.432 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:47.468 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:47.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:47 smithi204 ceph-mon[87071]: pgmap v831: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T14:53:47.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:47 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1179045228' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:47.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:47.571 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:47.609 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:47.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 24 16 52600 52584 136.907 104.312 0.00976929 0.00729891 2024-09-13T14:53:47.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:47.711 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:47.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:47.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:47.854 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:47.887 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:47.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:47.989 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:53:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:53:47] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.51.0" 2024-09-13T14:53:47.989 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:48.020 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:48.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:48.122 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:48.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.154 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:48.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:48.255 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:48.291 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:48.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:48.393 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:48.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.427 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:48.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:48.528 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:48.564 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:48.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:48.610 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 25 16 54317 54301 135.723 107.312 0.00629303 0.00736359 2024-09-13T14:53:48.665 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:48.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.707 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:48.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:48.808 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:48.845 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:48.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:48.947 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:48.995 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:48.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:49.097 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:49.137 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:49.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:49.239 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:49.278 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:49.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:49.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:49 smithi096 ceph-mon[98901]: pgmap v832: 99 pgs: 99 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-13T14:53:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:49 smithi096 ceph-mon[102928]: pgmap v832: 99 pgs: 99 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-13T14:53:49.380 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:49.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.425 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:49.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:49 smithi204 ceph-mon[87071]: pgmap v832: 99 pgs: 99 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-13T14:53:49.526 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:49.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.560 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:49.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:49.611 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 26 16 56000 55984 134.546 105.188 0.0040362 0.00742757 2024-09-13T14:53:49.663 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:49.699 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:49.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:49.801 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:49.839 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:49.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:49.941 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:49.976 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:49.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:50.078 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:50.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.111 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:50.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:50.212 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:50.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.250 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:50.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:50.351 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:50.379 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:50.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:50.442 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:53:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:53:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:50.482 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:50.523 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:50.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:50.611 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 27 16 57649 57633 133.377 103.062 0.0104266 0.0074927 2024-09-13T14:53:50.625 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:50.663 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:50.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:50.765 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:50.792 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:50.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:50.894 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:50.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.925 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:50.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:51.015 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:53:51.026 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:51.053 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:51.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:51.155 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:51.183 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:51.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:51.187 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:51 smithi096 ceph-mon[98901]: pgmap v833: 99 pgs: 99 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-13T14:53:51.187 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:51.187 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:51 smithi096 ceph-mon[102928]: pgmap v833: 99 pgs: 99 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-13T14:53:51.187 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:51.285 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:51.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.331 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:51.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:51.432 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:51 smithi204 ceph-mon[87071]: pgmap v833: 99 pgs: 99 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-13T14:53:51.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:51.470 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:51.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:51.572 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:51.613 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 28 16 59107 59091 131.865 91.125 0.0156531 0.00757847 2024-09-13T14:53:51.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.627 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:51.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:51.674 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T14:53:51.674 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T14:53:51.675 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T14:53:51.675 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T14:53:51.675 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T14:53:51.675 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T14:53:51.675 INFO:teuthology.orchestra.run.smithi096.stderr:bench [] [] [] [] 2024-09-13T14:53:51.675 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:53:51.675 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T14:53:51.675 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T14:53:51.675 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore allocator dump block 2024-09-13T14:53:51.675 INFO:teuthology.orchestra.run.smithi096.stderr:Error EINVAL: invalid command 2024-09-13T14:53:51.729 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:51.771 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:51.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:51.873 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:51.907 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:51.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:52.009 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:52.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.044 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:52.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:52.145 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:52.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.170 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:52.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:52.271 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:52.302 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:52.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:52.404 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:52.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.425 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:52.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:52.526 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:52.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.565 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:52.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:52.612 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 29 16 60524 60508 130.372 88.5625 0.00859752 0.00766512 2024-09-13T14:53:52.667 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:52.704 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:52.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:52.810 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:52.839 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:52.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:52.941 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:52.964 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:52.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:53.066 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:53.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.103 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:53.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:53.204 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:53.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.230 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:53.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:53.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:53 smithi096 ceph-mon[98901]: pgmap v834: 99 pgs: 99 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-13T14:53:53.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:53 smithi096 ceph-mon[102928]: pgmap v834: 99 pgs: 99 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-13T14:53:53.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:53.367 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:53.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:53.468 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:53 smithi204 ceph-mon[87071]: pgmap v834: 99 pgs: 99 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-13T14:53:53.500 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:53.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:53.602 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:53.612 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 30 16 62020 62004 129.143 93.5 0.012834 0.00773873 2024-09-13T14:53:53.634 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:53.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:53.736 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:53.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.767 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:53.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:53.868 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:53.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.904 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:53.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:54.005 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:54.050 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:54.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:54.152 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:54.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.191 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:54.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:54.293 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:54.325 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:54.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:54.427 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:54.459 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:54.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:54.561 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:54.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.586 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:54.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:54.612 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 31 16 63307 63291 127.571 80.4375 0.00318626 0.007824 2024-09-13T14:53:54.688 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:54.723 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:54.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:54.825 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:54.854 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:54.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:54.956 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:54.994 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:54.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:55.034 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:53:55.096 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:55.126 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:55.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:55.161 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:53:55.161 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:53:55.161 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.4 'trim stale osdmaps' 2024-09-13T14:53:55.228 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:55.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.289 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:55.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:55.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:55 smithi096 ceph-mon[98901]: pgmap v835: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-13T14:53:55.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:55 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4149395192' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:55 smithi096 ceph-mon[102928]: pgmap v835: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-13T14:53:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:55 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4149395192' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:55.391 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:55.447 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:55.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:55.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:55 smithi204 ceph-mon[87071]: pgmap v835: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-13T14:53:55.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:55 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4149395192' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:55.549 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:55.592 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:55.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:55.612 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 32 16 64709 64693 126.323 87.625 0.0191231 0.00791207 2024-09-13T14:53:55.695 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:55.737 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:55.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:55.839 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:55.870 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:55.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:55.972 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:56.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.015 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:56.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:56.117 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:56.148 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:56.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:56.250 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:56.282 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:56.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:56.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:56 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:56.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:56 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4149395192' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:53:56.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:56 smithi096 ceph-mon[98901]: osdmap e276: 8 total, 8 up, 8 in 2024-09-13T14:53:56.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:56 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:56.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:56 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4149395192' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:53:56.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:56 smithi096 ceph-mon[102928]: osdmap e276: 8 total, 8 up, 8 in 2024-09-13T14:53:56.384 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:56.422 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:56.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:56 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:56 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4149395192' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:53:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:56 smithi204 ceph-mon[87071]: osdmap e276: 8 total, 8 up, 8 in 2024-09-13T14:53:56.524 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:56.557 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:56.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:56.613 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 33 16 66218 66202 125.351 94.3125 0.0048962 0.00797211 2024-09-13T14:53:56.658 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:56.709 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:56.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:56.814 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:56.849 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:56.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:56.951 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:56.985 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:56.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:57.087 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:57.129 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:57.134 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:57.154 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:57.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:57.256 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:57.278 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:57 smithi096 ceph-mon[98901]: pgmap v837: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-09-13T14:53:57.278 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:57 smithi096 ceph-mon[102928]: pgmap v837: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-09-13T14:53:57.302 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:57.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:57.405 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:57.456 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:57.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:57.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:57 smithi204 ceph-mon[87071]: pgmap v837: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-09-13T14:53:57.558 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:57.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.603 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:57.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:57.613 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 34 15 67591 67576 124.19 85.875 0.0111308 0.00804778 2024-09-13T14:53:57.704 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:57.746 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:57.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:57.848 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:57.888 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:57.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:57.984 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:53:57.984 INFO:teuthology.orchestra.run.smithi096.stdout: "size": 20, 2024-09-13T14:53:57.984 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 600, 2024-09-13T14:53:57.985 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:53:57.985 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.985 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181921 6.14 6:291668c5:::benchmark_data_smithi096_167087_object64693:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:53:57.985 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:16.682370+0000", 2024-09-13T14:53:57.985 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 401.29779899800002, 2024-09-13T14:53:57.985 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.92264975199999999, 2024-09-13T14:53:57.985 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:57.985 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:57.985 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:57.985 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:57.985 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:57.986 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:57.986 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181921 2024-09-13T14:53:57.986 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.986 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:57.986 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.986 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:57.986 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682370+0000", 2024-09-13T14:53:57.986 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:57.986 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.986 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.986 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:57.987 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682370+0000", 2024-09-13T14:53:57.987 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:57.987 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.987 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.987 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:57.987 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682371+0000", 2024-09-13T14:53:57.987 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4750000000000001e-06 2024-09-13T14:53:57.987 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.987 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.987 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:57.987 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682377+0000", 2024-09-13T14:53:57.987 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7749999999999998e-06 2024-09-13T14:53:57.987 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.987 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.988 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:57.988 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682393+0000", 2024-09-13T14:53:57.988 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5710999999999999e-05 2024-09-13T14:53:57.988 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.988 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.988 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:57.988 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682401+0000", 2024-09-13T14:53:57.988 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9140000000000008e-06 2024-09-13T14:53:57.988 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.988 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.988 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:57.988 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682470+0000", 2024-09-13T14:53:57.989 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9227000000000007e-05 2024-09-13T14:53:57.989 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.989 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.989 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:57.989 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682479+0000", 2024-09-13T14:53:57.989 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9220000000000003e-06 2024-09-13T14:53:57.989 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.989 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.989 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:57.989 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.601256+0000", 2024-09-13T14:53:57.989 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.91877762500000004 2024-09-13T14:53:57.990 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.990 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.990 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:57.990 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.601372+0000", 2024-09-13T14:53:57.990 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011595 2024-09-13T14:53:57.990 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.990 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.990 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:53:57.990 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.601494+0000", 2024-09-13T14:53:57.990 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012179400000000001 2024-09-13T14:53:57.990 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.990 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.991 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:57.991 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.604870+0000", 2024-09-13T14:53:57.991 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0033754520000000001 2024-09-13T14:53:57.991 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.991 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.991 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:57.991 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.604936+0000", 2024-09-13T14:53:57.991 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5977999999999997e-05 2024-09-13T14:53:57.991 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.992 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.992 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:57.992 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.604946+0000", 2024-09-13T14:53:57.992 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0729e-05 2024-09-13T14:53:57.992 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.992 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.992 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:57.992 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.604983+0000", 2024-09-13T14:53:57.992 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6902e-05 2024-09-13T14:53:57.992 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.992 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.992 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:57.993 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.605020+0000", 2024-09-13T14:53:57.993 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6297999999999997e-05 2024-09-13T14:53:57.993 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:57.993 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:57.993 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:57.993 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.993 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.993 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181940 6.14 6:2de03b2d:::benchmark_data_smithi096_167087_object64712:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:53:57.993 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:16.696647+0000", 2024-09-13T14:53:57.994 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 401.28352220199997, 2024-09-13T14:53:57.994 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.91128050299999996, 2024-09-13T14:53:57.994 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:57.994 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:57.994 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:57.994 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:57.994 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:57.994 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:57.994 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181940 2024-09-13T14:53:57.994 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.994 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:57.994 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.994 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:57.994 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696647+0000", 2024-09-13T14:53:57.995 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:57.995 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.995 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.995 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:57.995 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696647+0000", 2024-09-13T14:53:57.995 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:57.995 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.995 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.995 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:57.995 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696648+0000", 2024-09-13T14:53:57.995 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.579e-06 2024-09-13T14:53:57.995 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.995 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.996 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:57.996 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696654+0000", 2024-09-13T14:53:57.996 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1349999999999997e-06 2024-09-13T14:53:57.996 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.996 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.996 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:57.996 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696667+0000", 2024-09-13T14:53:57.996 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2727e-05 2024-09-13T14:53:57.996 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.996 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.996 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:57.996 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696675+0000", 2024-09-13T14:53:57.996 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1899999999999995e-06 2024-09-13T14:53:57.996 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.997 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.997 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:57.997 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696911+0000", 2024-09-13T14:53:57.997 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00023602599999999999 2024-09-13T14:53:57.997 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.997 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.997 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:57.997 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696920+0000", 2024-09-13T14:53:57.997 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8300000000000002e-06 2024-09-13T14:53:57.997 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.997 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.997 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:57.997 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.603441+0000", 2024-09-13T14:53:57.997 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.90652066200000003 2024-09-13T14:53:57.998 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.998 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.998 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:57.998 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.603565+0000", 2024-09-13T14:53:57.998 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012396799999999999 2024-09-13T14:53:57.998 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.998 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.998 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:53:57.998 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.603668+0000", 2024-09-13T14:53:57.998 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000102938 2024-09-13T14:53:57.998 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.998 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.998 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:57.999 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.606432+0000", 2024-09-13T14:53:57.999 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027647000000000001 2024-09-13T14:53:57.999 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.999 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.999 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:57.999 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.606482+0000", 2024-09-13T14:53:57.999 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9243999999999998e-05 2024-09-13T14:53:57.999 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:57.999 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:57.999 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:57.999 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.607831+0000", 2024-09-13T14:53:57.999 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0013495219999999999 2024-09-13T14:53:57.999 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.000 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.000 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.000 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.607881+0000", 2024-09-13T14:53:58.000 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9648999999999998e-05 2024-09-13T14:53:58.000 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.000 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.000 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:58.002 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.002 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.607927+0000", 2024-09-13T14:53:58.002 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6332999999999998e-05 2024-09-13T14:53:58.002 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.002 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.003 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.003 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.003 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.003 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182060 6.14 6:2db5f18f:::benchmark_data_smithi096_167087_object64832:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:53:58.003 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:16.790348+0000", 2024-09-13T14:53:58.003 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 401.18982131400003, 2024-09-13T14:53:58.003 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.81781258199999995, 2024-09-13T14:53:58.003 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.003 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.004 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.004 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.004 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.004 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.004 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182060 2024-09-13T14:53:58.004 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.004 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.004 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.004 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.004 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790348+0000", 2024-09-13T14:53:58.004 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.004 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.004 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.005 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.005 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790348+0000", 2024-09-13T14:53:58.005 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.005 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.005 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.005 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.005 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790349+0000", 2024-09-13T14:53:58.006 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6309999999999999e-06 2024-09-13T14:53:58.006 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.006 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.006 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.006 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790360+0000", 2024-09-13T14:53:58.006 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1270999999999999e-05 2024-09-13T14:53:58.006 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.007 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.007 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.007 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790374+0000", 2024-09-13T14:53:58.007 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3995000000000001e-05 2024-09-13T14:53:58.007 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.007 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.007 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.007 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790383+0000", 2024-09-13T14:53:58.007 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.7520000000000002e-06 2024-09-13T14:53:58.007 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.008 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.008 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.008 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790459+0000", 2024-09-13T14:53:58.008 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.5407000000000005e-05 2024-09-13T14:53:58.008 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.008 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.008 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.008 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790467+0000", 2024-09-13T14:53:58.008 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.3089999999999996e-06 2024-09-13T14:53:58.008 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.008 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.008 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.008 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.605048+0000", 2024-09-13T14:53:58.009 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.81458113899999995 2024-09-13T14:53:58.009 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.009 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.009 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.009 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.605154+0000", 2024-09-13T14:53:58.009 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010594600000000001 2024-09-13T14:53:58.009 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.009 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.009 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:53:58.009 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.605265+0000", 2024-09-13T14:53:58.009 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011089700000000001 2024-09-13T14:53:58.009 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.010 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.010 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.010 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608038+0000", 2024-09-13T14:53:58.010 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027728029999999999 2024-09-13T14:53:58.010 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.010 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.010 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.010 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608082+0000", 2024-09-13T14:53:58.010 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4183e-05 2024-09-13T14:53:58.010 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.010 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.010 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.010 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608091+0000", 2024-09-13T14:53:58.011 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9770000000000006e-06 2024-09-13T14:53:58.011 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.011 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.011 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.011 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608118+0000", 2024-09-13T14:53:58.011 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7341000000000001e-05 2024-09-13T14:53:58.011 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.011 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.011 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.011 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608160+0000", 2024-09-13T14:53:58.011 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1931000000000002e-05 2024-09-13T14:53:58.011 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.011 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.011 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.012 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.012 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.012 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182076 6.14 6:2fdfb44e:::benchmark_data_smithi096_167087_object64848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:53:58.012 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:16.803957+0000", 2024-09-13T14:53:58.012 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 401.176212265, 2024-09-13T14:53:58.012 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.80631315599999998, 2024-09-13T14:53:58.012 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.012 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.012 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.012 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.012 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.012 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.012 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182076 2024-09-13T14:53:58.013 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.013 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.013 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.013 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.013 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803957+0000", 2024-09-13T14:53:58.013 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.013 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.013 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.013 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.016 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803957+0000", 2024-09-13T14:53:58.017 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.017 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.017 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.017 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.017 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803958+0000", 2024-09-13T14:53:58.017 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8989999999999999e-06 2024-09-13T14:53:58.017 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.017 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.017 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.017 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803964+0000", 2024-09-13T14:53:58.017 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6319999999999996e-06 2024-09-13T14:53:58.017 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.017 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.018 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.018 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803979+0000", 2024-09-13T14:53:58.018 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5045000000000001e-05 2024-09-13T14:53:58.018 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.018 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.018 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.018 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803987+0000", 2024-09-13T14:53:58.018 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3799999999999996e-06 2024-09-13T14:53:58.018 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.018 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.018 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.018 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.804041+0000", 2024-09-13T14:53:58.018 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4199000000000001e-05 2024-09-13T14:53:58.018 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.019 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.019 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.019 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.804048+0000", 2024-09-13T14:53:58.019 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3050000000000004e-06 2024-09-13T14:53:58.019 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.019 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.019 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.019 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.606528+0000", 2024-09-13T14:53:58.019 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.80247967200000003 2024-09-13T14:53:58.019 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.019 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.019 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.019 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.606642+0000", 2024-09-13T14:53:58.019 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011442699999999999 2024-09-13T14:53:58.020 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.020 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.020 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:53:58.020 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.606726+0000", 2024-09-13T14:53:58.020 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.4073000000000006e-05 2024-09-13T14:53:58.020 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.020 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.020 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.020 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608231+0000", 2024-09-13T14:53:58.020 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0015046090000000001 2024-09-13T14:53:58.020 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.020 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.020 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.020 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608248+0000", 2024-09-13T14:53:58.021 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7244e-05 2024-09-13T14:53:58.021 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.021 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.021 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.021 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.609959+0000", 2024-09-13T14:53:58.021 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001710563 2024-09-13T14:53:58.021 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.021 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.021 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.021 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.610178+0000", 2024-09-13T14:53:58.021 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00021929600000000001 2024-09-13T14:53:58.021 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.021 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.022 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.022 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.610270+0000", 2024-09-13T14:53:58.022 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.1811999999999994e-05 2024-09-13T14:53:58.022 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.022 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.022 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.022 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.022 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.022 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182107 6.14 6:2cba648f:::benchmark_data_smithi096_167087_object64879:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:53:58.022 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:16.833170+0000", 2024-09-13T14:53:58.022 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 401.14699840100002, 2024-09-13T14:53:58.022 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.78124621000000005, 2024-09-13T14:53:58.022 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.022 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.023 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.023 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.023 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.023 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.023 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182107 2024-09-13T14:53:58.023 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.023 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.023 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.023 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.023 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833170+0000", 2024-09-13T14:53:58.023 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.023 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.023 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.023 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.024 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833170+0000", 2024-09-13T14:53:58.024 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.024 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.024 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.024 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.024 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833172+0000", 2024-09-13T14:53:58.024 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6449999999999999e-06 2024-09-13T14:53:58.024 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.025 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.025 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.025 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833178+0000", 2024-09-13T14:53:58.025 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1770000000000003e-06 2024-09-13T14:53:58.025 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.025 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.025 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.025 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833192+0000", 2024-09-13T14:53:58.025 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4082e-05 2024-09-13T14:53:58.025 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.025 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.025 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.025 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833200+0000", 2024-09-13T14:53:58.025 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2520000000000002e-06 2024-09-13T14:53:58.026 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.026 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.026 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.026 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833256+0000", 2024-09-13T14:53:58.026 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6427e-05 2024-09-13T14:53:58.026 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.026 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.026 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.026 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833264+0000", 2024-09-13T14:53:58.026 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1629999999999996e-06 2024-09-13T14:53:58.026 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.026 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.026 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.026 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608283+0000", 2024-09-13T14:53:58.027 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.77501878800000001 2024-09-13T14:53:58.027 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.027 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.027 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.027 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608389+0000", 2024-09-13T14:53:58.027 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010569300000000001 2024-09-13T14:53:58.027 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.027 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.027 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:53:58.027 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608505+0000", 2024-09-13T14:53:58.027 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000116434 2024-09-13T14:53:58.027 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.028 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.028 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.028 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612122+0000", 2024-09-13T14:53:58.028 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0036171860000000001 2024-09-13T14:53:58.028 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.028 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.028 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.028 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.614303+0000", 2024-09-13T14:53:58.028 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0021807839999999998 2024-09-13T14:53:58.028 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.028 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.028 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.028 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.614323+0000", 2024-09-13T14:53:58.028 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9644e-05 2024-09-13T14:53:58.029 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.029 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.029 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.029 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.614369+0000", 2024-09-13T14:53:58.029 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6048999999999999e-05 2024-09-13T14:53:58.029 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.029 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.029 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.029 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.614417+0000", 2024-09-13T14:53:58.029 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7886000000000002e-05 2024-09-13T14:53:58.029 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.029 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.030 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.030 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.030 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.030 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181897 6.18 6:1a0fd876:::benchmark_data_smithi096_167087_object64669:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:53:58.030 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583616+0000", 2024-09-13T14:53:58.030 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 400.39655310099999, 2024-09-13T14:53:58.030 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.110115685, 2024-09-13T14:53:58.030 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.030 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.030 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.030 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.030 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.030 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.030 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181897 2024-09-13T14:53:58.030 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.031 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.031 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.031 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.031 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583616+0000", 2024-09-13T14:53:58.031 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.031 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.031 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.031 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.031 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583616+0000", 2024-09-13T14:53:58.031 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.031 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.031 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.031 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.031 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583617+0000", 2024-09-13T14:53:58.032 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.482e-06 2024-09-13T14:53:58.032 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.032 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.032 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.032 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583623+0000", 2024-09-13T14:53:58.032 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1190000000000002e-06 2024-09-13T14:53:58.032 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.032 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.032 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.032 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583637+0000", 2024-09-13T14:53:58.032 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3718e-05 2024-09-13T14:53:58.032 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.032 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.033 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.033 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583646+0000", 2024-09-13T14:53:58.033 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.602e-06 2024-09-13T14:53:58.033 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.033 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.033 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.033 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583943+0000", 2024-09-13T14:53:58.033 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00029740399999999997 2024-09-13T14:53:58.033 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.033 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.033 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.033 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583952+0000", 2024-09-13T14:53:58.033 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.4169999999999998e-06 2024-09-13T14:53:58.033 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.034 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.034 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.034 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.686789+0000", 2024-09-13T14:53:58.034 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1028368799999999 2024-09-13T14:53:58.034 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.034 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.034 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.034 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.686879+0000", 2024-09-13T14:53:58.034 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9676e-05 2024-09-13T14:53:58.034 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.034 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.034 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:53:58.034 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.687143+0000", 2024-09-13T14:53:58.034 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00026446800000000002 2024-09-13T14:53:58.035 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.035 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.035 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.035 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.692990+0000", 2024-09-13T14:53:58.035 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0058462310000000003 2024-09-13T14:53:58.035 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.035 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.035 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.035 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693001+0000", 2024-09-13T14:53:58.035 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1552999999999999e-05 2024-09-13T14:53:58.035 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.035 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.035 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.036 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693028+0000", 2024-09-13T14:53:58.036 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7104999999999999e-05 2024-09-13T14:53:58.036 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.036 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.036 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.036 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693064+0000", 2024-09-13T14:53:58.036 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.5503000000000002e-05 2024-09-13T14:53:58.036 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.036 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.036 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.036 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693489+0000", 2024-09-13T14:53:58.036 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00042502099999999998 2024-09-13T14:53:58.036 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.036 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.037 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.037 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693603+0000", 2024-09-13T14:53:58.037 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011386 2024-09-13T14:53:58.037 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.037 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.037 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.037 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693731+0000", 2024-09-13T14:53:58.037 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000128646 2024-09-13T14:53:58.037 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.037 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.037 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.037 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.037 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.038 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181961 6.18 6:1b4cf7bc:::benchmark_data_smithi096_167087_object64733:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:53:58.038 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583723+0000", 2024-09-13T14:53:58.038 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 400.396445884, 2024-09-13T14:53:58.038 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1144489200000001, 2024-09-13T14:53:58.038 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.038 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.038 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.038 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.038 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.038 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.038 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181961 2024-09-13T14:53:58.038 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.038 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.038 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.039 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.039 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583723+0000", 2024-09-13T14:53:58.039 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.039 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.039 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.039 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.039 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583723+0000", 2024-09-13T14:53:58.039 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.039 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.039 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.039 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.039 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583724+0000", 2024-09-13T14:53:58.039 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9100000000000004e-07 2024-09-13T14:53:58.039 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.040 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.040 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.040 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583727+0000", 2024-09-13T14:53:58.040 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.191e-06 2024-09-13T14:53:58.040 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.040 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.040 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.040 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583736+0000", 2024-09-13T14:53:58.040 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2480000000000002e-06 2024-09-13T14:53:58.040 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.040 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.040 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.040 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583742+0000", 2024-09-13T14:53:58.041 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3840000000000002e-06 2024-09-13T14:53:58.041 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.041 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.041 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.041 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583997+0000", 2024-09-13T14:53:58.041 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00025479700000000002 2024-09-13T14:53:58.041 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.041 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.041 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.041 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584006+0000", 2024-09-13T14:53:58.041 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9900000000000003e-06 2024-09-13T14:53:58.041 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.041 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.041 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.042 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.689501+0000", 2024-09-13T14:53:58.042 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.105495082 2024-09-13T14:53:58.042 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.042 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.042 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.042 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.689618+0000", 2024-09-13T14:53:58.042 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000116469 2024-09-13T14:53:58.042 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.042 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.042 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:53:58.042 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.689865+0000", 2024-09-13T14:53:58.042 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00024748899999999999 2024-09-13T14:53:58.042 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.042 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.043 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.043 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693075+0000", 2024-09-13T14:53:58.043 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0032099450000000001 2024-09-13T14:53:58.043 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.043 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.043 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.043 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693314+0000", 2024-09-13T14:53:58.043 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00023886600000000001 2024-09-13T14:53:58.043 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.043 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.043 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.043 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693417+0000", 2024-09-13T14:53:58.043 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000103482 2024-09-13T14:53:58.043 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.044 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.044 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.044 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693894+0000", 2024-09-13T14:53:58.044 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00047631099999999999 2024-09-13T14:53:58.044 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.044 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.044 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.044 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.697947+0000", 2024-09-13T14:53:58.044 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0040532220000000004 2024-09-13T14:53:58.044 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.044 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.044 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.045 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698010+0000", 2024-09-13T14:53:58.045 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3177999999999997e-05 2024-09-13T14:53:58.045 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.045 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.045 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.045 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698172+0000", 2024-09-13T14:53:58.045 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00016167500000000001 2024-09-13T14:53:58.045 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.045 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.045 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.045 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.046 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.046 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181983 6.18 6:1d5af90f:::benchmark_data_smithi096_167087_object64755:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:53:58.046 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583752+0000", 2024-09-13T14:53:58.046 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 400.396416902, 2024-09-13T14:53:58.046 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.114628411, 2024-09-13T14:53:58.046 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.046 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.046 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.046 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.046 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.046 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.046 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181983 2024-09-13T14:53:58.046 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.046 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.047 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.047 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.047 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583752+0000", 2024-09-13T14:53:58.047 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.047 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.047 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.047 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.047 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583752+0000", 2024-09-13T14:53:58.047 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.047 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.047 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.047 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.047 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583752+0000", 2024-09-13T14:53:58.047 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9099999999999999e-07 2024-09-13T14:53:58.048 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.048 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.048 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.048 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583754+0000", 2024-09-13T14:53:58.048 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7969999999999999e-06 2024-09-13T14:53:58.048 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.048 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.048 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.048 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583760+0000", 2024-09-13T14:53:58.048 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1600000000000003e-06 2024-09-13T14:53:58.048 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.048 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.048 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.048 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583765+0000", 2024-09-13T14:53:58.049 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0540000000000002e-06 2024-09-13T14:53:58.049 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.049 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.049 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.049 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584049+0000", 2024-09-13T14:53:58.049 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00028396299999999998 2024-09-13T14:53:58.049 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.049 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.049 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.049 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584057+0000", 2024-09-13T14:53:58.049 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.6920000000000002e-06 2024-09-13T14:53:58.049 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.049 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.049 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.051 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.691347+0000", 2024-09-13T14:53:58.051 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.107290417 2024-09-13T14:53:58.051 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.051 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.051 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.051 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.691416+0000", 2024-09-13T14:53:58.051 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9029000000000006e-05 2024-09-13T14:53:58.051 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.051 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.051 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:53:58.052 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.691630+0000", 2024-09-13T14:53:58.052 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00021387500000000001 2024-09-13T14:53:58.052 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.052 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.052 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.052 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693330+0000", 2024-09-13T14:53:58.052 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0016996730000000001 2024-09-13T14:53:58.052 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.052 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.053 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.053 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693794+0000", 2024-09-13T14:53:58.053 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00046411 2024-09-13T14:53:58.053 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.053 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.053 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.053 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693850+0000", 2024-09-13T14:53:58.053 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5733999999999999e-05 2024-09-13T14:53:58.053 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.053 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.053 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.053 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693924+0000", 2024-09-13T14:53:58.053 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3755000000000001e-05 2024-09-13T14:53:58.053 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.054 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.054 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.054 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698216+0000", 2024-09-13T14:53:58.054 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0042927490000000002 2024-09-13T14:53:58.054 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.054 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.054 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.054 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698273+0000", 2024-09-13T14:53:58.054 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6558000000000003e-05 2024-09-13T14:53:58.054 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.054 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.055 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.055 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698380+0000", 2024-09-13T14:53:58.055 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000107454 2024-09-13T14:53:58.055 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.055 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.055 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.055 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.055 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.055 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181992 6.18 6:1dd34154:::benchmark_data_smithi096_167087_object64764:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:53:58.055 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583771+0000", 2024-09-13T14:53:58.055 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 400.39639738400001, 2024-09-13T14:53:58.055 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1206707570000001, 2024-09-13T14:53:58.056 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.056 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.056 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.056 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.056 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.056 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.056 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181992 2024-09-13T14:53:58.056 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.056 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.057 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.057 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.057 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583771+0000", 2024-09-13T14:53:58.057 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.057 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.057 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.057 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.057 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583771+0000", 2024-09-13T14:53:58.057 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.057 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.057 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.057 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.057 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583772+0000", 2024-09-13T14:53:58.058 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6199999999999999e-07 2024-09-13T14:53:58.058 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.058 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.058 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.058 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583774+0000", 2024-09-13T14:53:58.058 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7179999999999999e-06 2024-09-13T14:53:58.058 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.058 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.058 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.058 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583779+0000", 2024-09-13T14:53:58.058 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9340000000000003e-06 2024-09-13T14:53:58.058 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.058 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.059 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.059 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583784+0000", 2024-09-13T14:53:58.059 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6569999999999996e-06 2024-09-13T14:53:58.059 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.059 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.059 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.059 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584091+0000", 2024-09-13T14:53:58.059 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00030664100000000002 2024-09-13T14:53:58.059 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.059 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.059 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.059 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584097+0000", 2024-09-13T14:53:58.059 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6640000000000001e-06 2024-09-13T14:53:58.059 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.060 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.060 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.060 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.694002+0000", 2024-09-13T14:53:58.060 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1099042059999999 2024-09-13T14:53:58.060 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.060 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.060 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.060 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.694103+0000", 2024-09-13T14:53:58.060 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000101208 2024-09-13T14:53:58.060 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.060 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.060 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:53:58.060 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.694412+0000", 2024-09-13T14:53:58.060 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00030881000000000002 2024-09-13T14:53:58.061 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.061 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.061 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.061 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698418+0000", 2024-09-13T14:53:58.061 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0040063989999999999 2024-09-13T14:53:58.061 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.061 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.061 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.061 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.703944+0000", 2024-09-13T14:53:58.061 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0055258440000000002 2024-09-13T14:53:58.061 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.061 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.061 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.062 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.703970+0000", 2024-09-13T14:53:58.062 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6594999999999999e-05 2024-09-13T14:53:58.062 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.062 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.062 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.062 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704017+0000", 2024-09-13T14:53:58.062 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6727999999999997e-05 2024-09-13T14:53:58.062 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.062 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.062 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.062 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704259+0000", 2024-09-13T14:53:58.062 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000241745 2024-09-13T14:53:58.062 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.062 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.063 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.063 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704318+0000", 2024-09-13T14:53:58.063 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9117000000000003e-05 2024-09-13T14:53:58.063 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.063 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.063 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.063 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704442+0000", 2024-09-13T14:53:58.063 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000124129 2024-09-13T14:53:58.063 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.063 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.063 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.063 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.063 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.063 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182029 6.18 6:1b014b20:::benchmark_data_smithi096_167087_object64801:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:53:58.064 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583790+0000", 2024-09-13T14:53:58.064 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 400.39637922600002, 2024-09-13T14:53:58.064 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1236003029999999, 2024-09-13T14:53:58.064 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.064 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.064 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.064 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.064 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.064 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.064 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182029 2024-09-13T14:53:58.064 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.064 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.064 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.064 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.065 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583790+0000", 2024-09-13T14:53:58.065 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.065 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.065 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.065 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.065 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583790+0000", 2024-09-13T14:53:58.065 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.065 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.065 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.065 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.065 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583790+0000", 2024-09-13T14:53:58.065 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.41e-07 2024-09-13T14:53:58.066 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.066 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.066 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.066 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583792+0000", 2024-09-13T14:53:58.066 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.734e-06 2024-09-13T14:53:58.066 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.066 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.066 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.066 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583798+0000", 2024-09-13T14:53:58.066 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8749999999999997e-06 2024-09-13T14:53:58.066 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.066 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.066 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.066 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583802+0000", 2024-09-13T14:53:58.067 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.685e-06 2024-09-13T14:53:58.067 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.067 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.067 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.067 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584124+0000", 2024-09-13T14:53:58.067 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00032207999999999998 2024-09-13T14:53:58.067 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.067 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.067 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.067 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584131+0000", 2024-09-13T14:53:58.067 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2439999999999998e-06 2024-09-13T14:53:58.067 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.067 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.067 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.068 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.695817+0000", 2024-09-13T14:53:58.068 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1116864829999999 2024-09-13T14:53:58.068 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.068 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.068 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.068 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.695938+0000", 2024-09-13T14:53:58.068 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000121177 2024-09-13T14:53:58.068 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.068 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.068 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:53:58.068 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.696188+0000", 2024-09-13T14:53:58.068 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00024953900000000001 2024-09-13T14:53:58.068 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.069 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.069 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.069 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698457+0000", 2024-09-13T14:53:58.069 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0022691069999999998 2024-09-13T14:53:58.069 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.069 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.069 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.069 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.703978+0000", 2024-09-13T14:53:58.069 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0055207859999999997 2024-09-13T14:53:58.069 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.069 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.069 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.069 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704061+0000", 2024-09-13T14:53:58.069 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.3146999999999995e-05 2024-09-13T14:53:58.070 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.070 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.070 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.070 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704145+0000", 2024-09-13T14:53:58.070 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.373e-05 2024-09-13T14:53:58.070 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.070 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.070 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.070 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707052+0000", 2024-09-13T14:53:58.070 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0029077640000000002 2024-09-13T14:53:58.070 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.070 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.070 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.070 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707257+0000", 2024-09-13T14:53:58.071 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00020468299999999999 2024-09-13T14:53:58.071 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.071 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.071 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.071 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707390+0000", 2024-09-13T14:53:58.071 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013292800000000001 2024-09-13T14:53:58.071 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.071 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.071 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.071 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.071 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.071 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182109 6.18 6:1e340a78:::benchmark_data_smithi096_167087_object64881:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:53:58.071 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583808+0000", 2024-09-13T14:53:58.071 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 400.39636119400001, 2024-09-13T14:53:58.072 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1239120069999999, 2024-09-13T14:53:58.072 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.072 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.072 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.072 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.072 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.072 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.072 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182109 2024-09-13T14:53:58.072 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.072 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.072 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.072 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.072 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583808+0000", 2024-09-13T14:53:58.073 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.073 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.073 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.073 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.073 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583808+0000", 2024-09-13T14:53:58.073 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.073 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.073 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.073 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.073 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583808+0000", 2024-09-13T14:53:58.073 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7099999999999997e-07 2024-09-13T14:53:58.074 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.074 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.074 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.074 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583810+0000", 2024-09-13T14:53:58.074 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.629e-06 2024-09-13T14:53:58.074 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.074 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.074 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.074 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583816+0000", 2024-09-13T14:53:58.074 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8930000000000002e-06 2024-09-13T14:53:58.074 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.074 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.074 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.074 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583820+0000", 2024-09-13T14:53:58.075 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7879999999999997e-06 2024-09-13T14:53:58.075 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.075 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.075 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.075 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584160+0000", 2024-09-13T14:53:58.075 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00034013899999999999 2024-09-13T14:53:58.075 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.075 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.075 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.075 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584168+0000", 2024-09-13T14:53:58.075 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.063e-06 2024-09-13T14:53:58.075 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.075 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.075 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.076 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698486+0000", 2024-09-13T14:53:58.076 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1143184820000001 2024-09-13T14:53:58.076 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.076 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.076 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.076 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698605+0000", 2024-09-13T14:53:58.076 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000119276 2024-09-13T14:53:58.076 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.076 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.076 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:53:58.076 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698944+0000", 2024-09-13T14:53:58.077 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0003389 2024-09-13T14:53:58.077 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.077 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.077 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.077 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.703983+0000", 2024-09-13T14:53:58.077 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0050390319999999997 2024-09-13T14:53:58.077 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.077 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.077 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.077 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704099+0000", 2024-09-13T14:53:58.078 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011535300000000001 2024-09-13T14:53:58.078 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.078 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.078 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.078 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704202+0000", 2024-09-13T14:53:58.078 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010301700000000001 2024-09-13T14:53:58.078 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.078 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.078 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.078 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707435+0000", 2024-09-13T14:53:58.078 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0032336909999999999 2024-09-13T14:53:58.078 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.078 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.079 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.079 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707572+0000", 2024-09-13T14:53:58.079 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000136923 2024-09-13T14:53:58.079 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.079 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.079 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.079 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707624+0000", 2024-09-13T14:53:58.079 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.1490000000000003e-05 2024-09-13T14:53:58.079 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.079 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.079 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.079 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707720+0000", 2024-09-13T14:53:58.079 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.5959999999999996e-05 2024-09-13T14:53:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182126 6.18 6:1d91b7c9:::benchmark_data_smithi096_167087_object64898:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:53:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583826+0000", 2024-09-13T14:53:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 400.39634296100002, 2024-09-13T14:53:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1240857499999999, 2024-09-13T14:53:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182126 2024-09-13T14:53:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583826+0000", 2024-09-13T14:53:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583826+0000", 2024-09-13T14:53:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.082 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.082 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.082 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.082 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583826+0000", 2024-09-13T14:53:58.082 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4900000000000001e-07 2024-09-13T14:53:58.082 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.082 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.082 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.082 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583828+0000", 2024-09-13T14:53:58.082 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6589999999999999e-06 2024-09-13T14:53:58.082 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.082 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.082 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.082 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583834+0000", 2024-09-13T14:53:58.083 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7799999999999997e-06 2024-09-13T14:53:58.083 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.083 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.083 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.083 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583839+0000", 2024-09-13T14:53:58.083 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.887e-06 2024-09-13T14:53:58.083 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.083 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.083 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.083 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584196+0000", 2024-09-13T14:53:58.083 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00035786699999999999 2024-09-13T14:53:58.083 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.083 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.083 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.084 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584204+0000", 2024-09-13T14:53:58.084 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2289999999999997e-06 2024-09-13T14:53:58.084 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.084 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.084 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.084 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.701272+0000", 2024-09-13T14:53:58.084 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1170683269999999 2024-09-13T14:53:58.084 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.084 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.084 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.084 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.701391+0000", 2024-09-13T14:53:58.084 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011928 2024-09-13T14:53:58.085 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.085 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.085 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:53:58.085 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.701768+0000", 2024-09-13T14:53:58.085 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00037707099999999998 2024-09-13T14:53:58.085 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.085 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.085 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.085 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707000+0000", 2024-09-13T14:53:58.085 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0052320379999999996 2024-09-13T14:53:58.085 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.085 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.085 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.085 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707013+0000", 2024-09-13T14:53:58.086 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2853999999999999e-05 2024-09-13T14:53:58.086 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.086 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.086 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.086 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707476+0000", 2024-09-13T14:53:58.086 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000462595 2024-09-13T14:53:58.086 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.086 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.086 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.086 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707518+0000", 2024-09-13T14:53:58.086 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2154000000000003e-05 2024-09-13T14:53:58.086 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.086 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.086 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.087 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707760+0000", 2024-09-13T14:53:58.087 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00024246099999999999 2024-09-13T14:53:58.087 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.087 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.087 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.087 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707802+0000", 2024-09-13T14:53:58.087 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1146999999999998e-05 2024-09-13T14:53:58.087 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.087 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.087 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.087 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707912+0000", 2024-09-13T14:53:58.087 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000110052 2024-09-13T14:53:58.087 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.088 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.088 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.088 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.088 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.088 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182128 6.18 6:1c1d9d13:::benchmark_data_smithi096_167087_object64900:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:53:58.088 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583844+0000", 2024-09-13T14:53:58.088 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 400.39632494699998, 2024-09-13T14:53:58.088 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1297894319999999, 2024-09-13T14:53:58.088 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.088 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.088 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.088 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.089 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.089 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.089 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182128 2024-09-13T14:53:58.089 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.089 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.089 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.089 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.089 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583844+0000", 2024-09-13T14:53:58.089 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.089 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.089 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.089 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.089 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583844+0000", 2024-09-13T14:53:58.090 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.090 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.090 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.090 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.090 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583844+0000", 2024-09-13T14:53:58.090 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6300000000000001e-07 2024-09-13T14:53:58.090 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.090 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.090 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.090 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583850+0000", 2024-09-13T14:53:58.090 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8209999999999998e-06 2024-09-13T14:53:58.090 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.090 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.090 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.091 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583856+0000", 2024-09-13T14:53:58.091 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8309999999999997e-06 2024-09-13T14:53:58.091 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.091 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.091 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.091 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583861+0000", 2024-09-13T14:53:58.091 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9969999999999998e-06 2024-09-13T14:53:58.091 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.091 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.091 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.091 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584231+0000", 2024-09-13T14:53:58.091 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00037027000000000002 2024-09-13T14:53:58.091 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.092 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.092 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.092 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584238+0000", 2024-09-13T14:53:58.092 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.562e-06 2024-09-13T14:53:58.092 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.092 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.092 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.092 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704485+0000", 2024-09-13T14:53:58.092 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.12024762 2024-09-13T14:53:58.092 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.092 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.092 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.092 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704619+0000", 2024-09-13T14:53:58.092 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013364200000000001 2024-09-13T14:53:58.093 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.093 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.093 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:53:58.093 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704986+0000", 2024-09-13T14:53:58.093 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00036733799999999999 2024-09-13T14:53:58.093 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.093 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.093 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.093 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713182+0000", 2024-09-13T14:53:58.093 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.008195239 2024-09-13T14:53:58.093 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.093 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.093 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.093 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713224+0000", 2024-09-13T14:53:58.094 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2001e-05 2024-09-13T14:53:58.094 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.094 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.094 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.094 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713287+0000", 2024-09-13T14:53:58.094 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3285000000000001e-05 2024-09-13T14:53:58.094 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.094 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.094 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.094 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713391+0000", 2024-09-13T14:53:58.094 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000104104 2024-09-13T14:53:58.094 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.094 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.095 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.095 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713434+0000", 2024-09-13T14:53:58.095 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3541000000000002e-05 2024-09-13T14:53:58.095 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.095 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.095 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.095 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713495+0000", 2024-09-13T14:53:58.095 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0489000000000003e-05 2024-09-13T14:53:58.095 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.095 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.095 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.095 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713633+0000", 2024-09-13T14:53:58.095 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000138329 2024-09-13T14:53:58.095 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.096 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.096 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.096 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.096 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.096 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182157 6.18 6:1a63e408:::benchmark_data_smithi096_167087_object64929:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:53:58.096 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583866+0000", 2024-09-13T14:53:58.096 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 400.39630240499997, 2024-09-13T14:53:58.096 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1353496999999999, 2024-09-13T14:53:58.096 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.096 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.096 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.096 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.096 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.096 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.097 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182157 2024-09-13T14:53:58.097 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.097 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.097 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.097 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.097 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583866+0000", 2024-09-13T14:53:58.097 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.097 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.097 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.097 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.097 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583866+0000", 2024-09-13T14:53:58.097 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.097 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.098 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.098 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.098 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583867+0000", 2024-09-13T14:53:58.098 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.41e-07 2024-09-13T14:53:58.098 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.098 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.098 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.098 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583868+0000", 2024-09-13T14:53:58.098 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5990000000000001e-06 2024-09-13T14:53:58.098 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.098 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.098 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.098 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583874+0000", 2024-09-13T14:53:58.098 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7579999999999998e-06 2024-09-13T14:53:58.099 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.099 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.099 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.099 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583890+0000", 2024-09-13T14:53:58.099 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6322999999999999e-05 2024-09-13T14:53:58.099 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.099 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.099 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.099 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584264+0000", 2024-09-13T14:53:58.099 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00037345899999999998 2024-09-13T14:53:58.099 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.099 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.099 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.099 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584271+0000", 2024-09-13T14:53:58.100 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6989999999999997e-06 2024-09-13T14:53:58.100 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.100 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.100 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.100 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707962+0000", 2024-09-13T14:53:58.100 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1236913660000001 2024-09-13T14:53:58.100 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.100 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.100 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.100 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.708079+0000", 2024-09-13T14:53:58.100 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000116842 2024-09-13T14:53:58.100 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.100 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.100 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:53:58.101 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.708445+0000", 2024-09-13T14:53:58.101 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00036596699999999997 2024-09-13T14:53:58.101 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.101 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.101 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.101 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713237+0000", 2024-09-13T14:53:58.101 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0047922049999999999 2024-09-13T14:53:58.101 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.101 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.101 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.101 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713680+0000", 2024-09-13T14:53:58.101 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000443163 2024-09-13T14:53:58.101 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.102 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.102 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.102 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713732+0000", 2024-09-13T14:53:58.102 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2092000000000002e-05 2024-09-13T14:53:58.102 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.102 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.102 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.102 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713822+0000", 2024-09-13T14:53:58.102 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9532999999999998e-05 2024-09-13T14:53:58.102 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.102 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.102 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.102 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.718875+0000", 2024-09-13T14:53:58.102 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0050529379999999999 2024-09-13T14:53:58.103 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.103 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.103 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.103 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719086+0000", 2024-09-13T14:53:58.103 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00021114800000000001 2024-09-13T14:53:58.103 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.103 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.103 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.103 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719216+0000", 2024-09-13T14:53:58.103 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013026699999999999 2024-09-13T14:53:58.103 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.103 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.103 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.103 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.104 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.104 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182208 6.18 6:1bb7076d:::benchmark_data_smithi096_167087_object64980:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:53:58.104 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583897+0000", 2024-09-13T14:53:58.104 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 400.39627165399997, 2024-09-13T14:53:58.104 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.135635011, 2024-09-13T14:53:58.104 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.104 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.104 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.104 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.104 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.104 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.104 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182208 2024-09-13T14:53:58.104 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.104 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.105 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.105 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.105 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583897+0000", 2024-09-13T14:53:58.105 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.105 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.105 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.105 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.105 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583897+0000", 2024-09-13T14:53:58.105 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.105 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.105 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.105 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.105 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583898+0000", 2024-09-13T14:53:58.106 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6300000000000001e-07 2024-09-13T14:53:58.106 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.106 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.106 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.106 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583899+0000", 2024-09-13T14:53:58.106 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.885e-06 2024-09-13T14:53:58.106 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.106 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.106 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.106 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583905+0000", 2024-09-13T14:53:58.106 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.003e-06 2024-09-13T14:53:58.106 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.106 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.106 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.107 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583910+0000", 2024-09-13T14:53:58.107 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8919999999999999e-06 2024-09-13T14:53:58.107 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.107 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.107 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.107 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584608+0000", 2024-09-13T14:53:58.107 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00069783300000000003 2024-09-13T14:53:58.107 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.107 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.107 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.107 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584616+0000", 2024-09-13T14:53:58.107 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.2209999999999997e-06 2024-09-13T14:53:58.107 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.107 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.108 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.108 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.710538+0000", 2024-09-13T14:53:58.108 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.125921197 2024-09-13T14:53:58.108 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.108 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.108 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.108 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.710659+0000", 2024-09-13T14:53:58.108 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000121261 2024-09-13T14:53:58.108 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.108 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.110 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:53:58.110 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.710998+0000", 2024-09-13T14:53:58.110 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00033875300000000001 2024-09-13T14:53:58.110 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.110 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.110 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.110 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713773+0000", 2024-09-13T14:53:58.110 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027755750000000002 2024-09-13T14:53:58.110 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.111 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.111 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.111 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.718788+0000", 2024-09-13T14:53:58.111 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0050151279999999998 2024-09-13T14:53:58.111 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.111 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.111 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.111 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.718829+0000", 2024-09-13T14:53:58.111 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0686000000000003e-05 2024-09-13T14:53:58.111 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.111 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.111 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.111 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719261+0000", 2024-09-13T14:53:58.111 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00043173400000000002 2024-09-13T14:53:58.112 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.112 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.112 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.112 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719336+0000", 2024-09-13T14:53:58.112 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.5703000000000003e-05 2024-09-13T14:53:58.112 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.112 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.112 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.112 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719425+0000", 2024-09-13T14:53:58.112 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.852e-05 2024-09-13T14:53:58.112 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.112 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.112 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.113 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719532+0000", 2024-09-13T14:53:58.113 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010725700000000001 2024-09-13T14:53:58.113 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.113 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.113 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.113 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.113 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.113 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182214 6.18 6:197a4439:::benchmark_data_smithi096_167087_object64986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:53:58.113 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.612524+0000", 2024-09-13T14:53:58.113 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 400.36764432799998, 2024-09-13T14:53:58.113 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.11521797, 2024-09-13T14:53:58.113 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.113 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.113 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.114 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.114 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.114 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.114 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182214 2024-09-13T14:53:58.114 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.114 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.114 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.114 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.114 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612524+0000", 2024-09-13T14:53:58.114 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.114 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.114 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.114 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.115 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612524+0000", 2024-09-13T14:53:58.115 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.115 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.115 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.115 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.115 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612526+0000", 2024-09-13T14:53:58.115 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.347e-06 2024-09-13T14:53:58.115 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.115 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.115 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.115 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612531+0000", 2024-09-13T14:53:58.115 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9799999999999998e-06 2024-09-13T14:53:58.115 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.115 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.116 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.116 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612542+0000", 2024-09-13T14:53:58.116 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0831e-05 2024-09-13T14:53:58.116 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.116 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.116 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.116 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612554+0000", 2024-09-13T14:53:58.116 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2283000000000001e-05 2024-09-13T14:53:58.116 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.116 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.116 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.116 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612613+0000", 2024-09-13T14:53:58.116 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8967e-05 2024-09-13T14:53:58.116 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.117 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.117 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.117 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612622+0000", 2024-09-13T14:53:58.117 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0399999999999998e-06 2024-09-13T14:53:58.117 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.117 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.117 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.117 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713854+0000", 2024-09-13T14:53:58.117 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.101231611 2024-09-13T14:53:58.117 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.117 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.117 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.117 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713963+0000", 2024-09-13T14:53:58.117 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010902 2024-09-13T14:53:58.118 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.118 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.118 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:53:58.118 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.714339+0000", 2024-09-13T14:53:58.118 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00037666499999999999 2024-09-13T14:53:58.118 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.118 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.118 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.118 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.718841+0000", 2024-09-13T14:53:58.118 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0045017240000000004 2024-09-13T14:53:58.118 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.118 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.118 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.118 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719298+0000", 2024-09-13T14:53:58.119 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00045695499999999999 2024-09-13T14:53:58.119 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.119 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.119 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.119 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.724722+0000", 2024-09-13T14:53:58.119 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0054238799999999998 2024-09-13T14:53:58.119 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.119 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.119 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.119 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.724759+0000", 2024-09-13T14:53:58.119 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6752999999999999e-05 2024-09-13T14:53:58.119 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.119 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.119 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.120 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727512+0000", 2024-09-13T14:53:58.120 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027530179999999999 2024-09-13T14:53:58.120 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.120 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.120 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.120 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727622+0000", 2024-09-13T14:53:58.120 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011055100000000001 2024-09-13T14:53:58.120 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.120 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.120 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.120 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727742+0000", 2024-09-13T14:53:58.120 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000120345 2024-09-13T14:53:58.120 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.120 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.121 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.121 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.121 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.121 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182224 6.18 6:1a82ea4f:::benchmark_data_smithi096_167087_object64996:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:53:58.121 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.621148+0000", 2024-09-13T14:53:58.121 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 400.35902083500002, 2024-09-13T14:53:58.121 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.115259811, 2024-09-13T14:53:58.121 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.121 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.121 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.121 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.121 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.121 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.121 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182224 2024-09-13T14:53:58.122 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.122 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.122 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.122 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.122 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621148+0000", 2024-09-13T14:53:58.122 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.122 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.122 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.122 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.122 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621148+0000", 2024-09-13T14:53:58.122 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.122 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.122 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.123 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.123 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621149+0000", 2024-09-13T14:53:58.123 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4729999999999999e-06 2024-09-13T14:53:58.123 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.123 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.123 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.123 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621154+0000", 2024-09-13T14:53:58.123 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0010000000000001e-06 2024-09-13T14:53:58.123 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.123 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.123 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.123 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621167+0000", 2024-09-13T14:53:58.123 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2558000000000001e-05 2024-09-13T14:53:58.123 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.124 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.124 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.124 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621174+0000", 2024-09-13T14:53:58.124 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.4390000000000003e-06 2024-09-13T14:53:58.124 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.124 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.124 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.124 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621688+0000", 2024-09-13T14:53:58.124 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00051370099999999998 2024-09-13T14:53:58.124 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.124 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.124 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.124 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621697+0000", 2024-09-13T14:53:58.124 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0529999999999996e-06 2024-09-13T14:53:58.125 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.125 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.125 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.125 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.716358+0000", 2024-09-13T14:53:58.125 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0946611879999999 2024-09-13T14:53:58.125 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.125 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.125 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.125 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.716475+0000", 2024-09-13T14:53:58.125 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000116692 2024-09-13T14:53:58.125 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.126 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.126 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:53:58.126 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.716866+0000", 2024-09-13T14:53:58.126 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00039121499999999999 2024-09-13T14:53:58.126 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.126 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.126 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.126 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.724682+0000", 2024-09-13T14:53:58.126 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.007815443 2024-09-13T14:53:58.126 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.126 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.126 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.126 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727435+0000", 2024-09-13T14:53:58.127 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027537030000000001 2024-09-13T14:53:58.127 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.127 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.127 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.127 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727474+0000", 2024-09-13T14:53:58.127 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9038000000000001e-05 2024-09-13T14:53:58.127 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.127 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.127 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.127 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727783+0000", 2024-09-13T14:53:58.127 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00030899999999999998 2024-09-13T14:53:58.127 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.127 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.127 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.128 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736174+0000", 2024-09-13T14:53:58.128 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0083904540000000003 2024-09-13T14:53:58.128 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.128 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.128 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.128 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736271+0000", 2024-09-13T14:53:58.128 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7455999999999997e-05 2024-09-13T14:53:58.128 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.128 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.128 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.128 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736408+0000", 2024-09-13T14:53:58.128 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013639699999999999 2024-09-13T14:53:58.128 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.129 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.129 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.129 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.129 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.129 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182244 6.18 6:1bc8bd61:::benchmark_data_smithi096_167087_object65016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:53:58.129 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.649532+0000", 2024-09-13T14:53:58.129 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 400.33063681499999, 2024-09-13T14:53:58.129 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0882464169999999, 2024-09-13T14:53:58.129 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.129 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.129 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.129 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.129 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.130 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.130 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182244 2024-09-13T14:53:58.130 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.130 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.130 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.130 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.130 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649532+0000", 2024-09-13T14:53:58.130 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.130 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.130 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.130 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.130 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649532+0000", 2024-09-13T14:53:58.130 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.130 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.131 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.131 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.131 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649534+0000", 2024-09-13T14:53:58.131 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.643e-06 2024-09-13T14:53:58.131 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.131 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.131 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.131 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649539+0000", 2024-09-13T14:53:58.131 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.835e-06 2024-09-13T14:53:58.131 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.131 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.131 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.131 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649556+0000", 2024-09-13T14:53:58.131 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6881e-05 2024-09-13T14:53:58.132 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.132 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.132 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.132 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649564+0000", 2024-09-13T14:53:58.132 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.8609999999999998e-06 2024-09-13T14:53:58.132 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.132 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.132 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.132 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649620+0000", 2024-09-13T14:53:58.132 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5701000000000001e-05 2024-09-13T14:53:58.132 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.132 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.132 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649629+0000", 2024-09-13T14:53:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.867e-06 2024-09-13T14:53:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719571+0000", 2024-09-13T14:53:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0699419960000001 2024-09-13T14:53:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719712+0000", 2024-09-13T14:53:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000140896 2024-09-13T14:53:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.133 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:53:58.134 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.720080+0000", 2024-09-13T14:53:58.134 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00036818400000000002 2024-09-13T14:53:58.134 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.134 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.134 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.134 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727822+0000", 2024-09-13T14:53:58.134 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0077419489999999997 2024-09-13T14:53:58.134 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.134 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.134 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.134 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736451+0000", 2024-09-13T14:53:58.134 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0086296340000000006 2024-09-13T14:53:58.134 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.134 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.135 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.135 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736465+0000", 2024-09-13T14:53:58.135 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3386e-05 2024-09-13T14:53:58.135 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.135 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.135 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.135 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737385+0000", 2024-09-13T14:53:58.135 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00092018600000000003 2024-09-13T14:53:58.135 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.135 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.135 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.135 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737579+0000", 2024-09-13T14:53:58.135 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00019378700000000001 2024-09-13T14:53:58.135 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.136 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.136 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.136 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737639+0000", 2024-09-13T14:53:58.136 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0445999999999997e-05 2024-09-13T14:53:58.136 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.136 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.136 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.136 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737778+0000", 2024-09-13T14:53:58.136 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013916500000000001 2024-09-13T14:53:58.136 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.136 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.136 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.136 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.136 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.137 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182259 6.18 6:188e1d88:::benchmark_data_smithi096_167087_object65031:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:53:58.137 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.675788+0000", 2024-09-13T14:53:58.137 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 400.30438104699999, 2024-09-13T14:53:58.137 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0622151870000001, 2024-09-13T14:53:58.137 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.137 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.137 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.137 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.137 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.137 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.137 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182259 2024-09-13T14:53:58.137 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.137 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.137 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.138 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.138 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675788+0000", 2024-09-13T14:53:58.138 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.138 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.138 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.138 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.138 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675788+0000", 2024-09-13T14:53:58.138 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.138 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.138 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.138 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.138 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675789+0000", 2024-09-13T14:53:58.138 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3960000000000001e-06 2024-09-13T14:53:58.138 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.139 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.139 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.139 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675795+0000", 2024-09-13T14:53:58.139 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7359999999999998e-06 2024-09-13T14:53:58.139 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.139 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.139 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.139 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675806+0000", 2024-09-13T14:53:58.139 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0953e-05 2024-09-13T14:53:58.139 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.139 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.139 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.139 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675813+0000", 2024-09-13T14:53:58.139 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7660000000000001e-06 2024-09-13T14:53:58.140 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.140 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.140 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.140 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675869+0000", 2024-09-13T14:53:58.140 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6610000000000002e-05 2024-09-13T14:53:58.140 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.140 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.140 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.140 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675878+0000", 2024-09-13T14:53:58.140 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6039999999999993e-06 2024-09-13T14:53:58.140 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.140 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.140 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.140 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.722283+0000", 2024-09-13T14:53:58.141 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.046405502 2024-09-13T14:53:58.141 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.141 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.141 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.141 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.722393+0000", 2024-09-13T14:53:58.141 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000109314 2024-09-13T14:53:58.141 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.141 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.141 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:53:58.141 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.722717+0000", 2024-09-13T14:53:58.141 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00032454599999999999 2024-09-13T14:53:58.141 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.141 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.142 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.142 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727862+0000", 2024-09-13T14:53:58.142 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.005144511 2024-09-13T14:53:58.142 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.142 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.142 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.142 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736481+0000", 2024-09-13T14:53:58.142 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0086192339999999999 2024-09-13T14:53:58.142 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.143 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.143 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.143 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737339+0000", 2024-09-13T14:53:58.143 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00085788100000000001 2024-09-13T14:53:58.143 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.143 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.143 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.143 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737433+0000", 2024-09-13T14:53:58.143 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3827999999999997e-05 2024-09-13T14:53:58.143 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.143 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.143 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.143 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737825+0000", 2024-09-13T14:53:58.143 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00039279099999999997 2024-09-13T14:53:58.144 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.144 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.144 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.144 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737878+0000", 2024-09-13T14:53:58.144 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2431e-05 2024-09-13T14:53:58.144 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.144 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.144 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.144 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.738003+0000", 2024-09-13T14:53:58.144 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000125084 2024-09-13T14:53:58.144 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.144 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.144 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.145 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.145 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.145 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182332 6.18 6:1cc35df1:::benchmark_data_smithi096_167087_object65104:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:53:58.145 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.849479+0000", 2024-09-13T14:53:58.145 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 400.13068943799999, 2024-09-13T14:53:58.145 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.90088822300000004, 2024-09-13T14:53:58.145 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:53:58.145 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:53:58.145 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:58.145 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:53:58.145 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:53:58.145 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:53:58.145 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182332 2024-09-13T14:53:58.145 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.146 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:53:58.146 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.146 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:53:58.146 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849479+0000", 2024-09-13T14:53:58.146 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.146 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.146 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.146 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:53:58.146 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849479+0000", 2024-09-13T14:53:58.146 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:53:58.146 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.146 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.146 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:53:58.146 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849481+0000", 2024-09-13T14:53:58.147 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1820000000000001e-06 2024-09-13T14:53:58.148 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.148 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.148 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:53:58.148 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849485+0000", 2024-09-13T14:53:58.148 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4660000000000001e-06 2024-09-13T14:53:58.148 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.148 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.148 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:53:58.148 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849495+0000", 2024-09-13T14:53:58.148 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7189999999999996e-06 2024-09-13T14:53:58.148 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.148 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.148 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:53:58.149 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849500+0000", 2024-09-13T14:53:58.149 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2650000000000004e-06 2024-09-13T14:53:58.149 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.149 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.149 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.149 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849548+0000", 2024-09-13T14:53:58.149 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7787999999999997e-05 2024-09-13T14:53:58.149 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.149 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.149 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:53:58.149 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849564+0000", 2024-09-13T14:53:58.149 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6016e-05 2024-09-13T14:53:58.149 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.149 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.150 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:53:58.150 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727890+0000", 2024-09-13T14:53:58.150 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.87832629399999995 2024-09-13T14:53:58.150 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.150 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.150 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:53:58.150 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727992+0000", 2024-09-13T14:53:58.150 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000101783 2024-09-13T14:53:58.150 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.150 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.150 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:53:58.150 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.728376+0000", 2024-09-13T14:53:58.150 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00038430400000000003 2024-09-13T14:53:58.150 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.151 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.151 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.151 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737484+0000", 2024-09-13T14:53:58.151 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.009108105 2024-09-13T14:53:58.151 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.151 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.151 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.151 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737529+0000", 2024-09-13T14:53:58.151 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4348999999999998e-05 2024-09-13T14:53:58.151 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.151 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.151 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.151 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.738046+0000", 2024-09-13T14:53:58.151 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00051743500000000001 2024-09-13T14:53:58.152 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.152 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.152 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:53:58.152 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.750093+0000", 2024-09-13T14:53:58.152 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.012047333 2024-09-13T14:53:58.152 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.152 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.152 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:58.152 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.750184+0000", 2024-09-13T14:53:58.152 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0740000000000002e-05 2024-09-13T14:53:58.152 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.152 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.152 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:53:58.152 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.750249+0000", 2024-09-13T14:53:58.153 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4417000000000003e-05 2024-09-13T14:53:58.153 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:53:58.153 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:53:58.153 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:53:58.153 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.750368+0000", 2024-09-13T14:53:58.153 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000119027 2024-09-13T14:53:58.153 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.153 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.153 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.153 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:53:58.153 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:53:58.153 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:53:58.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.155 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:58.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:58.183 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:53:58.205 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:53:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:53:57] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.51.0" 2024-09-13T14:53:58.256 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:58.284 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:58.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:58.386 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:58.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.422 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:58.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:58.523 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:58.550 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:58.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:58.613 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 35 16 68875 68859 122.933 80.1875 0.00929445 0.0081298 2024-09-13T14:53:58.651 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:58.682 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:58.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:58.784 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:58.813 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:58.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:58.915 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:58.951 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:58.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:59.052 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:59.091 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:59.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:59.193 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:59.222 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:59.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:59.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:53:59 smithi096 ceph-mon[98901]: pgmap v838: 99 pgs: 99 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:53:59.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:53:59 smithi096 ceph-mon[102928]: pgmap v838: 99 pgs: 99 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:53:59.325 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:59.368 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:59.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:59.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:53:59 smithi204 ceph-mon[87071]: pgmap v838: 99 pgs: 99 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:53:59.470 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:59.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.501 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:59.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:59.602 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:59.613 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 36 16 70293 70277 121.98 88.625 0.0124635 0.00819238 2024-09-13T14:53:59.639 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:59.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:59.741 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:59.772 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:53:59.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:59.874 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:59.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.913 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:53:59.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:59.954 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:54:00.014 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:00.082 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:00.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:00.185 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:00.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.216 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:00.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:00.317 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:00.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.348 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:00.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:00.449 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:00.472 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:54:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:54:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:00.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.490 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:00.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:00.591 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:00.614 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 37 16 71636 71620 120.949 83.9375 0.0163534 0.00826348 2024-09-13T14:54:00.642 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:00.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:00.743 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:00.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.773 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:00.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:00.874 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:00.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.910 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:00.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:01.011 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:01.043 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:01.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:01.145 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:01.183 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:01.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:01.285 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:01.319 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:01.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:01.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:01 smithi096 ceph-mon[98901]: pgmap v839: 99 pgs: 99 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-13T14:54:01.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:01 smithi096 ceph-mon[102928]: pgmap v839: 99 pgs: 99 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-13T14:54:01.421 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:01.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.455 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:01.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:01.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:01 smithi204 ceph-mon[87071]: pgmap v839: 99 pgs: 99 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-13T14:54:01.556 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:01.594 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:01.608 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:01.614 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 38 16 73030 73014 120.059 87.125 0.00911167 0.00832235 2024-09-13T14:54:01.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.625 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:01.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:01.727 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:01.768 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:01.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:01.869 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:01.895 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:01.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:01.998 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:02.038 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:02.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:02.139 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:02.170 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:02.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:02.252 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:02 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:54:02.253 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:02 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:54:02.271 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:02.310 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:02.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:02.412 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:02.437 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:02.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:02.439 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:54:02.439 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:54:02.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:02 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:54:02.476 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:54:02.476 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:54:02.513 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:54:02.514 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:54:02.539 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:02.555 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:54:02.555 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:54:02.580 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:02.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:02.588 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:54:02.589 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:54:02.614 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 39 16 74137 74121 118.755 69.1875 0.00523953 0.00841608 2024-09-13T14:54:02.631 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:54:02.631 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:54:02.670 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:54:02.670 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:54:02.682 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:02.699 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T14:54:02.700 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T14:54:02.700 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T14:54:02.700 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T14:54:02.700 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T14:54:02.700 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T14:54:02.700 INFO:teuthology.orchestra.run.smithi096.stderr:bench [] [] [] [] 2024-09-13T14:54:02.700 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:54:02.700 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T14:54:02.700 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T14:54:02.700 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore allocator dump block 2024-09-13T14:54:02.700 INFO:teuthology.orchestra.run.smithi096.stderr:Error EINVAL: invalid command 2024-09-13T14:54:02.701 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:54:02.701 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:54:02.710 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:02.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:02.812 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:02.845 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:02.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:02.948 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.986 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:02.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:03.088 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:03.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.120 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:03.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:03 smithi204 ceph-mon[87071]: pgmap v840: 99 pgs: 99 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T14:54:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:03 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:03 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:03 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:03 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:03 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1547670762' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:03.222 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:03.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.244 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:03.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:03.348 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:03.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.382 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:03.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:03.483 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:03.488 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:03 smithi096 ceph-mon[98901]: pgmap v840: 99 pgs: 99 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T14:54:03.488 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:03 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:03.488 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:03 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:03.488 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:03 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:03.489 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:03 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:03.489 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:03 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1547670762' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:03.489 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:03 smithi096 ceph-mon[102928]: pgmap v840: 99 pgs: 99 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T14:54:03.489 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:03 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:03.489 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:03 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:03.489 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:03 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:03.489 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:03 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:03.489 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:03 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1547670762' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:03.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.523 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:03.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:03.614 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:54:03.613831+0000 min lat: 0.00118538 max lat: 0.273309 avg lat: 0.00849265 2024-09-13T14:54:03.614 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:54:03.614 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 40 16 75353 75337 117.686 76 0.0155854 0.00849265 2024-09-13T14:54:03.625 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:03.667 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:03.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:03.769 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:03.812 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:03.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:03.914 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:03.949 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:03.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:04.051 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:04.085 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:04.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:04.188 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:04.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.217 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:04.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:04.318 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:04.345 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:04.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:04.447 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:04.479 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:04.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:04.584 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:04.614 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 41 16 76500 76484 116.564 71.6875 0.00564603 0.00857375 2024-09-13T14:54:04.628 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:04.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:04.730 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:04.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.770 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:04.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:04.871 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:04.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.903 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:04.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:05.004 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:05.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.045 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:05.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:05.146 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:05.295 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:05.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:05.385 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:05 smithi204 ceph-mon[87071]: pgmap v841: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:54:05.401 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:05.437 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:05.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:05.526 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:05 smithi096 ceph-mon[98901]: pgmap v841: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:54:05.526 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:05 smithi096 ceph-mon[102928]: pgmap v841: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:54:05.540 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:05.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.576 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:05.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:05.614 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 42 16 77764 77748 115.669 79 0.00630577 0.00864068 2024-09-13T14:54:05.677 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:05.707 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:05.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:05.812 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:05.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.845 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:05.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:05.946 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:05.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.994 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:05.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:06.096 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:06.129 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:06.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:06.231 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:06.267 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:06.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:06.369 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:06.398 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:06.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:06.501 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:06.541 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:06.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:06.614 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 43 16 79077 79061 114.888 82.0625 0.00465198 0.00869894 2024-09-13T14:54:06.643 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:06.676 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:06.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:06.777 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:06.818 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:06.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:06.920 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:06.959 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:06.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:07.061 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:07.096 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:07.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:07.154 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:07.154 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:07.154 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:06 smithi204 ceph-mon[87071]: pgmap v842: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:54:07.198 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:07.200 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:07.200 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:07.200 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:06 smithi096 ceph-mon[98901]: pgmap v842: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:54:07.200 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:07.200 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:07.200 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:06 smithi096 ceph-mon[102928]: pgmap v842: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:54:07.235 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:07.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:07.337 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:07.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.381 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:07.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:07.482 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:07.513 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:07.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:07.614 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 44 16 80344 80328 114.076 79.1875 0.0108735 0.00876054 2024-09-13T14:54:07.615 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:07.649 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:07.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:07.754 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:07.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.793 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:07.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:07.894 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:07.929 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:07.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:08.031 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:08.072 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:08.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:08.174 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:08.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.211 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:08.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:08.312 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:08.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:54:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:54:07] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.51.0" 2024-09-13T14:54:08.355 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:08.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:08.376 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:54:08.376 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:54:08.377 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.5 'trim stale osdmaps' 2024-09-13T14:54:08.456 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:54:08.457 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:08.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.497 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:08.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:08.598 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:08.615 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 45 16 81519 81503 113.173 73.4375 0.00604961 0.00883058 2024-09-13T14:54:08.623 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:08.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:08.725 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:08.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.771 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:08.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:08.872 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:08.909 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:08.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:09.010 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:09.038 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:09.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:09.140 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:09.166 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:09.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:09.268 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:09.307 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:09.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:09.321 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:08 smithi096 ceph-mon[98901]: pgmap v843: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-13T14:54:09.321 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:08 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:09.321 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:08 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:09.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:08 smithi096 ceph-mon[102928]: pgmap v843: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-13T14:54:09.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:08 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:09.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:08 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:09.409 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:09.437 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:09.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:08 smithi204 ceph-mon[87071]: pgmap v843: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-13T14:54:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:08 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:08 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:09.493 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:54:09.493 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:54:09.539 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:09.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.584 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:09.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:09.615 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 46 16 82571 82555 112.141 65.75 0.0100551 0.00891225 2024-09-13T14:54:09.685 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:09.721 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:09.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:09.756 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:09.822 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:09.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.844 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:09.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:09.945 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:09.984 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:09.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:10.002 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:54:10.086 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:10.120 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:10.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:10.222 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:10.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.245 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:10.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:10.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:09 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2231643843' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:10.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:09 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:10.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:09 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2231643843' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:10.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:09 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:10.346 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:10.378 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:10.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:09 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2231643843' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:09 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:54:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:54:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:10.481 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:10.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.511 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:10.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:10.612 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:10.615 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 47 16 83490 83474 110.977 57.4375 0.077063 0.00900061 2024-09-13T14:54:10.646 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:10.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:10.748 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:10.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.792 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:10.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:10.893 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:10.914 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:10.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:11.016 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:11.048 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:11.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:11.150 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:11.180 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:11.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:11.282 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:11.318 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:11.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:11.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:10 smithi096 ceph-mon[98901]: pgmap v844: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:54:11.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:10 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:11.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:10 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:11.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:10 smithi096 ceph-mon[98901]: osdmap e277: 8 total, 8 up, 8 in 2024-09-13T14:54:11.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:10 smithi096 ceph-mon[102928]: pgmap v844: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:54:11.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:10 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:11.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:10 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:11.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:10 smithi096 ceph-mon[102928]: osdmap e277: 8 total, 8 up, 8 in 2024-09-13T14:54:11.420 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:11.464 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:11.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:11.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:10 smithi204 ceph-mon[87071]: pgmap v844: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:54:11.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:10 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:11.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:10 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:11.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:10 smithi204 ceph-mon[87071]: osdmap e277: 8 total, 8 up, 8 in 2024-09-13T14:54:11.566 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:11.604 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:11.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:11.615 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 48 16 84566 84550 110.066 67.25 0.011384 0.00908096 2024-09-13T14:54:11.706 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:11.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.733 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:11.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:11.834 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:11.869 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:11.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:11.971 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:11.995 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:11.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:12.097 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:12.132 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:12.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:12.234 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:12.274 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:12.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:12.376 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:12.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.406 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:12.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:12.508 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:12.538 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:12.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:12.615 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 49 16 85666 85650 109.223 68.75 0.00601326 0.00915073 2024-09-13T14:54:12.640 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:12.673 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:12.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:12.775 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:12.798 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:12.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:12.900 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:12.935 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:12.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:13.036 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:13.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.076 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:13.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:13.177 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:13.210 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:13.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:13.240 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:12 smithi096 ceph-mon[98901]: pgmap v846: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:54:13.241 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:12 smithi096 ceph-mon[102928]: pgmap v846: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:54:13.312 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:13.341 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:13.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:13.443 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:13.464 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:13.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:13.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:12 smithi204 ceph-mon[87071]: pgmap v846: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:54:13.566 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:13.600 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:13.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:13.615 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 50 16 86647 86631 108.265 61.3125 0.007124 0.00923201 2024-09-13T14:54:13.702 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:13.741 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:13.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:13.843 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:13.864 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:13.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:13.966 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:13.990 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:13.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:14.092 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:14.119 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:14.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:14.221 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:14.257 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:14.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:14.359 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:14.396 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:14.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:14.483 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:54:14.498 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:14.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.570 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:14.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:14.615 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 51 16 87618 87602 107.332 60.6875 0.0125202 0.00931275 2024-09-13T14:54:14.671 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:14.717 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:14.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:14.819 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:14.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.850 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:14.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:14.951 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:14.981 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:14.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:15.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:14 smithi096 ceph-mon[98901]: pgmap v847: 99 pgs: 99 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:15.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:14 smithi096 ceph-mon[102928]: pgmap v847: 99 pgs: 99 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:15.083 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:15.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.119 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:15.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:15.220 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:15.256 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:15.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:15.265 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:15.358 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:15.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.404 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:15.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:15.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:14 smithi204 ceph-mon[87071]: pgmap v847: 99 pgs: 99 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:15.505 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:15.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.543 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:15.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:15.615 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 52 16 88789 88773 106.675 73.1875 0.00831555 0.00936884 2024-09-13T14:54:15.644 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:15.672 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:15.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:15.774 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:15.820 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:15.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:15.922 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:15.953 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:15.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:16.055 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:16.093 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:16.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:16.195 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:16.230 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:16.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:16.332 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:16.357 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:16.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:16.459 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:16.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.492 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:16.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:16.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:16.616 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 53 16 89861 89845 105.926 67 0.0131824 0.00943576 2024-09-13T14:54:16.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.636 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:16.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:16.737 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:16.771 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:16.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:16.873 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:16.908 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:16.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:17.010 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:17.039 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:17.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:17.141 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:17.173 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:17.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:17.194 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:54:17.238 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:17 smithi096 ceph-mon[98901]: pgmap v848: 99 pgs: 99 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:17.238 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:17 smithi096 ceph-mon[102928]: pgmap v848: 99 pgs: 99 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:17.275 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:17.301 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:17.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:17.403 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:17.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.454 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:17.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:17.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:17 smithi204 ceph-mon[87071]: pgmap v848: 99 pgs: 99 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:17.555 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:17.587 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:17.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:17.618 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 54 16 90864 90848 105.122 62.6875 0.0111255 0.00950769 2024-09-13T14:54:17.689 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:17.738 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:17.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:17.840 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:17.882 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:17.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:17.983 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:18.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.014 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:18.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:18.114 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:18.149 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:18.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:18.169 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:54:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:54:17] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.51.0" 2024-09-13T14:54:18.252 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:18.307 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:18.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:18.409 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:18.412 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T14:54:18.412 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T14:54:18.412 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T14:54:18.412 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T14:54:18.412 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T14:54:18.412 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T14:54:18.412 INFO:teuthology.orchestra.run.smithi096.stderr:bench [] [] [] [] 2024-09-13T14:54:18.412 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:54:18.412 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T14:54:18.413 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T14:54:18.413 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore allocator dump block 2024-09-13T14:54:18.413 INFO:teuthology.orchestra.run.smithi096.stderr:Error EINVAL: invalid command 2024-09-13T14:54:18.435 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:18.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:18.538 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:18.570 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:18.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:18.618 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 55 16 91860 91844 104.343 62.25 0.01232 0.00957896 2024-09-13T14:54:18.672 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:18.714 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:18.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:18.816 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:18.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.851 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:18.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:18.952 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:18.986 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:18.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:19.088 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:19.120 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:19.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:19.222 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:19.253 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:19.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:19.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:19 smithi096 ceph-mon[98901]: pgmap v849: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T14:54:19.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:19 smithi096 ceph-mon[102928]: pgmap v849: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T14:54:19.354 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:19.388 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:19.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:19.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:19 smithi204 ceph-mon[87071]: pgmap v849: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T14:54:19.490 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:19.525 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:19.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:19.618 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 56 16 93027 93011 103.782 72.9375 0.0161211 0.00963102 2024-09-13T14:54:19.627 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:19.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.661 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:19.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:19.763 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:19.792 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:19.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:19.894 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:19.928 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:19.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:20.030 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:20.073 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:20.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:20.175 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:20.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.203 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:20.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:20.304 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:20.337 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:20.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:20.439 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:54:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:54:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:20.477 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:20.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:20.579 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:20.610 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:20.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:20.618 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 57 16 94127 94111 103.167 68.75 0.0157567 0.00968756 2024-09-13T14:54:20.712 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:20.750 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:20.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:20.852 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:20.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.896 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:20.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:20.997 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:21.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.037 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:21.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:21.138 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:21.170 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:21.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:21.254 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:21.259 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:54:21.260 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:54:21.260 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.7 'trim stale osdmaps' 2024-09-13T14:54:21.272 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:21.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.344 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:21.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:21.445 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:21.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:21 smithi204 ceph-mon[87071]: pgmap v850: 99 pgs: 99 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:54:21.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:21 smithi096 ceph-mon[98901]: pgmap v850: 99 pgs: 99 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:54:21.482 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:21.482 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:21 smithi096 ceph-mon[102928]: pgmap v850: 99 pgs: 99 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:54:21.483 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:21.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.489 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:21.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:21.590 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:21.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.623 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:21.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:21.623 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 58 16 95187 95171 102.531 66.25 0.00769656 0.0097481 2024-09-13T14:54:21.724 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:21.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.765 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:21.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:21.865 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:21.892 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:21.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:21.952 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:21.994 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:22.018 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:54:22.019 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:54:22.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.034 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:22.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:22.052 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:54:22.052 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:54:22.092 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:54:22.093 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:54:22.125 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:54:22.125 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:54:22.136 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:22.160 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:54:22.160 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:54:22.196 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:54:22.196 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:54:22.197 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:22.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:22.225 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:54:22.226 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:54:22.241 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:54:22.242 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:54:22.299 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:22.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:22 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1167841579' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:22 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1167841579' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:22.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.334 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:22.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:22.435 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:22.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.470 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:22.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:22.571 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:22.615 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:22.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:22.618 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 59 16 96223 96207 101.89 64.75 0.00431278 0.00980796 2024-09-13T14:54:22.718 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:22.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:22 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1167841579' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:22.741 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:22.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:22.842 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:22.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.865 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:22.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:22.966 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:23.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.000 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:23.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:23.101 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:23.148 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:23.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:23.250 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:23.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.296 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:23.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:23.397 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:23.453 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:23.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:23.555 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:23.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:23 smithi096 ceph-mon[98901]: pgmap v851: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-13T14:54:23.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:23 smithi096 ceph-mon[102928]: pgmap v851: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-13T14:54:23.588 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:23.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:23.618 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:54:23.618073+0000 min lat: 0.00118538 max lat: 0.273309 avg lat: 0.00988035 2024-09-13T14:54:23.618 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:54:23.618 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 60 16 97158 97142 101.166 58.4375 0.0127847 0.00988035 2024-09-13T14:54:23.690 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:23.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:23 smithi204 ceph-mon[87071]: pgmap v851: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-13T14:54:23.724 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:23.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:23.826 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:23.862 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:23.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:23.964 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:23.993 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:23.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:24.095 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:24.132 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:24.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:24.234 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:24.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.279 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:24.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:24.380 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:24.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.415 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:24.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:24.517 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:24.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.560 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:24.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:24.619 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 61 16 98297 98281 100.673 71.1875 0.00629851 0.00992741 2024-09-13T14:54:24.662 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:24.706 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:24.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:24.808 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:24.844 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:24.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:24.946 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:24.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.986 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:24.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:25.086 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:25.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.133 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:25.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:25.234 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:25.285 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:25.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:25.387 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:25.430 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:25.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:25.533 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:25.560 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:25 smithi096 ceph-mon[98901]: pgmap v852: 99 pgs: 99 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-13T14:54:25.561 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:25 smithi096 ceph-mon[102928]: pgmap v852: 99 pgs: 99 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-13T14:54:25.569 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:25.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:25.622 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 62 16 99366 99350 100.123 66.8125 0.00777902 0.0099818 2024-09-13T14:54:25.635 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:25.671 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:25.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:25 smithi204 ceph-mon[87071]: pgmap v852: 99 pgs: 99 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-13T14:54:25.724 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:25.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:25.827 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:25.871 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:25.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:25.973 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:26.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.013 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:26.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:26.114 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:26.157 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:26.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:26.158 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:54:26.258 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:26.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.311 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:26.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:26.412 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:26.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.440 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:26.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:26.541 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:26.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.584 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:26.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:26.622 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 63 16 100306 100290 99.4665 58.75 0.00927701 0.0100489 2024-09-13T14:54:26.685 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:26.720 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:26.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:26.822 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:26.864 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:26.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:26.966 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:27.009 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:27.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:27.111 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:27.114 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:54:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:54:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:54:27.114 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:54:27.114 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:54:27.142 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:27.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:27.244 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:27.254 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:54:27.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.285 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:27.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:27.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:27 smithi096 ceph-mon[98901]: pgmap v853: 99 pgs: 99 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2024-09-13T14:54:27.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:27 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3893405152' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:27.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:27 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:27 smithi096 ceph-mon[102928]: pgmap v853: 99 pgs: 99 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2024-09-13T14:54:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:27 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3893405152' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:27 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:27.387 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:27.421 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:27.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:27.523 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:27.550 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:27.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:27.623 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 64 16 101276 101260 98.8576 60.625 0.0252834 0.0101107 2024-09-13T14:54:27.653 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:27.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.694 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:27.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:27.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:27 smithi204 ceph-mon[87071]: pgmap v853: 99 pgs: 99 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2024-09-13T14:54:27.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:27 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3893405152' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:27.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:27 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:27.795 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:27.833 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:27.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:27.934 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:27.970 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:27.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:28.072 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:28.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:54:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:54:27] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.51.0" 2024-09-13T14:54:28.104 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:28.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:28.206 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:28.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.244 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:28.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:28.345 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:28.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.371 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:28.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:28.472 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:28.505 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:28.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:28.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:28 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:28.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:28 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:54:28.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:28 smithi096 ceph-mon[98901]: osdmap e278: 8 total, 8 up, 8 in 2024-09-13T14:54:28.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:28 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:28.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:28 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:54:28.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:28 smithi096 ceph-mon[102928]: osdmap e278: 8 total, 8 up, 8 in 2024-09-13T14:54:28.607 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:28.623 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 65 16 102286 102270 98.3079 63.125 0.0293578 0.0101662 2024-09-13T14:54:28.629 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:28.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:28.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:28 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:28.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:28 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:54:28.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:28 smithi204 ceph-mon[87071]: osdmap e278: 8 total, 8 up, 8 in 2024-09-13T14:54:28.731 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:28.766 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:28.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:28.868 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:28.896 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:28.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:28.998 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:29.029 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:29.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:29.131 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:29.172 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:29.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:29.274 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:29.301 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:29.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:29.403 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:29.440 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:29.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:29.542 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:29.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:29 smithi096 ceph-mon[98901]: pgmap v855: 99 pgs: 99 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:54:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:29 smithi096 ceph-mon[102928]: pgmap v855: 99 pgs: 99 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:54:29.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.575 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:29.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:29.623 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 66 16 103336 103320 97.8127 65.625 0.0180476 0.010219 2024-09-13T14:54:29.676 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:29.708 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:29.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:29.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:29 smithi204 ceph-mon[87071]: pgmap v855: 99 pgs: 99 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:54:29.810 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:29.843 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:29.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:29.945 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:29.977 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:29.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:30.079 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:30.106 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:30.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:30.208 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:30.244 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:30.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:30.346 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:30.388 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:30.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:54:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:54:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:30.489 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:30.507 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:54:30.516 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:30.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:30.617 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:30.623 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 67 16 104399 104383 97.3444 66.4375 0.00762794 0.0102674 2024-09-13T14:54:30.646 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:30.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:30.748 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:30.803 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:30.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:30.905 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:30.954 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:30.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:31.056 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:31.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.089 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:31.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:31.190 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:31.233 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:31.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:31.335 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:31.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.384 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:31.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:31.485 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:31.526 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:31.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:31.560 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:31 smithi204 ceph-mon[87071]: pgmap v856: 99 pgs: 99 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:31.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:31 smithi096 ceph-mon[98901]: pgmap v856: 99 pgs: 99 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:31.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:31 smithi096 ceph-mon[102928]: pgmap v856: 99 pgs: 99 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:31.623 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 68 16 105408 105392 96.8403 63.0625 0.0146909 0.0103218 2024-09-13T14:54:31.628 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:31.671 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:31.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:31.773 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:31.812 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:31.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:31.913 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:31.923 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:31.942 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:31.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:32.044 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:32.076 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:32.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:32.178 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:32.207 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:32.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:32.293 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T14:54:32.293 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T14:54:32.293 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T14:54:32.293 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T14:54:32.293 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T14:54:32.293 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T14:54:32.293 INFO:teuthology.orchestra.run.smithi096.stderr:bench [] [] [] [] 2024-09-13T14:54:32.293 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:54:32.293 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T14:54:32.294 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T14:54:32.294 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore allocator dump block 2024-09-13T14:54:32.294 INFO:teuthology.orchestra.run.smithi096.stderr:Error EINVAL: invalid command 2024-09-13T14:54:32.308 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:32.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.348 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:32.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:32.449 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:32.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.480 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:32.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:32.581 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:32.613 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:32.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:32.624 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 69 16 106399 106383 96.3344 61.9375 0.0207797 0.0103761 2024-09-13T14:54:32.715 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:32.758 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:32.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:32.860 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:32.889 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:32.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:32.991 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:33.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.033 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:33.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:33.134 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:33.172 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:33.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:33.274 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:33.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.304 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:33.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:33.404 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:33.437 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:33.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:33.539 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:33.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:33 smithi096 ceph-mon[98901]: pgmap v857: 99 pgs: 99 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:54:33.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:33 smithi096 ceph-mon[102928]: pgmap v857: 99 pgs: 99 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:54:33.579 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:33.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:33.624 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 70 16 107447 107431 95.8939 65.5 0.00970244 0.0104235 2024-09-13T14:54:33.685 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:33.712 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:33.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:33.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:33 smithi204 ceph-mon[87071]: pgmap v857: 99 pgs: 99 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:54:33.727 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:54:33.814 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:33.839 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:33.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:33.941 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:33.965 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:33.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:34.066 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:34.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.104 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:34.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:34.205 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:34.237 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:34.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:34.339 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:34.374 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:34.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:34.476 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:34.510 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:34.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:34.612 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:34.624 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 71 16 108373 108357 95.3583 57.875 0.00824358 0.0104776 2024-09-13T14:54:34.633 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:34.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:34.734 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:34.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.769 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:34.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:34.870 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:34.908 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:34.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:35.009 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:35.043 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:35.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:35.146 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:35.177 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:35.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:35.279 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:35.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:35 smithi096 ceph-mon[98901]: pgmap v858: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:54:35.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:35 smithi096 ceph-mon[102928]: pgmap v858: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:54:35.330 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:35.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:35.432 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:35.472 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:35.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:35.541 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:54:35.542 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:54:35.542 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.6 'trim stale osdmaps' 2024-09-13T14:54:35.574 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:35.598 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:35.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:35.626 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 72 16 109324 109308 94.8561 59.4375 0.0235299 0.0105366 2024-09-13T14:54:35.700 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:35.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:35 smithi204 ceph-mon[87071]: pgmap v858: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:54:35.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.733 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:35.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:35.834 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:35.864 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:35.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:35.966 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:36.002 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:36.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:36.105 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:36.132 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:36.353 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:36.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.356 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:36.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:36.457 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:36.507 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:36.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:36.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:36.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:36.610 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:36.628 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 73 16 110259 110243 94.3562 58.4375 0.00986063 0.0105931 2024-09-13T14:54:36.661 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:36.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:36.763 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:36.803 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:36.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:36.905 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:36.946 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:36.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:37.047 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:37.082 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:37.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:37.183 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:37.211 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:37.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:37.313 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:37.349 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:37.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:37.451 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:37.469 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:54:37.469 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:54:37.470 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:37 smithi096 ceph-mon[98901]: pgmap v859: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:54:37.470 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:37 smithi096 ceph-mon[102928]: pgmap v859: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:54:37.479 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:37.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:37.506 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:54:37.507 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:54:37.543 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:54:37.544 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:54:37.568 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:54:37.568 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:54:37.584 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:37.597 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:54:37.597 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:54:37.609 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:37.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:37.626 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:54:37.627 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:54:37.627 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 74 16 111226 111210 93.8979 60.4375 0.0268718 0.0106445 2024-09-13T14:54:37.658 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:54:37.659 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:54:37.697 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:54:37.698 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:54:37.711 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:37.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:37 smithi204 ceph-mon[87071]: pgmap v859: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:54:37.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.754 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:37.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:37.855 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:37.880 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:37.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:37.982 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:38.022 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:38.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:38.124 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:38.157 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:38.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:38.201 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:54:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:54:37] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.51.0" 2024-09-13T14:54:38.259 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:38.296 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:38.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:38.398 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:38.425 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:38.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:38.526 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:38.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:38 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3303903080' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:38.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:38 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3303903080' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:38.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.576 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:38.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:38.628 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 75 16 112199 112183 93.4567 60.8125 0.00794359 0.0106956 2024-09-13T14:54:38.677 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:38.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.711 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:38.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:38.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:38 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3303903080' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:38.813 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:38.845 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:38.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:38.947 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:38.982 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:38.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:39.084 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:39.119 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:39.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:39.221 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:39.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.258 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:39.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:39.359 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:39.395 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:39.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:39.497 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:39.532 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:39.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:39.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:39 smithi096 ceph-mon[98901]: pgmap v860: 99 pgs: 99 active+clean; 6.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s 2024-09-13T14:54:39.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:39 smithi096 ceph-mon[102928]: pgmap v860: 99 pgs: 99 active+clean; 6.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s 2024-09-13T14:54:39.630 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 76 16 113236 113220 93.0774 64.8125 0.0328363 0.0107385 2024-09-13T14:54:39.634 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:39.660 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:39.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:39.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:39 smithi204 ceph-mon[87071]: pgmap v860: 99 pgs: 99 active+clean; 6.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s 2024-09-13T14:54:39.762 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:39.791 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:39.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:39.893 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:39.922 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:39.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:40.024 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:40.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.061 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:40.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:40.162 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:40.186 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:40.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:40.287 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:40.320 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:40.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:40.422 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:40.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.462 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:40.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:40.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:54:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:54:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:40.563 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:40.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.595 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:40.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:40.632 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 77 16 114109 114093 92.5773 54.5625 0.0134329 0.0107951 2024-09-13T14:54:40.633 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:40.697 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:40.739 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:40.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:40.841 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:40.872 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:40.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:40.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:40 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:40.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:40 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:40.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:40 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:54:40.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:40 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:54:40.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:40 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:40.973 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:41.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.010 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:41.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:41.037 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:40 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:41.037 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:40 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:41.037 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:40 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:54:41.038 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:40 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:54:41.038 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:40 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:41.038 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:40 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:41.038 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:40 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:41.038 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:40 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:54:41.038 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:40 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:54:41.038 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:40 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:41.111 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:41.137 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:41.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:41.239 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:41.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.285 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:41.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:41.386 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:41.414 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:41.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:41.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:41.550 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:41.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:41.630 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 78 16 115081 115065 92.1693 60.75 0.0125312 0.0108446 2024-09-13T14:54:41.653 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:41.687 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:41.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:41.789 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:41.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.826 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:41.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:41.927 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:41.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:41 smithi096 ceph-mon[98901]: pgmap v861: 99 pgs: 99 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:54:41.946 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:41 smithi096 ceph-mon[102928]: pgmap v861: 99 pgs: 99 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:54:41.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.970 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:41.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:41.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:41 smithi204 ceph-mon[87071]: pgmap v861: 99 pgs: 99 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:54:42.071 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:42.105 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:42.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:42.207 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:42.249 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:54:42.249 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:54:42.251 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:42.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:42.353 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:42.392 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:42.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:42.494 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:42.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.529 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:42.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:42.630 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:42.631 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 79 16 116047 116031 91.7669 60.375 0.0107013 0.0108921 2024-09-13T14:54:42.634 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:54:42.661 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:42.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:42.763 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:42.792 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:42.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:42.894 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:42.935 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:42.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:43.037 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:43.052 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:54:43.060 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:43.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:43.162 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:43.196 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:43.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:43.298 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:43.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.335 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:43.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:43.407 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:54:43.436 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:43.483 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:43.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:43.585 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:43.607 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:43.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:43.630 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:54:43.629961+0000 min lat: 0.00118538 max lat: 0.273309 avg lat: 0.0109492 2024-09-13T14:54:43.630 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:54:43.630 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 80 16 116896 116880 91.2831 53.0625 0.0104526 0.0109492 2024-09-13T14:54:43.709 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:43.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.746 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:43.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:43.805 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:43 smithi204 ceph-mon[87071]: pgmap v862: 99 pgs: 99 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s 2024-09-13T14:54:43.847 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:43.868 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:43.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:43.970 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:43.991 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:43.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:44.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:43 smithi096 ceph-mon[98901]: pgmap v862: 99 pgs: 99 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s 2024-09-13T14:54:44.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:43 smithi096 ceph-mon[102928]: pgmap v862: 99 pgs: 99 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s 2024-09-13T14:54:44.093 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:44.131 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:44.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:44.233 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:44.268 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:44.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:44.370 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:44.391 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:44.494 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:44.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.529 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:44.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:44.630 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 81 16 117630 117614 90.7226 45.875 0.0174186 0.011018 2024-09-13T14:54:44.630 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:44.667 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:44.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:44.769 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:44.802 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:44.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:44.903 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:44.930 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:44.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:45.032 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:45.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.066 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:45.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:45.167 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:45.201 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:45.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:45.303 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:45.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.343 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:45.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:45.444 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:45.478 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:45.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:45.580 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:45.615 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:45.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:45.630 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 82 16 118608 118592 90.3617 61.125 0.00995121 0.011061 2024-09-13T14:54:45.717 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:45.771 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:45.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:45.808 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:45.815 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:45 smithi204 ceph-mon[87071]: pgmap v863: 99 pgs: 99 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:45.822 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:54:45.822 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:54:45.873 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:45.910 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:45.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:45.955 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:45 smithi096 ceph-mon[98901]: pgmap v863: 99 pgs: 99 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:45.955 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:45 smithi096 ceph-mon[102928]: pgmap v863: 99 pgs: 99 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:46.013 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:46.045 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:46.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:46.148 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:46.180 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:46.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:46.282 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:46.319 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:46.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:46.421 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:46.457 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:46.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:46.503 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:54:46.559 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:46.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.593 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:46.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:46.630 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 83 16 119459 119443 89.9138 53.1875 0.0099977 0.0111168 2024-09-13T14:54:46.694 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:46.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.746 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:46.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:46.847 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:46.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.890 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:46.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:46.928 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:54:46.991 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:47.036 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:47.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:47.138 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:47.177 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:47.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:47.279 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:47.320 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:47.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:47.422 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:47.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.459 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:47.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:47.561 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:47.600 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T14:54:47.600 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T14:54:47.601 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T14:54:47.601 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T14:54:47.601 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T14:54:47.601 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T14:54:47.601 INFO:teuthology.orchestra.run.smithi096.stderr:bench [] [] [] [] 2024-09-13T14:54:47.601 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:54:47.601 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T14:54:47.601 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T14:54:47.601 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore allocator dump block 2024-09-13T14:54:47.601 INFO:teuthology.orchestra.run.smithi096.stderr:Error EINVAL: invalid command 2024-09-13T14:54:47.605 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:47.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:47.630 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 84 16 120236 120220 89.4216 48.5625 0.0243816 0.011178 2024-09-13T14:54:47.709 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:47.747 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:47.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:47.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:47 smithi096 ceph-mon[98901]: pgmap v864: 99 pgs: 99 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s 2024-09-13T14:54:47.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:47 smithi096 ceph-mon[102928]: pgmap v864: 99 pgs: 99 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s 2024-09-13T14:54:47.849 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:47.886 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:47.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:47.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:47 smithi204 ceph-mon[87071]: pgmap v864: 99 pgs: 99 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s 2024-09-13T14:54:47.988 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:48.018 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:48.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:48.120 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:48.161 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:48.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:48.250 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:54:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:54:47] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.51.0" 2024-09-13T14:54:48.262 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:48.304 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:48.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:48.406 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:48.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.447 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:48.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:48.549 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:48.581 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:48.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:48.633 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 85 16 121094 121078 89.0005 53.625 0.00628166 0.0112298 2024-09-13T14:54:48.683 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:48.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.711 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:48.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:48.812 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:48.825 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:54:48.825 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:54:48.850 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:48.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:48.953 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:48.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.992 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:48.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:49.093 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:49.115 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:49.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:49.216 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:49.256 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:49.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:49.358 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:49.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.393 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:49.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:49.494 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:49.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.537 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:49.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:49.631 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 86 16 121860 121844 88.5223 47.875 0.0051049 0.0112892 2024-09-13T14:54:49.638 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:49.677 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:49.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:49.690 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:54:49.725 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:54:49.778 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:49.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.826 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:49.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:49.927 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:49.960 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:49.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:49.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:49 smithi204 ceph-mon[87071]: pgmap v865: 99 pgs: 99 active+clean; 7.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2024-09-13T14:54:49.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:49 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1703615705' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:50.065 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:50.068 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:54:50.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:49 smithi096 ceph-mon[98901]: pgmap v865: 99 pgs: 99 active+clean; 7.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2024-09-13T14:54:50.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:49 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1703615705' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:50.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:49 smithi096 ceph-mon[102928]: pgmap v865: 99 pgs: 99 active+clean; 7.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2024-09-13T14:54:50.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:49 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1703615705' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:50.097 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:50.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:50.199 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:50.238 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:50.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:50.340 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:50.377 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:50.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:54:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:54:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:50.480 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:50.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.513 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:50.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:50.614 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:50.631 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 87 16 122749 122733 88.1434 55.5625 0.0204629 0.01134 2024-09-13T14:54:50.641 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:50.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:50.735 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:54:50.736 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:54:50.742 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:50.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.771 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:50.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:50.774 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:54:50.775 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:54:50.814 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:54:50.814 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:54:50.847 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:54:50.847 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:54:50.872 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:50.881 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:50 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:54:50.881 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:50 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:54:50.881 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:50 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1703615705' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:50.881 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:50 smithi204 ceph-mon[87071]: osdmap e279: 8 total, 8 up, 8 in 2024-09-13T14:54:50.881 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:50 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3008158204' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:50.882 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:54:50.882 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:54:50.906 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:50.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:50.911 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:54:50.911 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:54:50.939 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:54:50.939 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:54:50.971 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:54:50.971 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:54:50.972 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:50 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:54:50.972 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:50 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:54:50.972 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:50 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1703615705' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:50.972 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:50 smithi096 ceph-mon[98901]: osdmap e279: 8 total, 8 up, 8 in 2024-09-13T14:54:50.972 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:50 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3008158204' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:50.972 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:50 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:54:50.972 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:50 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:54:50.972 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:50 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1703615705' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:50.972 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:50 smithi096 ceph-mon[102928]: osdmap e279: 8 total, 8 up, 8 in 2024-09-13T14:54:50.972 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:50 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3008158204' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:51.007 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:51.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.039 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:51.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:51.140 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:51.167 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:51.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:51.269 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:51.302 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:51.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:51.404 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:51.445 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:51.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:51.450 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:54:51.450 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:54:51.547 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:51.584 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:51.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:51.631 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 88 16 123635 123619 87.7711 55.375 0.0186355 0.0113887 2024-09-13T14:54:51.686 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:51.718 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:51.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:51.820 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:51.831 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:51 smithi204 ceph-mon[87071]: pgmap v867: 99 pgs: 99 active+clean; 7.4 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2024-09-13T14:54:51.832 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:51.832 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:51.876 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:51.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:51.978 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:52.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.020 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:52.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:52.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:51 smithi096 ceph-mon[98901]: pgmap v867: 99 pgs: 99 active+clean; 7.4 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2024-09-13T14:54:52.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:52.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:51 smithi096 ceph-mon[102928]: pgmap v867: 99 pgs: 99 active+clean; 7.4 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2024-09-13T14:54:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:54:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:52.105 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:54:52.105 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T14:54:52.105 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T14:54:52.105 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:54:52.105 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.106 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:136882 6.2 6:4d112999:::benchmark_data_smithi096_167087_object19654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-09-13T14:54:52.106 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:02.049745+0000", 2024-09-13T14:54:52.106 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 530.05228831500006, 2024-09-13T14:54:52.106 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.150460636, 2024-09-13T14:54:52.106 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.106 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:54:52.106 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.106 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:54:52.106 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:54:52.106 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 136882 2024-09-13T14:54:52.106 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.106 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.106 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.107 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.107 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049745+0000", 2024-09-13T14:54:52.107 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.107 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.107 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.107 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.107 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049745+0000", 2024-09-13T14:54:52.107 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.107 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.107 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.107 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.107 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049744+0000", 2024-09-13T14:54:52.107 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:52.108 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.108 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.108 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.108 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049749+0000", 2024-09-13T14:54:52.108 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8510000000000004e-06 2024-09-13T14:54:52.108 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.108 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.108 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.108 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049763+0000", 2024-09-13T14:54:52.108 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3518e-05 2024-09-13T14:54:52.108 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.108 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.108 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.108 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049769+0000", 2024-09-13T14:54:52.109 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0009999999999998e-06 2024-09-13T14:54:52.109 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.109 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.109 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.109 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049809+0000", 2024-09-13T14:54:52.109 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0071000000000003e-05 2024-09-13T14:54:52.109 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.109 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.109 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.109 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.049959+0000", 2024-09-13T14:54:52.109 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015019400000000001 2024-09-13T14:54:52.109 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.109 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.110 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:54:52.110 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.050081+0000", 2024-09-13T14:54:52.110 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000121418 2024-09-13T14:54:52.110 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.110 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.110 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.110 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.053462+0000", 2024-09-13T14:54:52.110 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0033818400000000001 2024-09-13T14:54:52.110 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.110 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.110 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.110 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068368+0000", 2024-09-13T14:54:52.110 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.014905295000000001 2024-09-13T14:54:52.111 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.111 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.111 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.111 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.081052+0000", 2024-09-13T14:54:52.111 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012684012999999999 2024-09-13T14:54:52.111 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.111 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.111 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.111 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.081061+0000", 2024-09-13T14:54:52.111 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8999999999999995e-06 2024-09-13T14:54:52.111 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.111 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.111 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.111 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082907+0000", 2024-09-13T14:54:52.112 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0018459310000001 2024-09-13T14:54:52.112 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.112 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.112 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.112 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082914+0000", 2024-09-13T14:54:52.112 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8520000000000004e-06 2024-09-13T14:54:52.112 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.112 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.112 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.112 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.200133+0000", 2024-09-13T14:54:52.112 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.117218641 2024-09-13T14:54:52.112 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.112 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.112 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.113 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.200205+0000", 2024-09-13T14:54:52.113 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2381999999999999e-05 2024-09-13T14:54:52.113 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.113 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.113 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.113 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.113 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.113 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:136895 6.2 6:487a132d:::benchmark_data_smithi096_167087_object19667:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-09-13T14:54:52.113 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:02.068487+0000", 2024-09-13T14:54:52.113 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 530.03354625199995, 2024-09-13T14:54:52.113 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1320098289999998, 2024-09-13T14:54:52.114 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.114 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:54:52.114 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.114 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:54:52.114 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:54:52.114 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 136895 2024-09-13T14:54:52.114 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.114 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.114 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.114 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.114 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068487+0000", 2024-09-13T14:54:52.114 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.115 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.115 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.115 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.115 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068487+0000", 2024-09-13T14:54:52.115 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.115 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.115 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.115 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.115 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068485+0000", 2024-09-13T14:54:52.115 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:54:52.115 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.115 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.115 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.115 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068492+0000", 2024-09-13T14:54:52.116 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1040000000000001e-06 2024-09-13T14:54:52.116 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.116 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.116 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.116 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068507+0000", 2024-09-13T14:54:52.116 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4707e-05 2024-09-13T14:54:52.116 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.116 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.116 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.116 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068512+0000", 2024-09-13T14:54:52.116 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7830000000000004e-06 2024-09-13T14:54:52.116 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.116 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.117 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.117 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068560+0000", 2024-09-13T14:54:52.117 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7141000000000001e-05 2024-09-13T14:54:52.117 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.117 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.117 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.117 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.068680+0000", 2024-09-13T14:54:52.117 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000120183 2024-09-13T14:54:52.117 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.117 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.117 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:54:52.117 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.069244+0000", 2024-09-13T14:54:52.117 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00056373199999999995 2024-09-13T14:54:52.118 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.118 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.118 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.118 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.072489+0000", 2024-09-13T14:54:52.118 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0032459580000000002 2024-09-13T14:54:52.118 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.118 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.118 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.118 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.081086+0000", 2024-09-13T14:54:52.118 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0085960910000000001 2024-09-13T14:54:52.119 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.119 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.119 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.119 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.081092+0000", 2024-09-13T14:54:52.119 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7970000000000003e-06 2024-09-13T14:54:52.119 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.119 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.119 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.119 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082944+0000", 2024-09-13T14:54:52.119 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.001852076 2024-09-13T14:54:52.119 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.119 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.119 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.119 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082952+0000", 2024-09-13T14:54:52.120 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2660000000000004e-06 2024-09-13T14:54:52.120 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.120 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.120 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.120 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.200444+0000", 2024-09-13T14:54:52.121 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.117492039 2024-09-13T14:54:52.121 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.121 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.121 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.121 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.200497+0000", 2024-09-13T14:54:52.121 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2961999999999999e-05 2024-09-13T14:54:52.121 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.121 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.121 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.121 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.121 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.122 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:136900 6.2 6:4c459d5b:::benchmark_data_smithi096_167087_object19672:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-09-13T14:54:52.122 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:02.077300+0000", 2024-09-13T14:54:52.122 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 530.02473298500001, 2024-09-13T14:54:52.122 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1405926819999999, 2024-09-13T14:54:52.122 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.122 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.122 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.122 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:54:52.122 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:54:52.122 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 136900 2024-09-13T14:54:52.122 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.122 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.122 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.123 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.123 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.077300+0000", 2024-09-13T14:54:52.123 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.123 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.123 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.123 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.123 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.077300+0000", 2024-09-13T14:54:52.123 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.123 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.123 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.123 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.123 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.077298+0000", 2024-09-13T14:54:52.123 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:54:52.123 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.124 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.124 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.124 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.077305+0000", 2024-09-13T14:54:52.124 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9770000000000003e-06 2024-09-13T14:54:52.124 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.124 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.124 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.124 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.077317+0000", 2024-09-13T14:54:52.124 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.155e-05 2024-09-13T14:54:52.124 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.124 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.124 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.124 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.077323+0000", 2024-09-13T14:54:52.125 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9449999999999998e-06 2024-09-13T14:54:52.125 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.125 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.125 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.125 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.083184+0000", 2024-09-13T14:54:52.125 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0058613329999999998 2024-09-13T14:54:52.125 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.125 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.125 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.125 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.083193+0000", 2024-09-13T14:54:52.125 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1379999999999996e-06 2024-09-13T14:54:52.125 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.125 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.125 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.126 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082972+0000", 2024-09-13T14:54:52.126 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99977896499999996 2024-09-13T14:54:52.126 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.126 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.126 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.126 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082979+0000", 2024-09-13T14:54:52.126 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5819999999999998e-06 2024-09-13T14:54:52.126 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.126 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.126 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.126 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.200530+0000", 2024-09-13T14:54:52.126 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1175511789999999 2024-09-13T14:54:52.126 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.126 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.127 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.127 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.200642+0000", 2024-09-13T14:54:52.127 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000112376 2024-09-13T14:54:52.127 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.127 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.127 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T14:54:52.127 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.201279+0000", 2024-09-13T14:54:52.127 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00063697600000000004 2024-09-13T14:54:52.127 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.127 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.127 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.127 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.209711+0000", 2024-09-13T14:54:52.127 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0084313359999999993 2024-09-13T14:54:52.128 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.128 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.128 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.128 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.209770+0000", 2024-09-13T14:54:52.128 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8968000000000002e-05 2024-09-13T14:54:52.128 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.128 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.128 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.128 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.212282+0000", 2024-09-13T14:54:52.128 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0025126110000000001 2024-09-13T14:54:52.128 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.128 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.128 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.128 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.217709+0000", 2024-09-13T14:54:52.129 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.005426859 2024-09-13T14:54:52.129 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.129 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.129 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.129 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.217766+0000", 2024-09-13T14:54:52.129 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7182000000000002e-05 2024-09-13T14:54:52.129 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.129 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.129 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.129 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.217893+0000", 2024-09-13T14:54:52.129 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012656200000000001 2024-09-13T14:54:52.129 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.129 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.129 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.130 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.130 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.130 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:136908 6.e 6:740111d7:::benchmark_data_smithi096_167087_object19680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:54:52.130 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:02.112700+0000", 2024-09-13T14:54:52.130 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 529.98933340200006, 2024-09-13T14:54:52.130 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1197194330000002, 2024-09-13T14:54:52.130 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.130 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.130 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.130 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:54:52.130 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:54:52.130 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 136908 2024-09-13T14:54:52.130 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.130 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.131 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.131 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.131 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112700+0000", 2024-09-13T14:54:52.131 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.131 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.131 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.131 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.131 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112700+0000", 2024-09-13T14:54:52.131 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.131 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.131 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.131 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.131 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112699+0000", 2024-09-13T14:54:52.132 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:52.132 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.132 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.132 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.132 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112703+0000", 2024-09-13T14:54:52.132 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9089999999999999e-06 2024-09-13T14:54:52.132 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.132 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.132 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.132 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112714+0000", 2024-09-13T14:54:52.132 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1071e-05 2024-09-13T14:54:52.132 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.132 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.132 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.133 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112720+0000", 2024-09-13T14:54:52.133 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5470000000000004e-06 2024-09-13T14:54:52.133 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.133 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.133 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.133 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112782+0000", 2024-09-13T14:54:52.133 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2302000000000001e-05 2024-09-13T14:54:52.133 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.133 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.133 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.133 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.112796+0000", 2024-09-13T14:54:52.133 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3495999999999999e-05 2024-09-13T14:54:52.133 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.134 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.134 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.134 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082520+0000", 2024-09-13T14:54:52.134 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.96972424199999996 2024-09-13T14:54:52.134 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.134 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.134 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.134 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.082548+0000", 2024-09-13T14:54:52.134 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8402999999999999e-05 2024-09-13T14:54:52.134 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.134 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.134 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.134 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.218682+0000", 2024-09-13T14:54:52.134 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.136133767 2024-09-13T14:54:52.134 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.135 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.135 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.135 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.218802+0000", 2024-09-13T14:54:52.135 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119988 2024-09-13T14:54:52.135 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.135 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.135 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:54:52.135 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.219302+0000", 2024-09-13T14:54:52.135 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00050009499999999999 2024-09-13T14:54:52.135 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.135 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.135 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.135 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.224040+0000", 2024-09-13T14:54:52.135 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0047376420000000002 2024-09-13T14:54:52.136 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.136 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.136 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.136 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.229213+0000", 2024-09-13T14:54:52.136 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0051726039999999999 2024-09-13T14:54:52.136 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.136 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.136 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.136 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.231384+0000", 2024-09-13T14:54:52.136 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0021715990000000002 2024-09-13T14:54:52.136 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.136 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.137 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.137 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.232326+0000", 2024-09-13T14:54:52.137 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00094199799999999999 2024-09-13T14:54:52.137 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.137 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.137 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.137 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.232360+0000", 2024-09-13T14:54:52.137 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3411999999999998e-05 2024-09-13T14:54:52.137 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.137 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.137 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.137 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.232419+0000", 2024-09-13T14:54:52.137 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9555999999999997e-05 2024-09-13T14:54:52.137 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.138 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.138 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.138 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.138 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.138 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:136915 6.2 6:4a34972c:::benchmark_data_smithi096_167087_object19687:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:54:52.138 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:02.127704+0000", 2024-09-13T14:54:52.138 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 529.97432882800001, 2024-09-13T14:54:52.138 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.090359758, 2024-09-13T14:54:52.138 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.138 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.138 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.138 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:54:52.138 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:54:52.139 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 136915 2024-09-13T14:54:52.139 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.139 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.139 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.139 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.139 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127704+0000", 2024-09-13T14:54:52.139 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.139 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.139 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.139 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.139 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127704+0000", 2024-09-13T14:54:52.139 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.139 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.139 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.140 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.140 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127703+0000", 2024-09-13T14:54:52.140 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:52.140 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.140 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.140 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.140 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127708+0000", 2024-09-13T14:54:52.140 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3990000000000003e-06 2024-09-13T14:54:52.140 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.140 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.140 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.140 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127719+0000", 2024-09-13T14:54:52.140 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.065e-05 2024-09-13T14:54:52.141 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.141 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.141 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.141 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127726+0000", 2024-09-13T14:54:52.141 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8789999999999997e-06 2024-09-13T14:54:52.141 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.141 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.141 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.141 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127847+0000", 2024-09-13T14:54:52.141 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012154099999999999 2024-09-13T14:54:52.141 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.141 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.141 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.142 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.127854+0000", 2024-09-13T14:54:52.142 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1570000000000003e-06 2024-09-13T14:54:52.142 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.142 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.142 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.142 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.083001+0000", 2024-09-13T14:54:52.142 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95514685899999996 2024-09-13T14:54:52.142 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.142 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.142 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.142 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.083008+0000", 2024-09-13T14:54:52.142 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4230000000000002e-06 2024-09-13T14:54:52.142 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.142 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.142 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.143 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.206475+0000", 2024-09-13T14:54:52.143 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.123467217 2024-09-13T14:54:52.143 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.143 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.143 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.143 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.206608+0000", 2024-09-13T14:54:52.143 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013340800000000001 2024-09-13T14:54:52.143 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.143 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.143 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T14:54:52.143 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.206693+0000", 2024-09-13T14:54:52.143 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4627000000000001e-05 2024-09-13T14:54:52.143 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.143 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.144 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.144 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.209759+0000", 2024-09-13T14:54:52.144 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00306646 2024-09-13T14:54:52.144 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.144 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.144 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.144 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.209777+0000", 2024-09-13T14:54:52.144 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7135000000000001e-05 2024-09-13T14:54:52.144 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.144 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.144 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.144 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.216584+0000", 2024-09-13T14:54:52.144 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0068075100000000001 2024-09-13T14:54:52.144 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.145 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.145 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.145 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.217946+0000", 2024-09-13T14:54:52.145 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001361791 2024-09-13T14:54:52.145 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.145 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.145 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.145 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.217981+0000", 2024-09-13T14:54:52.145 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4632999999999999e-05 2024-09-13T14:54:52.145 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.145 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.145 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.145 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.218064+0000", 2024-09-13T14:54:52.145 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3517999999999996e-05 2024-09-13T14:54:52.146 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.146 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.146 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.146 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.146 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.146 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:136925 6.2 6:47d3f3a2:::benchmark_data_smithi096_167087_object19697:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:54:52.146 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:02.144759+0000", 2024-09-13T14:54:52.146 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 529.957274568, 2024-09-13T14:54:52.146 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0783393989999999, 2024-09-13T14:54:52.146 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.146 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.146 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.146 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:54:52.146 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:54:52.147 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 136925 2024-09-13T14:54:52.147 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.147 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.147 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.147 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.147 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144759+0000", 2024-09-13T14:54:52.147 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.147 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.147 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.147 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.147 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144759+0000", 2024-09-13T14:54:52.147 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.147 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.148 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.148 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.148 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144757+0000", 2024-09-13T14:54:52.148 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:52.148 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.148 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.148 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.148 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144762+0000", 2024-09-13T14:54:52.148 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1549999999999998e-06 2024-09-13T14:54:52.148 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.148 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.148 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.148 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144771+0000", 2024-09-13T14:54:52.148 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1849999999999993e-06 2024-09-13T14:54:52.149 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.149 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.149 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.149 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144776+0000", 2024-09-13T14:54:52.149 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7419999999999997e-06 2024-09-13T14:54:52.149 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.149 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.149 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.149 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144820+0000", 2024-09-13T14:54:52.149 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4200999999999999e-05 2024-09-13T14:54:52.149 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.149 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.149 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.150 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:02.144828+0000", 2024-09-13T14:54:52.150 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5989999999999996e-06 2024-09-13T14:54:52.150 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.150 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.150 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.150 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.083036+0000", 2024-09-13T14:54:52.150 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93820851599999999 2024-09-13T14:54:52.150 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.150 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.150 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.150 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:03.083043+0000", 2024-09-13T14:54:52.150 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6390000000000003e-06 2024-09-13T14:54:52.150 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.150 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.151 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.151 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.209811+0000", 2024-09-13T14:54:52.151 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1267674599999999 2024-09-13T14:54:52.151 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.151 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.151 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.151 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.209923+0000", 2024-09-13T14:54:52.151 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000112247 2024-09-13T14:54:52.151 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.151 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.151 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T14:54:52.151 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.210021+0000", 2024-09-13T14:54:52.151 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7788000000000006e-05 2024-09-13T14:54:52.152 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.152 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.152 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.152 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.216521+0000", 2024-09-13T14:54:52.152 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.006500151 2024-09-13T14:54:52.152 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.152 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.152 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.152 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.216531+0000", 2024-09-13T14:54:52.152 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0180000000000001e-05 2024-09-13T14:54:52.152 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.152 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.152 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.152 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.216620+0000", 2024-09-13T14:54:52.153 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.9054999999999994e-05 2024-09-13T14:54:52.153 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.153 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.153 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.153 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.222936+0000", 2024-09-13T14:54:52.153 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.006315668 2024-09-13T14:54:52.153 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.153 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.153 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.153 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.222982+0000", 2024-09-13T14:54:52.153 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6338999999999998e-05 2024-09-13T14:54:52.153 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.153 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.153 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.154 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:04.223098+0000", 2024-09-13T14:54:52.154 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000115958 2024-09-13T14:54:52.154 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.154 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.154 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.154 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.154 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.154 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181248 6.17 6:eda41412:::benchmark_data_smithi096_167087_object64020:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:54:52.154 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.569149+0000", 2024-09-13T14:54:52.154 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 457.53288378100001, 2024-09-13T14:54:52.154 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.071124813, 2024-09-13T14:54:52.154 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.154 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.155 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.155 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:54:52.155 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:54:52.155 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181248 2024-09-13T14:54:52.155 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.155 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.155 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.155 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.155 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569149+0000", 2024-09-13T14:54:52.155 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.155 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.155 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.155 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.155 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569149+0000", 2024-09-13T14:54:52.156 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.156 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.156 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.156 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.156 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569148+0000", 2024-09-13T14:54:52.156 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:54:52.156 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.156 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.156 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.156 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569154+0000", 2024-09-13T14:54:52.156 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8190000000000003e-06 2024-09-13T14:54:52.156 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.156 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.156 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.157 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569178+0000", 2024-09-13T14:54:52.157 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4122e-05 2024-09-13T14:54:52.157 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.157 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.157 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.157 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569184+0000", 2024-09-13T14:54:52.157 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9100000000000002e-06 2024-09-13T14:54:52.157 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.157 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.157 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.157 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569235+0000", 2024-09-13T14:54:52.157 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0711000000000003e-05 2024-09-13T14:54:52.157 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.158 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.158 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.158 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.569245+0000", 2024-09-13T14:54:52.158 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6770000000000007e-06 2024-09-13T14:54:52.158 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.158 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.158 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.158 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565701+0000", 2024-09-13T14:54:52.158 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99645585800000003 2024-09-13T14:54:52.158 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.158 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.158 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.158 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565711+0000", 2024-09-13T14:54:52.158 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9750000000000002e-06 2024-09-13T14:54:52.159 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.159 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.159 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.159 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.634822+0000", 2024-09-13T14:54:52.159 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0691110779999999 2024-09-13T14:54:52.159 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.159 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.159 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.159 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.634916+0000", 2024-09-13T14:54:52.159 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.3894999999999995e-05 2024-09-13T14:54:52.159 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.159 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.159 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:54:52.159 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.635198+0000", 2024-09-13T14:54:52.159 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00028242499999999999 2024-09-13T14:54:52.160 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.160 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.160 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.160 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640038+0000", 2024-09-13T14:54:52.160 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0048403659999999996 2024-09-13T14:54:52.160 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.160 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.160 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.160 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640075+0000", 2024-09-13T14:54:52.160 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6956000000000003e-05 2024-09-13T14:54:52.160 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.160 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.160 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.160 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640121+0000", 2024-09-13T14:54:52.161 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5466999999999997e-05 2024-09-13T14:54:52.161 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.161 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.161 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.161 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640168+0000", 2024-09-13T14:54:52.161 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7579e-05 2024-09-13T14:54:52.161 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.161 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.161 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.161 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640202+0000", 2024-09-13T14:54:52.161 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3290999999999997e-05 2024-09-13T14:54:52.161 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.161 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.161 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.161 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640274+0000", 2024-09-13T14:54:52.162 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2492999999999999e-05 2024-09-13T14:54:52.162 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.162 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.162 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.162 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.162 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.162 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181259 6.17 6:eec3e1cc:::benchmark_data_smithi096_167087_object64031:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:54:52.162 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:52.163 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.585855+0000", 2024-09-13T14:54:52.163 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 457.51617822499998, 2024-09-13T14:54:52.163 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0547445899999999, 2024-09-13T14:54:52.163 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.164 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.164 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.164 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:54:52.164 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:54:52.164 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181259 2024-09-13T14:54:52.164 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.164 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.164 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.164 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.164 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585855+0000", 2024-09-13T14:54:52.164 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.164 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.164 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.165 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.165 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585855+0000", 2024-09-13T14:54:52.165 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.165 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.165 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.165 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.165 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585853+0000", 2024-09-13T14:54:52.165 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:54:52.165 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.165 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.165 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.165 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585861+0000", 2024-09-13T14:54:52.165 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0360000000000006e-06 2024-09-13T14:54:52.165 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.166 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.166 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.166 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585878+0000", 2024-09-13T14:54:52.166 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6558e-05 2024-09-13T14:54:52.166 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.166 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.166 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.166 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585884+0000", 2024-09-13T14:54:52.166 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6490000000000002e-06 2024-09-13T14:54:52.166 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.166 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.166 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.166 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585953+0000", 2024-09-13T14:54:52.167 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8206000000000004e-05 2024-09-13T14:54:52.167 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.167 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.167 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.167 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.585965+0000", 2024-09-13T14:54:52.167 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2094e-05 2024-09-13T14:54:52.167 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.167 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.167 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.167 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565736+0000", 2024-09-13T14:54:52.167 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97977119700000004 2024-09-13T14:54:52.167 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.167 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.167 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.168 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565744+0000", 2024-09-13T14:54:52.168 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8150000000000006e-06 2024-09-13T14:54:52.168 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.168 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.168 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.168 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.636443+0000", 2024-09-13T14:54:52.168 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.070699402 2024-09-13T14:54:52.168 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.168 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.168 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.168 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.636515+0000", 2024-09-13T14:54:52.168 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1812999999999998e-05 2024-09-13T14:54:52.168 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.168 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.169 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:54:52.169 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.636650+0000", 2024-09-13T14:54:52.169 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013464799999999999 2024-09-13T14:54:52.169 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.169 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.169 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.169 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640130+0000", 2024-09-13T14:54:52.169 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0034803210000000002 2024-09-13T14:54:52.169 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.169 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.169 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.169 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640331+0000", 2024-09-13T14:54:52.169 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00020073899999999999 2024-09-13T14:54:52.169 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.170 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.170 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.170 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640378+0000", 2024-09-13T14:54:52.170 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7766999999999999e-05 2024-09-13T14:54:52.170 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.170 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.170 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.170 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640429+0000", 2024-09-13T14:54:52.170 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0692999999999997e-05 2024-09-13T14:54:52.170 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.170 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.170 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.170 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640482+0000", 2024-09-13T14:54:52.170 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3282999999999998e-05 2024-09-13T14:54:52.171 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.171 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.171 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.171 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640599+0000", 2024-09-13T14:54:52.171 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011709099999999999 2024-09-13T14:54:52.171 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.171 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.171 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.171 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.171 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.171 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181267 6.17 6:ef08088c:::benchmark_data_smithi096_167087_object64039:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:54:52.171 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.591094+0000", 2024-09-13T14:54:52.171 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 457.51093913599999, 2024-09-13T14:54:52.171 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.052348388, 2024-09-13T14:54:52.172 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.172 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.172 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.172 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:54:52.172 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:54:52.172 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181267 2024-09-13T14:54:52.172 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.172 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.172 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.172 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.172 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591094+0000", 2024-09-13T14:54:52.172 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.172 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.173 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.173 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.173 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591094+0000", 2024-09-13T14:54:52.173 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.173 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.173 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.173 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.173 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591092+0000", 2024-09-13T14:54:52.173 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:54:52.173 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.173 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.173 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.173 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591099+0000", 2024-09-13T14:54:52.173 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9190000000000002e-06 2024-09-13T14:54:52.174 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.174 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.174 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.174 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591113+0000", 2024-09-13T14:54:52.174 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4099e-05 2024-09-13T14:54:52.174 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.174 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.174 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.174 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591119+0000", 2024-09-13T14:54:52.174 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5849999999999999e-06 2024-09-13T14:54:52.174 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.174 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.174 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.174 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591182+0000", 2024-09-13T14:54:52.174 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3466999999999994e-05 2024-09-13T14:54:52.175 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.175 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.175 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.175 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.591196+0000", 2024-09-13T14:54:52.175 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4435e-05 2024-09-13T14:54:52.175 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.175 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.175 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.175 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565767+0000", 2024-09-13T14:54:52.175 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97457061199999995 2024-09-13T14:54:52.175 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.175 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.175 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.175 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565775+0000", 2024-09-13T14:54:52.176 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7409999999999992e-06 2024-09-13T14:54:52.176 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.176 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.176 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.176 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.637678+0000", 2024-09-13T14:54:52.176 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.071902879 2024-09-13T14:54:52.176 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.176 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.176 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.176 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.637765+0000", 2024-09-13T14:54:52.176 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.6873000000000007e-05 2024-09-13T14:54:52.176 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.176 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.176 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:54:52.177 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.637918+0000", 2024-09-13T14:54:52.177 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015319899999999999 2024-09-13T14:54:52.177 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.177 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.177 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.177 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640137+0000", 2024-09-13T14:54:52.177 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0022196059999999998 2024-09-13T14:54:52.177 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.177 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.177 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.177 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640638+0000", 2024-09-13T14:54:52.177 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00050057400000000005 2024-09-13T14:54:52.177 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.178 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.178 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.178 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640683+0000", 2024-09-13T14:54:52.178 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4583999999999999e-05 2024-09-13T14:54:52.178 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.178 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.178 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.178 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643279+0000", 2024-09-13T14:54:52.178 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0025964400000000002 2024-09-13T14:54:52.178 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.178 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.178 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.178 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643331+0000", 2024-09-13T14:54:52.179 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1684000000000002e-05 2024-09-13T14:54:52.179 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.179 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.179 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.179 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643442+0000", 2024-09-13T14:54:52.179 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000111742 2024-09-13T14:54:52.179 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.179 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.179 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.179 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.179 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.179 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181270 6.17 6:ea4ae503:::benchmark_data_smithi096_167087_object64042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:54:52.179 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.594659+0000", 2024-09-13T14:54:52.179 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 457.50737394800001, 2024-09-13T14:54:52.179 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.048986019, 2024-09-13T14:54:52.180 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.180 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.180 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.180 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:54:52.180 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:54:52.180 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181270 2024-09-13T14:54:52.180 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.180 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.180 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.180 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.180 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594659+0000", 2024-09-13T14:54:52.180 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.180 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.181 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.181 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.181 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594659+0000", 2024-09-13T14:54:52.181 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.181 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.181 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.181 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.181 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594657+0000", 2024-09-13T14:54:52.181 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:54:52.181 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.181 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.181 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.181 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594663+0000", 2024-09-13T14:54:52.182 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0480000000000004e-06 2024-09-13T14:54:52.182 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.182 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.182 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.182 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594677+0000", 2024-09-13T14:54:52.182 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3688e-05 2024-09-13T14:54:52.182 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.182 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.182 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.182 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594684+0000", 2024-09-13T14:54:52.182 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3890000000000002e-06 2024-09-13T14:54:52.182 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.182 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.182 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.182 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594723+0000", 2024-09-13T14:54:52.183 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8982e-05 2024-09-13T14:54:52.183 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.183 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.183 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.183 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.594731+0000", 2024-09-13T14:54:52.183 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1380000000000007e-06 2024-09-13T14:54:52.183 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.183 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.183 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.183 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565798+0000", 2024-09-13T14:54:52.183 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97106689300000004 2024-09-13T14:54:52.183 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.183 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.183 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.184 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565805+0000", 2024-09-13T14:54:52.184 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8210000000000002e-06 2024-09-13T14:54:52.184 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.184 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.184 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.184 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.638755+0000", 2024-09-13T14:54:52.184 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0729500999999999 2024-09-13T14:54:52.184 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.184 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.184 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.184 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.638832+0000", 2024-09-13T14:54:52.184 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6369e-05 2024-09-13T14:54:52.184 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.184 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.185 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:54:52.185 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.638985+0000", 2024-09-13T14:54:52.185 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015271699999999999 2024-09-13T14:54:52.185 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.185 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.185 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.185 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640145+0000", 2024-09-13T14:54:52.185 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001160269 2024-09-13T14:54:52.185 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.185 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.185 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.185 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640711+0000", 2024-09-13T14:54:52.185 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00056574800000000003 2024-09-13T14:54:52.185 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.186 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.186 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.186 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640744+0000", 2024-09-13T14:54:52.186 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3510000000000003e-05 2024-09-13T14:54:52.186 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.186 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.186 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.186 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643496+0000", 2024-09-13T14:54:52.186 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0027522599999999999 2024-09-13T14:54:52.186 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.186 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.186 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.186 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643541+0000", 2024-09-13T14:54:52.186 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5124999999999999e-05 2024-09-13T14:54:52.187 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.187 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.187 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.187 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643645+0000", 2024-09-13T14:54:52.187 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000103733 2024-09-13T14:54:52.187 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.187 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.187 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.187 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.187 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.187 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181272 6.17 6:ef450286:::benchmark_data_smithi096_167087_object64044:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:54:52.187 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.595486+0000", 2024-09-13T14:54:52.187 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 457.50654735400002, 2024-09-13T14:54:52.187 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0541925989999998, 2024-09-13T14:54:52.188 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.188 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.188 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.188 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:54:52.188 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:54:52.188 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181272 2024-09-13T14:54:52.188 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.188 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.188 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.188 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.188 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595486+0000", 2024-09-13T14:54:52.188 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.188 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.189 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.189 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.189 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595486+0000", 2024-09-13T14:54:52.189 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.189 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.189 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.189 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.189 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595485+0000", 2024-09-13T14:54:52.189 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:54:52.189 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.189 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.189 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.189 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595489+0000", 2024-09-13T14:54:52.189 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0160000000000002e-06 2024-09-13T14:54:52.190 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.190 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.190 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.190 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595499+0000", 2024-09-13T14:54:52.190 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.006e-05 2024-09-13T14:54:52.190 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.190 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.190 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.190 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595504+0000", 2024-09-13T14:54:52.190 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6689999999999997e-06 2024-09-13T14:54:52.190 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.190 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.190 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.190 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595539+0000", 2024-09-13T14:54:52.191 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5673000000000002e-05 2024-09-13T14:54:52.191 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.191 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.191 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.191 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.595547+0000", 2024-09-13T14:54:52.191 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.515e-06 2024-09-13T14:54:52.191 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.191 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.191 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.191 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565827+0000", 2024-09-13T14:54:52.191 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97028026899999997 2024-09-13T14:54:52.191 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.191 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.192 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.192 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565834+0000", 2024-09-13T14:54:52.192 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7440000000000001e-06 2024-09-13T14:54:52.192 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.192 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.192 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.192 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640765+0000", 2024-09-13T14:54:52.192 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.074930806 2024-09-13T14:54:52.192 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.192 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.192 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.192 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640833+0000", 2024-09-13T14:54:52.192 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7933000000000002e-05 2024-09-13T14:54:52.192 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.193 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.193 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:54:52.193 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.640974+0000", 2024-09-13T14:54:52.193 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014157500000000001 2024-09-13T14:54:52.193 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.193 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.193 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.193 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643458+0000", 2024-09-13T14:54:52.193 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0024839609999999998 2024-09-13T14:54:52.193 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.193 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.193 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.193 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.645516+0000", 2024-09-13T14:54:52.194 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0020581850000000001 2024-09-13T14:54:52.194 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.194 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.194 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.194 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649470+0000", 2024-09-13T14:54:52.194 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0039536559999999998 2024-09-13T14:54:52.194 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.194 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.194 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.194 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649506+0000", 2024-09-13T14:54:52.194 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5920999999999997e-05 2024-09-13T14:54:52.194 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.194 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.194 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.195 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649554+0000", 2024-09-13T14:54:52.195 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7545e-05 2024-09-13T14:54:52.195 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.195 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.195 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.195 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649678+0000", 2024-09-13T14:54:52.195 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012479599999999999 2024-09-13T14:54:52.195 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.195 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.195 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.195 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.195 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.195 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181418 6.17 6:e9041057:::benchmark_data_smithi096_167087_object64190:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:54:52.195 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.860748+0000", 2024-09-13T14:54:52.195 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 457.24128485199998, 2024-09-13T14:54:52.196 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7891968889999998, 2024-09-13T14:54:52.196 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.196 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.196 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.196 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:54:52.196 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:54:52.196 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181418 2024-09-13T14:54:52.196 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.196 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.196 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.196 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.196 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860748+0000", 2024-09-13T14:54:52.196 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.196 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.197 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.197 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.197 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860748+0000", 2024-09-13T14:54:52.197 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.197 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.197 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.197 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.197 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860747+0000", 2024-09-13T14:54:52.197 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:52.197 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.197 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.197 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.197 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860755+0000", 2024-09-13T14:54:52.197 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5299999999999996e-06 2024-09-13T14:54:52.198 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.198 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.198 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.198 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860767+0000", 2024-09-13T14:54:52.198 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1935e-05 2024-09-13T14:54:52.198 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.198 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.198 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.198 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860772+0000", 2024-09-13T14:54:52.198 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0459999999999996e-06 2024-09-13T14:54:52.198 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.198 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.198 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.199 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860813+0000", 2024-09-13T14:54:52.199 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0963000000000001e-05 2024-09-13T14:54:52.199 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.199 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.199 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.199 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.860821+0000", 2024-09-13T14:54:52.199 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.391e-06 2024-09-13T14:54:52.199 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.199 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.199 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.199 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565856+0000", 2024-09-13T14:54:52.199 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.70503548599999999 2024-09-13T14:54:52.199 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.200 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.200 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.200 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565864+0000", 2024-09-13T14:54:52.200 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5399999999999998e-06 2024-09-13T14:54:52.200 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.200 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.200 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.200 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.641841+0000", 2024-09-13T14:54:52.200 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.07597739 2024-09-13T14:54:52.200 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.200 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.200 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.200 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.641962+0000", 2024-09-13T14:54:52.200 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012125600000000001 2024-09-13T14:54:52.201 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.201 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.201 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:54:52.201 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.642177+0000", 2024-09-13T14:54:52.201 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000214916 2024-09-13T14:54:52.201 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.201 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.201 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.201 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643655+0000", 2024-09-13T14:54:52.201 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0014778770000000001 2024-09-13T14:54:52.201 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.201 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.201 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.201 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.645548+0000", 2024-09-13T14:54:52.201 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001892864 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649754+0000", 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0042061920000000001 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649790+0000", 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5929000000000001e-05 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649839+0000", 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8649999999999997e-05 2024-09-13T14:54:52.202 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.203 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.203 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.203 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649945+0000", 2024-09-13T14:54:52.203 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000106252 2024-09-13T14:54:52.203 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.203 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.203 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.203 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.203 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.203 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:181419 6.17 6:ecfb7c25:::benchmark_data_smithi096_167087_object64191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-13T14:54:52.203 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:14.861452+0000", 2024-09-13T14:54:52.203 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 457.24058087899999, 2024-09-13T14:54:52.203 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7981438779999999, 2024-09-13T14:54:52.203 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.204 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.204 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.204 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:54:52.204 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:54:52.204 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 181419 2024-09-13T14:54:52.204 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.204 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.204 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.204 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.204 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861452+0000", 2024-09-13T14:54:52.205 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.205 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.205 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.206 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.206 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861452+0000", 2024-09-13T14:54:52.206 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.206 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.206 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.206 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.206 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861451+0000", 2024-09-13T14:54:52.206 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:54:52.206 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.206 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.206 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.206 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861459+0000", 2024-09-13T14:54:52.206 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.379e-06 2024-09-13T14:54:52.206 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.207 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.207 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.207 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861470+0000", 2024-09-13T14:54:52.207 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1311000000000001e-05 2024-09-13T14:54:52.207 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.207 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.207 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.207 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861475+0000", 2024-09-13T14:54:52.207 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1510000000000003e-06 2024-09-13T14:54:52.207 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.207 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.207 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.207 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861500+0000", 2024-09-13T14:54:52.208 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.423e-05 2024-09-13T14:54:52.208 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.208 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.208 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.208 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:14.861505+0000", 2024-09-13T14:54:52.208 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9930000000000003e-06 2024-09-13T14:54:52.208 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.208 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.208 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.208 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565910+0000", 2024-09-13T14:54:52.208 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.70440582600000001 2024-09-13T14:54:52.208 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.208 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.209 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.209 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:15.565918+0000", 2024-09-13T14:54:52.209 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9370000000000004e-06 2024-09-13T14:54:52.209 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.209 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.209 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.209 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643685+0000", 2024-09-13T14:54:52.209 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0777663710000001 2024-09-13T14:54:52.209 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.209 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.209 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.209 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.643798+0000", 2024-09-13T14:54:52.209 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000112866 2024-09-13T14:54:52.209 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.210 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.210 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:54:52.210 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.644019+0000", 2024-09-13T14:54:52.210 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00022168000000000001 2024-09-13T14:54:52.210 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.210 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.210 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.210 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649720+0000", 2024-09-13T14:54:52.210 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0057008689999999999 2024-09-13T14:54:52.210 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.210 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.210 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.210 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.649982+0000", 2024-09-13T14:54:52.210 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00026224400000000002 2024-09-13T14:54:52.211 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.211 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.211 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.211 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.650014+0000", 2024-09-13T14:54:52.211 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2039999999999998e-05 2024-09-13T14:54:52.211 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.211 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.211 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.211 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.659434+0000", 2024-09-13T14:54:52.211 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0094201530000000006 2024-09-13T14:54:52.211 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.211 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.211 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.212 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.659489+0000", 2024-09-13T14:54:52.212 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4169999999999998e-05 2024-09-13T14:54:52.212 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.212 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.212 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.212 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:16.659596+0000", 2024-09-13T14:54:52.212 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000107458 2024-09-13T14:54:52.212 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.212 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.212 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.212 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.212 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.212 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169258 7.1 7:89ed45e8:::benchmark_data_smithi096_196313_object49127:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T14:54:52.212 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.061777+0000", 2024-09-13T14:54:52.213 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 222.04025655800001, 2024-09-13T14:54:52.213 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2721801959999999, 2024-09-13T14:54:52.213 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.213 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.213 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.213 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:54:52.213 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:54:52.213 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169258 2024-09-13T14:54:52.213 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.213 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.213 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.214 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.214 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061777+0000", 2024-09-13T14:54:52.214 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.214 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.214 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.214 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.214 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061777+0000", 2024-09-13T14:54:52.214 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.214 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.214 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.214 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.214 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061775+0000", 2024-09-13T14:54:52.214 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:52.215 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.215 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.215 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.215 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061780+0000", 2024-09-13T14:54:52.215 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4140000000000002e-06 2024-09-13T14:54:52.215 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.215 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.215 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.215 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061793+0000", 2024-09-13T14:54:52.215 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2444e-05 2024-09-13T14:54:52.215 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.215 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.215 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.215 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061800+0000", 2024-09-13T14:54:52.216 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8789999999999997e-06 2024-09-13T14:54:52.216 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.216 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.216 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.216 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061860+0000", 2024-09-13T14:54:52.216 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0251000000000003e-05 2024-09-13T14:54:52.216 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.216 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.216 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.216 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061870+0000", 2024-09-13T14:54:52.216 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.8609999999999993e-06 2024-09-13T14:54:52.216 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.216 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.216 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.216 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.049861+0000", 2024-09-13T14:54:52.217 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.98799087500000005 2024-09-13T14:54:52.217 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.217 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.217 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.217 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.049872+0000", 2024-09-13T14:54:52.217 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1044e-05 2024-09-13T14:54:52.217 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.217 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.217 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.217 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.306782+0000", 2024-09-13T14:54:52.217 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.256909821 2024-09-13T14:54:52.217 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.217 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.217 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.218 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.306898+0000", 2024-09-13T14:54:52.218 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000116332 2024-09-13T14:54:52.218 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.218 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.218 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3,6", 2024-09-13T14:54:52.218 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.306989+0000", 2024-09-13T14:54:52.218 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1566999999999999e-05 2024-09-13T14:54:52.218 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.218 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.218 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.218 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.310582+0000", 2024-09-13T14:54:52.219 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035929719999999998 2024-09-13T14:54:52.219 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.219 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.219 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.219 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.318699+0000", 2024-09-13T14:54:52.219 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0081170830000000006 2024-09-13T14:54:52.219 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.219 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.219 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.219 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327316+0000", 2024-09-13T14:54:52.219 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0086162870000000003 2024-09-13T14:54:52.219 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.219 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.220 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.220 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.333834+0000", 2024-09-13T14:54:52.220 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0065179210000000003 2024-09-13T14:54:52.220 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.220 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.220 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.220 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.333866+0000", 2024-09-13T14:54:52.220 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2592000000000002e-05 2024-09-13T14:54:52.220 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.220 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.220 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.220 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.333957+0000", 2024-09-13T14:54:52.220 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.0482000000000005e-05 2024-09-13T14:54:52.220 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.221 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.221 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.221 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.221 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.221 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169259 7.1 7:8a1070f0:::benchmark_data_smithi096_196313_object49128:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T14:54:52.221 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.062386+0000", 2024-09-13T14:54:52.221 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 222.03964736099999, 2024-09-13T14:54:52.221 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.278361297, 2024-09-13T14:54:52.221 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.221 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.221 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.221 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:54:52.221 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:54:52.221 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169259 2024-09-13T14:54:52.222 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.222 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.222 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.222 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.222 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062386+0000", 2024-09-13T14:54:52.222 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.222 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.222 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.222 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.222 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062386+0000", 2024-09-13T14:54:52.222 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.222 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.222 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.222 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.222 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062385+0000", 2024-09-13T14:54:52.223 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:52.223 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.223 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.223 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.223 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062393+0000", 2024-09-13T14:54:52.223 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7110000000000001e-06 2024-09-13T14:54:52.223 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.223 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.223 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.223 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062408+0000", 2024-09-13T14:54:52.223 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4751e-05 2024-09-13T14:54:52.223 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.223 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.223 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.224 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062414+0000", 2024-09-13T14:54:52.224 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0399999999999998e-06 2024-09-13T14:54:52.224 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.224 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.224 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.224 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062473+0000", 2024-09-13T14:54:52.224 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8730999999999997e-05 2024-09-13T14:54:52.224 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.224 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.224 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.224 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062478+0000", 2024-09-13T14:54:52.224 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5829999999999997e-06 2024-09-13T14:54:52.224 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.224 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.225 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.225 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.049956+0000", 2024-09-13T14:54:52.225 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.98747715400000002 2024-09-13T14:54:52.225 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.225 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.225 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.225 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.049974+0000", 2024-09-13T14:54:52.225 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8345000000000001e-05 2024-09-13T14:54:52.225 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.225 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.225 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.225 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.310608+0000", 2024-09-13T14:54:52.225 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.260634598 2024-09-13T14:54:52.225 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.226 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.226 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.226 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.310728+0000", 2024-09-13T14:54:52.226 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119803 2024-09-13T14:54:52.226 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.226 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.226 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3,6", 2024-09-13T14:54:52.226 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.310805+0000", 2024-09-13T14:54:52.226 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6697000000000001e-05 2024-09-13T14:54:52.226 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.226 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.226 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.226 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.326994+0000", 2024-09-13T14:54:52.227 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.01618909 2024-09-13T14:54:52.227 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.227 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.227 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.227 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327004+0000", 2024-09-13T14:54:52.227 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0219e-05 2024-09-13T14:54:52.227 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.227 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.227 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.227 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327369+0000", 2024-09-13T14:54:52.227 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000365125 2024-09-13T14:54:52.227 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.227 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.227 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.228 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.340622+0000", 2024-09-13T14:54:52.228 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013252956999999999 2024-09-13T14:54:52.228 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.228 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.228 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.228 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.340678+0000", 2024-09-13T14:54:52.228 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5451999999999997e-05 2024-09-13T14:54:52.228 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.228 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.228 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.228 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.340747+0000", 2024-09-13T14:54:52.228 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9251000000000005e-05 2024-09-13T14:54:52.228 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.228 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.229 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.229 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.229 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.229 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169261 7.1 7:8e31827b:::benchmark_data_smithi096_196313_object49130:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T14:54:52.229 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.063536+0000", 2024-09-13T14:54:52.229 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 222.038496888, 2024-09-13T14:54:52.229 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2803364450000001, 2024-09-13T14:54:52.229 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.229 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.229 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.229 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:54:52.229 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:54:52.229 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169261 2024-09-13T14:54:52.229 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.230 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.230 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.230 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.230 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063536+0000", 2024-09-13T14:54:52.230 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.230 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.230 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.230 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.230 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063536+0000", 2024-09-13T14:54:52.230 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.230 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.230 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.230 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.230 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063535+0000", 2024-09-13T14:54:52.230 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:54:52.231 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.231 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.231 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.231 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063541+0000", 2024-09-13T14:54:52.231 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3190000000000002e-06 2024-09-13T14:54:52.231 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.231 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.231 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.231 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063551+0000", 2024-09-13T14:54:52.231 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0671e-05 2024-09-13T14:54:52.231 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.231 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.231 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.231 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063557+0000", 2024-09-13T14:54:52.232 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.874e-06 2024-09-13T14:54:52.232 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.232 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.232 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.232 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063600+0000", 2024-09-13T14:54:52.232 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2549999999999997e-05 2024-09-13T14:54:52.232 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.232 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.232 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.232 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063606+0000", 2024-09-13T14:54:52.232 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5490000000000003e-06 2024-09-13T14:54:52.232 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.232 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.232 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.233 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050009+0000", 2024-09-13T14:54:52.233 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.98640244499999996 2024-09-13T14:54:52.233 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.233 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.233 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.233 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050017+0000", 2024-09-13T14:54:52.233 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9740000000000002e-06 2024-09-13T14:54:52.233 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.233 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.233 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.233 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.314552+0000", 2024-09-13T14:54:52.233 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2645352220000001 2024-09-13T14:54:52.234 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.234 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.234 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.234 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.314667+0000", 2024-09-13T14:54:52.234 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000115258 2024-09-13T14:54:52.234 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.234 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.234 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3,6", 2024-09-13T14:54:52.234 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.314789+0000", 2024-09-13T14:54:52.234 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012234499999999999 2024-09-13T14:54:52.234 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.234 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.234 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.234 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327061+0000", 2024-09-13T14:54:52.235 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012271855999999999 2024-09-13T14:54:52.235 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.235 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.235 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.235 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327269+0000", 2024-09-13T14:54:52.235 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00020811700000000001 2024-09-13T14:54:52.235 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.235 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.235 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.235 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.340791+0000", 2024-09-13T14:54:52.235 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013521403 2024-09-13T14:54:52.235 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.235 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.235 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.235 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.343708+0000", 2024-09-13T14:54:52.236 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029173559999999999 2024-09-13T14:54:52.236 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.236 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.236 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.236 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.343759+0000", 2024-09-13T14:54:52.236 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0615000000000002e-05 2024-09-13T14:54:52.236 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.236 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.236 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.236 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.343873+0000", 2024-09-13T14:54:52.236 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000113931 2024-09-13T14:54:52.236 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.236 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.236 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.237 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.237 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.237 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169277 7.1 7:866405f8:::benchmark_data_smithi096_196313_object49146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T14:54:52.237 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.098666+0000", 2024-09-13T14:54:52.237 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 222.00336673300001, 2024-09-13T14:54:52.237 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.249298177, 2024-09-13T14:54:52.237 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.237 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.237 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.237 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:54:52.237 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:54:52.237 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169277 2024-09-13T14:54:52.237 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.237 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.238 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.238 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.238 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098666+0000", 2024-09-13T14:54:52.238 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.238 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.238 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.238 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.238 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098666+0000", 2024-09-13T14:54:52.238 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.238 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.238 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.238 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.238 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098665+0000", 2024-09-13T14:54:52.238 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:52.239 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.239 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.239 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.239 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098671+0000", 2024-09-13T14:54:52.239 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7470000000000002e-06 2024-09-13T14:54:52.239 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.239 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.239 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.239 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098683+0000", 2024-09-13T14:54:52.239 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2499e-05 2024-09-13T14:54:52.239 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.239 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.239 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.239 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098689+0000", 2024-09-13T14:54:52.240 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.913e-06 2024-09-13T14:54:52.240 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.240 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.240 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.240 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098749+0000", 2024-09-13T14:54:52.240 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0109000000000003e-05 2024-09-13T14:54:52.240 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.240 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.240 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.240 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098760+0000", 2024-09-13T14:54:52.240 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1022000000000001e-05 2024-09-13T14:54:52.240 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.240 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.241 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.241 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050038+0000", 2024-09-13T14:54:52.241 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95127815999999998 2024-09-13T14:54:52.241 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.241 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.241 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.241 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050045+0000", 2024-09-13T14:54:52.241 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0580000000000003e-06 2024-09-13T14:54:52.241 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.241 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.241 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.241 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.318726+0000", 2024-09-13T14:54:52.241 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2686819759999999 2024-09-13T14:54:52.241 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.242 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.242 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.242 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.318865+0000", 2024-09-13T14:54:52.242 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013841099999999999 2024-09-13T14:54:52.242 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.242 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.242 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3,6", 2024-09-13T14:54:52.242 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.318984+0000", 2024-09-13T14:54:52.242 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119265 2024-09-13T14:54:52.242 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.242 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.242 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.242 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327276+0000", 2024-09-13T14:54:52.242 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0082922559999999996 2024-09-13T14:54:52.243 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.243 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.243 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.243 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.333803+0000", 2024-09-13T14:54:52.243 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0065261069999999997 2024-09-13T14:54:52.243 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.243 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.243 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.243 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.340820+0000", 2024-09-13T14:54:52.243 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0070172510000000004 2024-09-13T14:54:52.243 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.243 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.243 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.243 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.347790+0000", 2024-09-13T14:54:52.244 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0069697939999999996 2024-09-13T14:54:52.244 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.244 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.244 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.244 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.347845+0000", 2024-09-13T14:54:52.244 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5781e-05 2024-09-13T14:54:52.244 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.244 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.244 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.244 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.347965+0000", 2024-09-13T14:54:52.244 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119197 2024-09-13T14:54:52.244 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.244 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.244 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.244 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.245 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.245 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169282 7.1 7:8cccf50f:::benchmark_data_smithi096_196313_object49151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T14:54:52.245 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.109522+0000", 2024-09-13T14:54:52.245 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 221.99251074099999, 2024-09-13T14:54:52.245 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.246319723, 2024-09-13T14:54:52.245 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.245 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.245 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.245 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:54:52.245 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:54:52.245 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169282 2024-09-13T14:54:52.245 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.245 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.245 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.246 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.246 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109522+0000", 2024-09-13T14:54:52.246 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.246 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.246 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.246 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.246 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109522+0000", 2024-09-13T14:54:52.246 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.246 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.246 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.246 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.246 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109520+0000", 2024-09-13T14:54:52.246 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:54:52.246 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.247 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.247 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.247 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109530+0000", 2024-09-13T14:54:52.247 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1239999999999994e-06 2024-09-13T14:54:52.247 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.247 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.247 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.247 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109550+0000", 2024-09-13T14:54:52.247 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0876000000000001e-05 2024-09-13T14:54:52.247 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.247 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.247 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.247 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109559+0000", 2024-09-13T14:54:52.247 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2709999999999992e-06 2024-09-13T14:54:52.247 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.248 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.248 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.248 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109616+0000", 2024-09-13T14:54:52.248 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7714999999999999e-05 2024-09-13T14:54:52.248 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.248 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.248 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.248 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109627+0000", 2024-09-13T14:54:52.248 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0054999999999999e-05 2024-09-13T14:54:52.248 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.248 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.248 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.248 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050065+0000", 2024-09-13T14:54:52.248 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.94043828399999996 2024-09-13T14:54:52.249 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.249 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.249 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.249 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050071+0000", 2024-09-13T14:54:52.249 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2489999999999998e-06 2024-09-13T14:54:52.249 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.249 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.249 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.249 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.322884+0000", 2024-09-13T14:54:52.249 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2728127630000001 2024-09-13T14:54:52.249 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.249 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.249 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.249 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.323002+0000", 2024-09-13T14:54:52.250 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011807999999999999 2024-09-13T14:54:52.250 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.250 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.250 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3,6", 2024-09-13T14:54:52.250 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.323120+0000", 2024-09-13T14:54:52.250 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000117837 2024-09-13T14:54:52.250 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.250 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.250 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.250 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.333809+0000", 2024-09-13T14:54:52.250 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010688848000000001 2024-09-13T14:54:52.250 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.250 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.250 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.251 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.333996+0000", 2024-09-13T14:54:52.251 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018746 2024-09-13T14:54:52.251 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.251 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.251 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.251 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.348004+0000", 2024-09-13T14:54:52.251 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.014007798 2024-09-13T14:54:52.251 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.251 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.251 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.251 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.355709+0000", 2024-09-13T14:54:52.251 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0077048639999999996 2024-09-13T14:54:52.251 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.251 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.252 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.252 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.355756+0000", 2024-09-13T14:54:52.252 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6893e-05 2024-09-13T14:54:52.252 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.252 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.252 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.252 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.355842+0000", 2024-09-13T14:54:52.252 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.6512000000000001e-05 2024-09-13T14:54:52.252 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.252 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.252 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.252 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.252 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.252 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169285 7.1 7:8fd36576:::benchmark_data_smithi096_196313_object49154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T14:54:52.253 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.113664+0000", 2024-09-13T14:54:52.253 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 221.988369111, 2024-09-13T14:54:52.253 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2423260599999999, 2024-09-13T14:54:52.253 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.253 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.253 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.253 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:54:52.253 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:54:52.253 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169285 2024-09-13T14:54:52.253 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.253 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.253 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.253 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.253 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113664+0000", 2024-09-13T14:54:52.254 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.254 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.254 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.254 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.254 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113664+0000", 2024-09-13T14:54:52.254 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.254 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.254 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.254 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.254 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113662+0000", 2024-09-13T14:54:52.254 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:54:52.254 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.254 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.254 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.254 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113672+0000", 2024-09-13T14:54:52.255 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7110000000000007e-06 2024-09-13T14:54:52.255 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.255 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.255 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.255 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113688+0000", 2024-09-13T14:54:52.255 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6013000000000001e-05 2024-09-13T14:54:52.255 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.255 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.255 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.255 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113695+0000", 2024-09-13T14:54:52.255 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1069999999999999e-06 2024-09-13T14:54:52.255 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.255 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.255 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.256 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113750+0000", 2024-09-13T14:54:52.256 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5210999999999997e-05 2024-09-13T14:54:52.256 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.256 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.256 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.256 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113758+0000", 2024-09-13T14:54:52.256 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7670000000000003e-06 2024-09-13T14:54:52.256 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.256 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.256 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.256 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050094+0000", 2024-09-13T14:54:52.256 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93633562699999995 2024-09-13T14:54:52.256 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.256 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.257 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.257 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050100+0000", 2024-09-13T14:54:52.257 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8639999999999998e-06 2024-09-13T14:54:52.257 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.257 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.257 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.257 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327406+0000", 2024-09-13T14:54:52.257 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.277305248 2024-09-13T14:54:52.257 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.257 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.257 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.257 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327544+0000", 2024-09-13T14:54:52.257 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013814900000000001 2024-09-13T14:54:52.257 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.258 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.258 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3,6", 2024-09-13T14:54:52.258 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327685+0000", 2024-09-13T14:54:52.258 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000140913 2024-09-13T14:54:52.258 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.258 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.258 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.258 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.334001+0000", 2024-09-13T14:54:52.258 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0063161620000000002 2024-09-13T14:54:52.258 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.258 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.258 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.258 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.334030+0000", 2024-09-13T14:54:52.258 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8631000000000001e-05 2024-09-13T14:54:52.259 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.259 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.259 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.259 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.354706+0000", 2024-09-13T14:54:52.259 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.020676019 2024-09-13T14:54:52.259 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.259 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.259 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.259 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.355880+0000", 2024-09-13T14:54:52.259 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001173978 2024-09-13T14:54:52.259 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.259 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.259 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.259 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.355908+0000", 2024-09-13T14:54:52.260 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8975e-05 2024-09-13T14:54:52.260 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.260 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.260 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.260 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.355990+0000", 2024-09-13T14:54:52.260 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1589999999999996e-05 2024-09-13T14:54:52.260 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.260 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.260 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.260 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.260 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.261 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169295 7.1 7:8b7eab30:::benchmark_data_smithi096_196313_object49164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T14:54:52.261 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.141322+0000", 2024-09-13T14:54:52.261 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 221.960711277, 2024-09-13T14:54:52.262 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2148848929999998, 2024-09-13T14:54:52.262 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:52.262 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:52.262 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:54:52.262 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:54:52.262 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:54:52.262 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169295 2024-09-13T14:54:52.262 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.262 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:52.262 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.262 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:52.262 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141322+0000", 2024-09-13T14:54:52.262 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.262 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.263 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.263 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:52.263 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141322+0000", 2024-09-13T14:54:52.263 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:52.263 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.263 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.263 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:52.263 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141320+0000", 2024-09-13T14:54:52.263 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:52.263 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.263 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.263 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:52.263 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141328+0000", 2024-09-13T14:54:52.263 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6229999999999997e-06 2024-09-13T14:54:52.264 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.264 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.264 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:52.264 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141341+0000", 2024-09-13T14:54:52.264 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.256e-05 2024-09-13T14:54:52.264 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.264 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.264 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:52.264 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141346+0000", 2024-09-13T14:54:52.264 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9960000000000001e-06 2024-09-13T14:54:52.264 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.264 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.264 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.265 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141387+0000", 2024-09-13T14:54:52.265 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0958999999999999e-05 2024-09-13T14:54:52.265 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.265 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.265 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.265 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141394+0000", 2024-09-13T14:54:52.265 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1709999999999996e-06 2024-09-13T14:54:52.265 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.265 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.265 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.265 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050123+0000", 2024-09-13T14:54:52.265 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.90872976599999999 2024-09-13T14:54:52.265 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.266 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.266 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:54:52.266 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050129+0000", 2024-09-13T14:54:52.266 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3839999999999996e-06 2024-09-13T14:54:52.266 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.266 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.266 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:52.266 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.331078+0000", 2024-09-13T14:54:52.266 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2809490889999999 2024-09-13T14:54:52.266 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.266 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.266 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:52.266 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.331165+0000", 2024-09-13T14:54:52.266 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7390000000000002e-05 2024-09-13T14:54:52.267 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.267 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.267 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3,6", 2024-09-13T14:54:52.267 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.331245+0000", 2024-09-13T14:54:52.267 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9727000000000004e-05 2024-09-13T14:54:52.267 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.267 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.267 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.267 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.334050+0000", 2024-09-13T14:54:52.267 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002804529 2024-09-13T14:54:52.267 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.267 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.267 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:54:52.267 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.334053+0000", 2024-09-13T14:54:52.268 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6100000000000002e-06 2024-09-13T14:54:52.268 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.268 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.268 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.268 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.354731+0000", 2024-09-13T14:54:52.268 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.020678215999999999 2024-09-13T14:54:52.268 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.268 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.268 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:52.268 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.356026+0000", 2024-09-13T14:54:52.268 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001294466 2024-09-13T14:54:52.268 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.268 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.268 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:54:52.269 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.356070+0000", 2024-09-13T14:54:52.269 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4295999999999997e-05 2024-09-13T14:54:52.269 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:52.269 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:52.269 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:54:52.269 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.356207+0000", 2024-09-13T14:54:52.269 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013657 2024-09-13T14:54:52.269 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.269 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.269 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.269 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:52.269 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:52.269 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:54:52.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.271 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:52.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:52.372 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:52.406 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:52.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:52.474 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:54:52.507 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:52.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.540 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:52.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:52.631 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 89 16 124557 124541 87.4324 57.625 0.0247288 0.0114326 2024-09-13T14:54:52.640 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:52.685 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:52.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:52.786 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:52.846 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:52.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:52.942 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:52.948 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:53.003 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:53.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:53.105 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:53.145 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:53.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:53.203 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:54:53.247 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:53.290 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:53.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:53.392 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:53.423 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:53.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:53.525 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:53.559 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:53.559 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:54:53.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:53.631 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Total time run: 90.0051 2024-09-13T14:54:53.631 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Total writes made: 125392 2024-09-13T14:54:53.632 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Write size: 65536 2024-09-13T14:54:53.632 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Object size: 65536 2024-09-13T14:54:53.632 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Bandwidth (MB/sec): 87.0728 2024-09-13T14:54:53.632 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev Bandwidth: 39.6326 2024-09-13T14:54:53.632 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max bandwidth (MB/sec): 245.625 2024-09-13T14:54:53.632 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min bandwidth (MB/sec): 45.875 2024-09-13T14:54:53.632 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Average IOPS: 1393 2024-09-13T14:54:53.632 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev IOPS: 634.122 2024-09-13T14:54:53.632 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max IOPS: 3930 2024-09-13T14:54:53.632 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min IOPS: 734 2024-09-13T14:54:53.632 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Average Latency(s): 0.0114808 2024-09-13T14:54:53.633 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev Latency(s): 0.0103963 2024-09-13T14:54:53.633 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max latency(s): 0.273309 2024-09-13T14:54:53.633 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min latency(s): 0.00118538 2024-09-13T14:54:53.637 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:54:53.661 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:53.686 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:53.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:53.787 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:53.821 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:53.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:53.912 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:54:53.923 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:53.951 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:53 smithi096 ceph-mon[98901]: pgmap v868: 99 pgs: 99 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2024-09-13T14:54:53.951 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:53 smithi096 ceph-mon[102928]: pgmap v868: 99 pgs: 99 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2024-09-13T14:54:53.952 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:53.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:53.958 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:53 smithi204 ceph-mon[87071]: pgmap v868: 99 pgs: 99 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2024-09-13T14:54:54.055 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:54.103 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:54.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:54.205 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:54.236 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:54.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:54.337 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:54.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.363 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:54.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:54.464 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:54.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.488 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:54.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:54.589 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:54.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.631 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:54.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:54.732 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:54.768 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:54.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:54.870 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:54.898 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:54.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:55.000 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:55.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.028 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:55.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:55.053 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:55.129 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:55.180 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:55.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:55.284 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:55.315 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:55.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:55.418 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:55.461 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:55.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:55.563 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:55.597 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:55.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:55.699 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:55.718 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:54:55.718 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:54:55.718 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.718 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45749.0:127264 8.a e279/270 8:560903e8:::benchmark_data_smithi096_227542_object1869:head v 279'7929, mlcod=279'7927)", 2024-09-13T14:54:55.718 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:54:55.709268+0000", 2024-09-13T14:54:55.718 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0067240249999999998, 2024-09-13T14:54:55.718 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0067534359999999998, 2024-09-13T14:54:55.718 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:55.718 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:54:55.718 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:55.719 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.719 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:55.719 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.709268+0000", 2024-09-13T14:54:55.719 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:55.719 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:55.719 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.719 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:55.719 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.709268+0000", 2024-09-13T14:54:55.719 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:55.719 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:55.719 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.719 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:55.719 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.709267+0000", 2024-09-13T14:54:55.719 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:54:55.719 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:55.720 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.720 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:55.720 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.709274+0000", 2024-09-13T14:54:55.720 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7700000000000004e-06 2024-09-13T14:54:55.720 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:55.720 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.720 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:55.720 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.709289+0000", 2024-09-13T14:54:55.720 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4929000000000001e-05 2024-09-13T14:54:55.720 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:55.720 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.720 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:55.721 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.709295+0000", 2024-09-13T14:54:55.721 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5430000000000001e-06 2024-09-13T14:54:55.721 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:55.721 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.721 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:55.721 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.709377+0000", 2024-09-13T14:54:55.721 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2640999999999997e-05 2024-09-13T14:54:55.721 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:55.721 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.721 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:55.722 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.709397+0000", 2024-09-13T14:54:55.722 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9378e-05 2024-09-13T14:54:55.722 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:55.722 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:55.722 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:55.722 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:55.722 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.722 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45749.0:127265 8.2 e279/270 8:477e6424:::benchmark_data_smithi096_227542_object1870:head v 279'7982, mlcod=279'7980)", 2024-09-13T14:54:55.722 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:54:55.713703+0000", 2024-09-13T14:54:55.722 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0022890990000000002, 2024-09-13T14:54:55.722 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00237879, 2024-09-13T14:54:55.722 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:54:55.722 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:54:55.722 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:54:55.723 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.723 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:54:55.723 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.713703+0000", 2024-09-13T14:54:55.723 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:55.723 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:55.723 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.723 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:54:55.723 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.713703+0000", 2024-09-13T14:54:55.723 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:54:55.723 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:55.723 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.723 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:54:55.723 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.713702+0000", 2024-09-13T14:54:55.723 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:54:55.724 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:55.724 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.724 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:54:55.724 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.713709+0000", 2024-09-13T14:54:55.724 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.4900000000000003e-06 2024-09-13T14:54:55.724 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:55.724 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.724 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:54:55.724 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.713725+0000", 2024-09-13T14:54:55.724 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6059e-05 2024-09-13T14:54:55.724 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:55.724 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.724 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:54:55.724 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.713731+0000", 2024-09-13T14:54:55.725 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9279999999999998e-06 2024-09-13T14:54:55.725 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:55.725 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.725 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:54:55.725 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.713828+0000", 2024-09-13T14:54:55.725 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6880999999999996e-05 2024-09-13T14:54:55.725 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:54:55.725 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:54:55.725 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:54:55.725 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:55.713847+0000", 2024-09-13T14:54:55.725 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8508999999999998e-05 2024-09-13T14:54:55.725 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:55.725 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:54:55.725 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:55.726 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:54:55.726 INFO:teuthology.orchestra.run.smithi204.stdout: ], 2024-09-13T14:54:55.726 INFO:teuthology.orchestra.run.smithi204.stdout: "num_ops": 2 2024-09-13T14:54:55.726 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:54:55.733 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:55.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:55.751 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:54:55.836 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:55.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.873 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:55.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:55 smithi204 ceph-mon[87071]: pgmap v869: 99 pgs: 99 active+clean; 7.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2024-09-13T14:54:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:55 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2216460911' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:54:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:55 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:54:55.973 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:56.009 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:55 smithi096 ceph-mon[98901]: pgmap v869: 99 pgs: 99 active+clean; 7.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2024-09-13T14:54:56.009 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:55 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2216460911' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:54:56.010 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:55 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:54:56.010 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:55 smithi096 ceph-mon[102928]: pgmap v869: 99 pgs: 99 active+clean; 7.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2024-09-13T14:54:56.010 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:55 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2216460911' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:54:56.010 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:55 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:54:56.012 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:56.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:56.114 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:56.143 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:56.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:56.246 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:56.284 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:56.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:56.386 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:56.410 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:54:56.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.423 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:56.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:56.451 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:54:56.451 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:54:56.452 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-09-13T14:54:56.452 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:54:56.452 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:54:56.452 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-09-13T14:54:56.525 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:56.552 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:56.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:56.654 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:56.683 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:56.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:56.785 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:56.836 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:56.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:56.938 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:56.941 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:54:56.950 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:54:56.999 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:57.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:57.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:56 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:57.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:56 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:54:57.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:56 smithi096 ceph-mon[98901]: osdmap e280: 8 total, 8 up, 8 in 2024-09-13T14:54:57.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:56 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:57.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:56 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:54:57.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:56 smithi096 ceph-mon[102928]: osdmap e280: 8 total, 8 up, 8 in 2024-09-13T14:54:57.086 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:56 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:57.086 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:56 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:54:57.087 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:56 smithi204 ceph-mon[87071]: osdmap e280: 8 total, 8 up, 8 in 2024-09-13T14:54:57.102 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:57.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.143 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:57.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:57.244 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:57.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.280 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:57.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:57.381 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:57.416 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:57.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:57.518 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:57.557 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:57.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:57.659 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:57.688 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:57.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:57.790 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:57.817 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:57.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:57.918 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:57.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.956 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:57.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:58.057 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:58.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:57 smithi096 ceph-mon[98901]: pgmap v871: 99 pgs: 99 active+clean; 7.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2024-09-13T14:54:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:54:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:54:57] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.51.0" 2024-09-13T14:54:58.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:57 smithi096 ceph-mon[102928]: pgmap v871: 99 pgs: 99 active+clean; 7.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2024-09-13T14:54:58.087 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:58.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:58.095 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:57 smithi204 ceph-mon[87071]: pgmap v871: 99 pgs: 99 active+clean; 7.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2024-09-13T14:54:58.189 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:58.225 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:58.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:58.326 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:58.362 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:58.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:58.456 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:54:58.456 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:54:58.465 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:58.484 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:54:58.484 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:54:58.492 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:58.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:58.516 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:54:58.517 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:54:58.547 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:54:58.547 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:54:58.581 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:54:58.581 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:54:58.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:58.617 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:54:58.617 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:54:58.627 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:58.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:58.648 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:54:58.648 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:54:58.684 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:54:58.685 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:54:58.728 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:58.753 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:58.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:58.855 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:58.865 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:58 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3053493770' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:58.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.889 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:58.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:58.990 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:58.998 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:58 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3053493770' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:58.999 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:58 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3053493770' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:59.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.032 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:59.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:59.133 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:59.164 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:59.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:59.231 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:54:59.266 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:59.304 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:59.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:59.405 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:59.431 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:54:59.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:59.533 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:59.576 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:59.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:59.648 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:59.678 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:59.726 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:59.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:59.807 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:54:59 smithi096 ceph-mon[98901]: pgmap v872: 99 pgs: 99 active+clean; 7.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 123 B/s rd, 37 MiB/s wr, 723 op/s 2024-09-13T14:54:59.807 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:54:59 smithi096 ceph-mon[102928]: pgmap v872: 99 pgs: 99 active+clean; 7.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 123 B/s rd, 37 MiB/s wr, 723 op/s 2024-09-13T14:54:59.831 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:59.868 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:54:59.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:59.970 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:00.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.003 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:00.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:00.065 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:54:59 smithi204 ceph-mon[87071]: pgmap v872: 99 pgs: 99 active+clean; 7.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 123 B/s rd, 37 MiB/s wr, 723 op/s 2024-09-13T14:55:00.105 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:00.128 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:00.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:00.230 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:00.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.263 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:00.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:00.364 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:00.398 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:55:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:55:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:00.404 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:00.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:00.508 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:00.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.557 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:00.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:00.658 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:00.665 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:00.665 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:00.691 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:00.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:00.793 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:00.821 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:00.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:00.919 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:55:00.919 INFO:teuthology.orchestra.run.smithi204.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:55:00.919 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:55:00.922 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:00.962 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:00.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:01.064 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:01.093 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:01.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:01.195 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:01.226 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:01.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:01.329 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:01.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.357 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:01.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:01.458 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:01.490 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:01.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:01.542 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:55:01.581 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:55:01.592 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:01.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.659 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:01.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:01.760 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:01.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.802 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:01.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:01.903 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:01.912 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:01 smithi096 ceph-mon[98901]: pgmap v873: 99 pgs: 99 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 31 MiB/s wr, 889 op/s 2024-09-13T14:55:01.913 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:01 smithi096 ceph-mon[102928]: pgmap v873: 99 pgs: 99 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 31 MiB/s wr, 889 op/s 2024-09-13T14:55:01.929 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:01.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:02.032 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:02.061 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:02.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:02.163 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:02.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.185 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:02.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:02.217 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:01 smithi204 ceph-mon[87071]: pgmap v873: 99 pgs: 99 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 31 MiB/s wr, 889 op/s 2024-09-13T14:55:02.287 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:02.312 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:02.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:02.356 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:02.414 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:02.436 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:02.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:02.538 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:02.559 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:02.661 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:02.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.697 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:02.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:02.798 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:02.822 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:02.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:02.924 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:02.967 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:02.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:03.069 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:03.095 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:03.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:03.196 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:03.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.237 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:03.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:03.339 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:03.367 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:03.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:03.470 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:03.500 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:03.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:03.602 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:03.629 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:03.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:03.679 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:03.731 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:03.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.780 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:03.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:03.785 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:03 smithi096 ceph-mon[102928]: pgmap v874: 99 pgs: 99 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 891 op/s 2024-09-13T14:55:03.881 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:03.894 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:03 smithi204 ceph-mon[87071]: pgmap v874: 99 pgs: 99 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 891 op/s 2024-09-13T14:55:03.901 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:03.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:04.003 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:04.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.031 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:04.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:04.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:03 smithi096 ceph-mon[98901]: pgmap v874: 99 pgs: 99 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 891 op/s 2024-09-13T14:55:04.132 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:04.162 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:04.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:04.264 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:04.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.300 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:04.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:04.401 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:04.436 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:04.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:04.531 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:04.532 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:04.537 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:04.576 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:04.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:04.678 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:04.717 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:04.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:04.800 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:55:04.819 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:04.852 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:04.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:04.954 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:04.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.993 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:04.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:05.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:04 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2621141612' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:04 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:05.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:04 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2621141612' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:05.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:04 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:05.094 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:05.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.133 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:05.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:05.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:04 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2621141612' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:05.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:04 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:05.233 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:05.269 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:05.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:05.335 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:55:05.371 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:05.412 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:05.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:05.514 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:05.550 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:05.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:05.653 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:05.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.696 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:05.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:05.739 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:05.797 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:05.827 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:05.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:05.932 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:05.964 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:05.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:06.056 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:05 smithi204 ceph-mon[87071]: pgmap v875: 99 pgs: 99 active+clean; 7.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 3.8 MiB/s wr, 865 op/s 2024-09-13T14:55:06.056 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:05 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:06.056 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:05 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:06.056 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:05 smithi204 ceph-mon[87071]: osdmap e281: 8 total, 8 up, 8 in 2024-09-13T14:55:06.066 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:06.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:05 smithi096 ceph-mon[98901]: pgmap v875: 99 pgs: 99 active+clean; 7.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 3.8 MiB/s wr, 865 op/s 2024-09-13T14:55:06.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:05 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:06.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:05 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:06.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:05 smithi096 ceph-mon[98901]: osdmap e281: 8 total, 8 up, 8 in 2024-09-13T14:55:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:05 smithi096 ceph-mon[102928]: pgmap v875: 99 pgs: 99 active+clean; 7.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 3.8 MiB/s wr, 865 op/s 2024-09-13T14:55:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:05 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:05 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:05 smithi096 ceph-mon[102928]: osdmap e281: 8 total, 8 up, 8 in 2024-09-13T14:55:06.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.113 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:06.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:06.214 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:06.241 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:06.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:06.343 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:06.375 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:06.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:06.478 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:06.510 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:06.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:06.613 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:06.649 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:06.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:06.750 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:06.793 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:06.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:06.861 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:55:06.861 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:06.895 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:06.898 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:55:06.898 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:06.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.931 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:06.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:06.932 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:55:06.933 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:06.933 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:55:06.934 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:06.934 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:06 smithi096 ceph-mon[98901]: pgmap v877: 99 pgs: 99 active+clean; 7.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 3.8 MiB/s wr, 865 op/s 2024-09-13T14:55:06.934 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:06 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1240733304' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:06.934 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:55:06.935 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:06.935 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:06 smithi096 ceph-mon[102928]: pgmap v877: 99 pgs: 99 active+clean; 7.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 3.8 MiB/s wr, 865 op/s 2024-09-13T14:55:06.935 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:06 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1240733304' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:06.969 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:55:06.969 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:07.003 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:55:07.004 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:07.032 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:07.043 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:55:07.043 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:07.068 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:07.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:07.075 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:55:07.189 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:07.190 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:55:07.190 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:07.190 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:07.191 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:55:07.191 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:07.191 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:06 smithi204 ceph-mon[87071]: pgmap v877: 99 pgs: 99 active+clean; 7.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 3.8 MiB/s wr, 865 op/s 2024-09-13T14:55:07.191 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:06 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1240733304' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:07.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.229 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:07.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:07.240 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:55:07.240 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:55:07.240 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-09-13T14:55:07.240 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:55:07.240 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:55:07.241 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-09-13T14:55:07.330 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:07.369 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:07.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:07.471 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:07.510 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:07.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:07.612 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:07.634 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:55:07.641 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:07.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:07.743 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:07.770 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:07.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:07.872 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:07.915 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:07.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:07.953 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:55:07.954 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:55:07.954 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.954 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45749.0:137744 8.9 8:939ff98c:::benchmark_data_smithi096_227542_object12349:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-09-13T14:55:07.954 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:55:07.936235+0000", 2024-09-13T14:55:07.954 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.015844097000000001, 2024-09-13T14:55:07.954 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.015875165, 2024-09-13T14:55:07.954 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:07.954 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:55:07.954 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:55:07.954 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45749", 2024-09-13T14:55:07.954 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:55:07.954 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 137744 2024-09-13T14:55:07.955 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.955 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:07.955 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.955 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:07.955 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.936235+0000", 2024-09-13T14:55:07.955 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:07.955 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.955 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.955 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:07.955 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.936235+0000", 2024-09-13T14:55:07.955 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:07.955 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.956 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.956 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:07.956 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.936234+0000", 2024-09-13T14:55:07.956 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:55:07.956 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.956 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.956 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:07.956 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.936238+0000", 2024-09-13T14:55:07.956 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9530000000000001e-06 2024-09-13T14:55:07.956 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.956 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.956 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:07.957 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.936245+0000", 2024-09-13T14:55:07.957 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.99e-06 2024-09-13T14:55:07.957 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.957 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.957 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:07.957 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.936250+0000", 2024-09-13T14:55:07.957 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5140000000000003e-06 2024-09-13T14:55:07.957 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.957 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.957 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:07.957 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.936341+0000", 2024-09-13T14:55:07.957 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1847000000000003e-05 2024-09-13T14:55:07.958 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.958 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.958 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:07.958 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.936439+0000", 2024-09-13T14:55:07.958 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7146999999999997e-05 2024-09-13T14:55:07.958 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.958 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.958 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:55:07.958 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.936561+0000", 2024-09-13T14:55:07.958 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000122031 2024-09-13T14:55:07.958 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.958 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.958 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:55:07.958 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.948553+0000", 2024-09-13T14:55:07.959 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011991943999999999 2024-09-13T14:55:07.959 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:07.959 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:07.959 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:07.959 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.959 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.959 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45749.0:137748 8.3 e281/270 8:cd96093a:::benchmark_data_smithi096_227542_object12353:head v 281'8560, mlcod=281'8556)", 2024-09-13T14:55:07.959 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:55:07.945950+0000", 2024-09-13T14:55:07.959 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0061291740000000003, 2024-09-13T14:55:07.959 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0062441349999999996, 2024-09-13T14:55:07.959 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:07.959 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:55:07.959 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:07.960 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.960 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:07.960 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.945950+0000", 2024-09-13T14:55:07.960 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:07.960 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.960 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.960 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:07.960 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.945950+0000", 2024-09-13T14:55:07.960 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:07.960 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.960 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.960 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:07.960 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.945948+0000", 2024-09-13T14:55:07.961 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:55:07.961 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.961 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.961 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:07.961 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.945956+0000", 2024-09-13T14:55:07.961 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7249999999999997e-06 2024-09-13T14:55:07.961 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.961 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.961 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:07.961 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.945971+0000", 2024-09-13T14:55:07.961 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5109000000000001e-05 2024-09-13T14:55:07.961 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.961 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.962 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:07.962 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.945978+0000", 2024-09-13T14:55:07.962 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4570000000000002e-06 2024-09-13T14:55:07.962 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.962 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.962 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:07.962 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.948937+0000", 2024-09-13T14:55:07.962 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029589450000000002 2024-09-13T14:55:07.962 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.962 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.962 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:07.962 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.948955+0000", 2024-09-13T14:55:07.962 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8513e-05 2024-09-13T14:55:07.963 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:07.963 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:07.963 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:07.963 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.963 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.963 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45749.0:137752 8.7 e281/270 8:ef9c2ffb:::benchmark_data_smithi096_227542_object12357:head v 281'8399, mlcod=281'8397)", 2024-09-13T14:55:07.963 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:55:07.946005+0000", 2024-09-13T14:55:07.963 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0060737849999999999, 2024-09-13T14:55:07.963 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0062456839999999996, 2024-09-13T14:55:07.963 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:07.963 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:55:07.963 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:07.963 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.964 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:07.964 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.946005+0000", 2024-09-13T14:55:07.964 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:07.964 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.964 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.964 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:07.964 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.946005+0000", 2024-09-13T14:55:07.964 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:07.964 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.964 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.964 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:07.964 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.946005+0000", 2024-09-13T14:55:07.964 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:55:07.965 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.965 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.965 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:07.965 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.946009+0000", 2024-09-13T14:55:07.965 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4039999999999997e-06 2024-09-13T14:55:07.965 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.965 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.965 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:07.965 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.946016+0000", 2024-09-13T14:55:07.965 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8539999999999999e-06 2024-09-13T14:55:07.965 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.965 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.966 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:07.966 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.946020+0000", 2024-09-13T14:55:07.966 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8920000000000002e-06 2024-09-13T14:55:07.966 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.966 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.966 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:07.966 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.946092+0000", 2024-09-13T14:55:07.966 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2224999999999994e-05 2024-09-13T14:55:07.966 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:07.966 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:07.966 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:07.966 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.946111+0000", 2024-09-13T14:55:07.966 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9054000000000001e-05 2024-09-13T14:55:07.967 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:07.967 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:07.967 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:07.967 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:07.967 INFO:teuthology.orchestra.run.smithi204.stdout: ], 2024-09-13T14:55:07.967 INFO:teuthology.orchestra.run.smithi204.stdout: "num_ops": 3 2024-09-13T14:55:07.967 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:55:08.017 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:08.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.068 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:08.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:08.071 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:08.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:55:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:55:07] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-13T14:55:08.170 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:08.218 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:08.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:08.320 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:08.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.351 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:08.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:08.452 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:08.486 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:08.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:08.588 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:08.621 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:08.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:08.720 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:08.723 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:08.748 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:55:08.761 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:08.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:08.863 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:08.901 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:08.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:09.003 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:09.037 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:09.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:09.139 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:09.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.175 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:09.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:09.276 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:09.316 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:09.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:09.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:09 smithi096 ceph-mon[98901]: pgmap v878: 99 pgs: 99 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 435 KiB/s wr, 849 op/s 2024-09-13T14:55:09.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:09 smithi096 ceph-mon[102928]: pgmap v878: 99 pgs: 99 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 435 KiB/s wr, 849 op/s 2024-09-13T14:55:09.418 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:09.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.453 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:09.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:09.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:09 smithi204 ceph-mon[87071]: pgmap v878: 99 pgs: 99 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 435 KiB/s wr, 849 op/s 2024-09-13T14:55:09.553 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:09.599 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:09.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:09.701 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:09.731 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:09.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:09.833 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:09.844 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:55:09.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.870 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:09.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:09.972 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:09.998 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:09.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:10.101 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:10.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.168 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:10.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:10.269 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:10.298 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:10.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:10.373 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:10.401 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:10.464 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:10.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:55:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:55:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:10.566 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:10.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.606 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:10.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:10.708 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:10.744 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:10.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:10.845 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:10.887 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:10.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:10.989 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:11.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.026 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:11.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:11.084 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:55:11.128 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:11.184 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:11.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:11.248 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:11.286 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:11.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:11 smithi096 ceph-mon[98901]: pgmap v879: 99 pgs: 99 active+clean; 6.9 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2024-09-13T14:55:11.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:11 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/659884589' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:11.325 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:11 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:11.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:11 smithi096 ceph-mon[102928]: pgmap v879: 99 pgs: 99 active+clean; 6.9 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2024-09-13T14:55:11.326 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:11 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/659884589' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:11.326 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:11 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:11.330 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:11 smithi204 ceph-mon[87071]: pgmap v879: 99 pgs: 99 active+clean; 6.9 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2024-09-13T14:55:11.330 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:11 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/659884589' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:11.330 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:11 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:11.330 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:11.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:11.432 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:11.468 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:11.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:11.570 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:11.606 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:11.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:11.708 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:11.749 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:11.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:11.792 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:55:11.792 INFO:teuthology.orchestra.run.smithi204.stdout: "success": "" 2024-09-13T14:55:11.792 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:55:11.851 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:11.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.895 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:11.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:11.996 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:12.029 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:12.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:12.131 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:12.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.163 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:12.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:12.264 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:12.302 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:12.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:12.345 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:12 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:12.345 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:12 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:55:12.345 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:12 smithi204 ceph-mon[87071]: osdmap e282: 8 total, 8 up, 8 in 2024-09-13T14:55:12.361 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:12 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:12.361 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:12 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:55:12.361 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:12 smithi096 ceph-mon[102928]: osdmap e282: 8 total, 8 up, 8 in 2024-09-13T14:55:12.361 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:12 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:12.361 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:12 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:55:12.361 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:12 smithi096 ceph-mon[98901]: osdmap e282: 8 total, 8 up, 8 in 2024-09-13T14:55:12.404 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:12.433 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:12.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:12.535 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:12.574 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:12.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:12.676 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:12.699 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:12.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:12.801 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:12.832 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:12.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:12.934 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:12.962 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:12.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:13.064 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:13.086 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:13.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:13.188 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:13.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:13 smithi204 ceph-mon[87071]: pgmap v881: 99 pgs: 99 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2024-09-13T14:55:13.234 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:13.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:13.309 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:13 smithi096 ceph-mon[98901]: pgmap v881: 99 pgs: 99 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2024-09-13T14:55:13.309 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:13 smithi096 ceph-mon[102928]: pgmap v881: 99 pgs: 99 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2024-09-13T14:55:13.336 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:13.353 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:55:13.354 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:13.370 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:13.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:13.399 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:55:13.399 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:13.446 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:55:13.446 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:13.472 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:13.480 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:55:13.480 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:13.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.511 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:13.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:13.513 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:55:13.513 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:13.546 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:55:13.547 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:13.575 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:55:13.575 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:13.613 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:13.615 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:55:13.615 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:13.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.644 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:13.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:13.745 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:13.783 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:13.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:13.885 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:13.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.922 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:13.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:14.023 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:14.068 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:14.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:14.169 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:14.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.203 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:14.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:14.249 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:55:14.282 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:14 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/912116067' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:14.282 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:14 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/912116067' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:14.287 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:14 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/912116067' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:14.305 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:14.334 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:14.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:14.437 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:14.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.506 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:14.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:14.562 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:14.562 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:14.607 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:14.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.658 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:14.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:14.708 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:14.760 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:14.811 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:14.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:14.913 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:14.955 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:14.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:15.056 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:15.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.090 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:15.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:15.191 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:15.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.218 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:15.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:15.319 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:15.350 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:15.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:15.452 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:15.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:15 smithi204 ceph-mon[87071]: pgmap v882: 99 pgs: 99 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2024-09-13T14:55:15.482 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:15.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:15.566 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:55:15.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:15 smithi096 ceph-mon[98901]: pgmap v882: 99 pgs: 99 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2024-09-13T14:55:15.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:15 smithi096 ceph-mon[102928]: pgmap v882: 99 pgs: 99 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2024-09-13T14:55:15.586 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:15.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.665 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:15.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:15.767 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:15.800 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:15.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:15.902 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:15.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.931 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:15.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:15.963 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:16.032 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:16.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.058 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:16.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:16.159 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:16.180 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:16.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:16.282 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:16.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.318 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:16.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:16.419 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:16.461 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:16.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:16.465 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:55:16.563 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:16.616 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:16.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:16.718 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:16.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.758 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:16.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:16.859 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:16.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.902 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:16.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:17.003 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:17.041 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:17.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:17.143 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:17.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.182 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:17.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:17.283 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:17.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.307 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:17.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:17.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:17 smithi096 ceph-mon[102928]: pgmap v883: 99 pgs: 99 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2024-09-13T14:55:17.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:17 smithi096 ceph-mon[98901]: pgmap v883: 99 pgs: 99 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2024-09-13T14:55:17.371 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:17 smithi204 ceph-mon[87071]: pgmap v883: 99 pgs: 99 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2024-09-13T14:55:17.409 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:17.430 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:17.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:17.532 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:17.553 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:17.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:17.655 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:17.675 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:17.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:17.777 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:17.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.801 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:17.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:17.902 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:17.938 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:17.938 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:17.942 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:17.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:18.044 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:18.076 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:18.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:18.178 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:18.220 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:18.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:18.325 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:18.326 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:55:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:55:17] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-13T14:55:18.352 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:18.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:18.454 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:18.490 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:18.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:18.592 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:18.633 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:18.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:18.673 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:55:18.735 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:18.770 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:18.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:18.871 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:18.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.907 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:18.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:18.910 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:19.008 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:19.045 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:19.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:19.057 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:55:19.057 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:55:19.057 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2024-09-13T14:55:19.057 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:55:19.057 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:55:19.058 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 config set filestore_inject_stall 3 2024-09-13T14:55:19.134 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:19.147 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:19.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.183 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:19.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:19.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:19 smithi204 ceph-mon[87071]: pgmap v884: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2024-09-13T14:55:19.284 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:19.315 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:19.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:19.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:19 smithi096 ceph-mon[98901]: pgmap v884: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2024-09-13T14:55:19.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:19 smithi096 ceph-mon[102928]: pgmap v884: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2024-09-13T14:55:19.417 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:19.466 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:19.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:19.568 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:19.599 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:19.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:19.701 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:19.734 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:19.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:19.836 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:19.876 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:19.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:19.978 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:20.030 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:20.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:20.132 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:20.136 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:55:20.192 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:20.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:20.294 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:20.305 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:20 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/229517101' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:20.305 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:20 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:20.306 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:55:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:55:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:20.325 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:20.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:20.427 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:20.464 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:20.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:20.496 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:20 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/229517101' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:20.496 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:20 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:20.497 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:20 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/229517101' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:20.497 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:20 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:20.566 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:20.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.602 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:20.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:20.703 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:20.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.742 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:20.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:20.843 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:20.899 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:20.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:21.001 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:21.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.039 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:21.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:21.140 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:21.173 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:21.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:21.275 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:21.311 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:21.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:21.413 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:21.418 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:21 smithi204 ceph-mon[87071]: pgmap v885: 99 pgs: 99 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2024-09-13T14:55:21.418 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:21 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:55:21.418 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:21 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:55:21.418 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:21 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:21.418 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:21 smithi204 ceph-mon[87071]: osdmap e283: 8 total, 8 up, 8 in 2024-09-13T14:55:21.418 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:55:21.419 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:21.435 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:21 smithi096 ceph-mon[98901]: pgmap v885: 99 pgs: 99 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2024-09-13T14:55:21.435 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:21 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:55:21.435 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:21 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:55:21.435 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:21 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:21.435 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:21 smithi096 ceph-mon[98901]: osdmap e283: 8 total, 8 up, 8 in 2024-09-13T14:55:21.435 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:55:21.436 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:21.436 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:21 smithi096 ceph-mon[102928]: pgmap v885: 99 pgs: 99 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2024-09-13T14:55:21.436 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:21 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:55:21.437 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:21 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:55:21.437 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:21 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:21.437 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:21 smithi096 ceph-mon[102928]: osdmap e283: 8 total, 8 up, 8 in 2024-09-13T14:55:21.437 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:55:21.437 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:21.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.450 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:21.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:21.549 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:21.550 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:21.552 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:21.596 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:21.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:21.698 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:21.729 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:21.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:21.831 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:21.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.882 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:21.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:21.919 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:21.983 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:22.040 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:22.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:22.142 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:22.177 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:22.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:22.279 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:22.309 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:22.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:22.411 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:22.440 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:55:22.451 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:22.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:22.554 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:22.590 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:22.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:22.692 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:22.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.742 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:22.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:22.843 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:22.862 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:22.873 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:22.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:22.975 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:23.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.014 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:23.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:23.115 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:23.152 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:23.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:23.254 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:23.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.284 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:23.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:23.286 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:23 smithi204 ceph-mon[87071]: pgmap v887: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2024-09-13T14:55:23.346 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:55:23.384 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:23.392 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:23 smithi096 ceph-mon[102928]: pgmap v887: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2024-09-13T14:55:23.393 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:23 smithi096 ceph-mon[98901]: pgmap v887: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2024-09-13T14:55:23.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.414 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:23.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:23.515 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:23.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.575 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:23.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:23.677 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:23.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.715 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:23.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:23.816 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:23.876 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:23.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:23.978 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:24.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.020 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:24.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:24.121 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:24.150 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:55:24.150 INFO:teuthology.orchestra.run.smithi096.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:55:24.150 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:55:24.164 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:24.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:24.214 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:55:24.215 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:55:24.215 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:55:24.215 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:55:24.215 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:55:24.215 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:24.215 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:24.215 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:24.215 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:24.215 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:24.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:24 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1846491671' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:24.241 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:55:24.241 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:24.267 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:24.272 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:55:24.272 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:24.272 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:24 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1846491671' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:24.273 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:24 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1846491671' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:24.294 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:24.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:24.305 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:55:24.305 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:24.397 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:24.433 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:24.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:24.534 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:24.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.569 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:24.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:24.670 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:24.698 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:24.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:24.800 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:24.835 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:24.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:24.935 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:55:24.935 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:55:24.935 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T14:55:24.935 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T14:55:24.935 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:55:24.937 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:24.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.973 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:24.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:25.074 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:25.102 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:25.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:25.204 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:25.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.236 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:25.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:25.337 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:25.373 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:25.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:25 smithi204 ceph-mon[87071]: pgmap v888: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 826 op/s 2024-09-13T14:55:25.475 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:25.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.509 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:25.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:25.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:25 smithi096 ceph-mon[98901]: pgmap v888: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 826 op/s 2024-09-13T14:55:25.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:25 smithi096 ceph-mon[102928]: pgmap v888: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 826 op/s 2024-09-13T14:55:25.610 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:25.647 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:25.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:25.671 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:55:25.749 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:25.798 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:25.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:25.813 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:25.901 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:25.939 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:25.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:26.041 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:26.075 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:26.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.087 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:26.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:26.188 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:26.232 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:26.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:26.334 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:26.373 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:26.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:26.475 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:26.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.507 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:26.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:26.608 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:26.644 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:26.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:26.746 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:26.779 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:26.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:26.880 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:26.911 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:26.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:27.013 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:27.047 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:27.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:27.149 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:27.172 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:55:27.197 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:27.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:27.299 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:27.299 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:27 smithi204 ceph-mon[87071]: pgmap v889: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 826 op/s 2024-09-13T14:55:27.300 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:27 smithi096 ceph-mon[98901]: pgmap v889: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 826 op/s 2024-09-13T14:55:27.300 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:27 smithi096 ceph-mon[102928]: pgmap v889: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 826 op/s 2024-09-13T14:55:27.349 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:27.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:27.451 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:27.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.480 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:27.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:27.581 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:27.619 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:27.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:27.720 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:27.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.755 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:27.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:27.826 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:27.856 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:27.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.887 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:27.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:27.946 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:55:27.946 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T14:55:27.947 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T14:55:27.947 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:55:27.947 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.947 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156611 6.3 6.ccd62063 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-13T14:55:27.947 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.006819+0000", 2024-09-13T14:55:27.947 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 535.93648104700003, 2024-09-13T14:55:27.947 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.30103220400000003, 2024-09-13T14:55:27.948 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:27.948 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:55:27.948 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:55:27.948 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:55:27.948 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:27.948 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156611 2024-09-13T14:55:27.949 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.949 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:27.949 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.949 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:27.949 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006819+0000", 2024-09-13T14:55:27.949 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.950 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.950 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.950 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:27.950 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006819+0000", 2024-09-13T14:55:27.951 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.951 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.951 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.951 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:27.951 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006816+0000", 2024-09-13T14:55:27.951 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999971 2024-09-13T14:55:27.952 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.952 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.952 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:27.952 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006831+0000", 2024-09-13T14:55:27.952 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5e-05 2024-09-13T14:55:27.952 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.952 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.953 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:27.953 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006848+0000", 2024-09-13T14:55:27.953 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6429000000000001e-05 2024-09-13T14:55:27.953 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.953 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.954 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:27.954 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.006856+0000", 2024-09-13T14:55:27.954 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8420000000000005e-06 2024-09-13T14:55:27.954 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.954 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.954 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:27.954 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.307840+0000", 2024-09-13T14:55:27.955 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.30098485899999999 2024-09-13T14:55:27.955 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.955 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.955 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:27.955 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.307851+0000", 2024-09-13T14:55:27.955 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0763e-05 2024-09-13T14:55:27.955 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:27.956 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:27.956 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:27.956 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.956 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.956 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:156604 6.9 e173/169)", 2024-09-13T14:55:27.956 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.013215+0000", 2024-09-13T14:55:27.956 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 535.93008532600004, 2024-09-13T14:55:27.957 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.26933052299999999, 2024-09-13T14:55:27.957 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:27.957 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:55:27.957 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:27.957 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.958 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:27.958 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013215+0000", 2024-09-13T14:55:27.958 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.958 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.958 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.958 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:27.958 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013215+0000", 2024-09-13T14:55:27.958 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.959 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.959 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.959 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:27.959 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013214+0000", 2024-09-13T14:55:27.959 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:55:27.959 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.960 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.960 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:27.960 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013223+0000", 2024-09-13T14:55:27.960 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.0750000000000004e-06 2024-09-13T14:55:27.960 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.960 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.960 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:27.960 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013236+0000", 2024-09-13T14:55:27.960 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2925000000000001e-05 2024-09-13T14:55:27.960 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.960 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.960 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:27.961 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.013242+0000", 2024-09-13T14:55:27.961 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0839999999999997e-06 2024-09-13T14:55:27.961 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.961 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.961 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:27.961 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282529+0000", 2024-09-13T14:55:27.961 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.26928756599999998 2024-09-13T14:55:27.961 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.961 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.961 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:27.961 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282545+0000", 2024-09-13T14:55:27.961 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5888000000000001e-05 2024-09-13T14:55:27.961 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:156612 6.9 e173/169)", 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.023006+0000", 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 535.92029372399998, 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.25955990099999998, 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023006+0000", 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.962 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.963 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.963 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:27.963 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023006+0000", 2024-09-13T14:55:27.963 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.963 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.963 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.963 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:27.964 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023004+0000", 2024-09-13T14:55:27.964 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999976 2024-09-13T14:55:27.964 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.964 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.964 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:27.965 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023023+0000", 2024-09-13T14:55:27.965 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9327999999999999e-05 2024-09-13T14:55:27.965 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.965 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.965 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:27.965 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023054+0000", 2024-09-13T14:55:27.965 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0972000000000001e-05 2024-09-13T14:55:27.966 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.966 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.966 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:27.966 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023064+0000", 2024-09-13T14:55:27.966 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0198e-05 2024-09-13T14:55:27.966 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.966 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.966 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:27.966 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282559+0000", 2024-09-13T14:55:27.966 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.25949471600000001 2024-09-13T14:55:27.967 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.967 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.967 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:27.967 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282566+0000", 2024-09-13T14:55:27.967 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1570000000000003e-06 2024-09-13T14:55:27.967 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:27.967 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:27.967 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:27.967 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.967 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.967 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156611 6.3 6:c6046b33:::benchmark_data_smithi096_167087_object39383:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:55:27.967 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.023013+0000", 2024-09-13T14:55:27.967 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 535.92028701499999, 2024-09-13T14:55:27.968 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0343409539999999, 2024-09-13T14:55:27.968 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:27.968 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:27.968 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:55:27.968 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:55:27.968 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:27.968 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156611 2024-09-13T14:55:27.968 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.968 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:27.968 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.968 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:27.968 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023013+0000", 2024-09-13T14:55:27.968 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.969 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.969 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.969 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:27.969 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023013+0000", 2024-09-13T14:55:27.969 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.969 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.969 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.969 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:27.969 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023004+0000", 2024-09-13T14:55:27.969 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999909 2024-09-13T14:55:27.969 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.969 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.969 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:27.970 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.023024+0000", 2024-09-13T14:55:27.970 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9893000000000001e-05 2024-09-13T14:55:27.970 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.970 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.970 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:27.970 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.039994+0000", 2024-09-13T14:55:27.970 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.016970180000000001 2024-09-13T14:55:27.970 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.971 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.971 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:27.971 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040019+0000", 2024-09-13T14:55:27.971 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4919e-05 2024-09-13T14:55:27.971 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.971 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.971 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:27.971 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.308192+0000", 2024-09-13T14:55:27.972 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.26817356999999997 2024-09-13T14:55:27.972 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.972 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.972 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:55:27.972 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.308207+0000", 2024-09-13T14:55:27.972 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4960000000000001e-05 2024-09-13T14:55:27.972 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.972 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.972 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:27.972 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.043149+0000", 2024-09-13T14:55:27.972 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.73494118500000005 2024-09-13T14:55:27.972 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.972 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.973 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:27.973 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.043279+0000", 2024-09-13T14:55:27.973 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013044200000000001 2024-09-13T14:55:27.973 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.973 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.973 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:55:27.973 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.043839+0000", 2024-09-13T14:55:27.973 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00056042099999999997 2024-09-13T14:55:27.973 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.973 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.973 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:55:27.973 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.047415+0000", 2024-09-13T14:55:27.973 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035758959999999998 2024-09-13T14:55:27.974 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.974 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.974 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:27.974 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.052702+0000", 2024-09-13T14:55:27.974 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0052870019999999998 2024-09-13T14:55:27.974 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.974 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.974 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:27.974 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.057268+0000", 2024-09-13T14:55:27.974 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0045653200000000003 2024-09-13T14:55:27.974 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.974 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.974 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:27.974 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.057316+0000", 2024-09-13T14:55:27.975 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8798000000000003e-05 2024-09-13T14:55:27.975 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.975 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.975 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:27.975 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.057354+0000", 2024-09-13T14:55:27.975 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7571999999999999e-05 2024-09-13T14:55:27.975 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:27.975 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:27.975 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:27.975 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.975 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.975 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156616 6.15 6:ac5310a6:::benchmark_data_smithi096_167087_object39388:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:55:27.975 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.040042+0000", 2024-09-13T14:55:27.976 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 535.90325837700004, 2024-09-13T14:55:27.976 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1222388599999999, 2024-09-13T14:55:27.976 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:27.976 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:27.976 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:55:27.976 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:55:27.976 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:27.976 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156616 2024-09-13T14:55:27.976 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.976 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:27.976 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.976 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:27.976 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040042+0000", 2024-09-13T14:55:27.976 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.977 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.977 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.977 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:27.977 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040042+0000", 2024-09-13T14:55:27.977 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.977 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.977 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.977 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:27.977 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040039+0000", 2024-09-13T14:55:27.977 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999976 2024-09-13T14:55:27.977 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.977 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.977 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:27.977 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040051+0000", 2024-09-13T14:55:27.977 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2446000000000001e-05 2024-09-13T14:55:27.978 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.978 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.978 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:27.978 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040068+0000", 2024-09-13T14:55:27.978 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6059e-05 2024-09-13T14:55:27.978 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.978 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.978 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:27.978 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040072+0000", 2024-09-13T14:55:27.978 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2370000000000003e-06 2024-09-13T14:55:27.978 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.978 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.978 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:27.979 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.309500+0000", 2024-09-13T14:55:27.979 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.26942850299999999 2024-09-13T14:55:27.979 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.979 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.979 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:55:27.979 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.309506+0000", 2024-09-13T14:55:27.979 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.049e-06 2024-09-13T14:55:27.979 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.979 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.979 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:27.979 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.158349+0000", 2024-09-13T14:55:27.979 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.84884293700000002 2024-09-13T14:55:27.979 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.979 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.980 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:27.980 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.158451+0000", 2024-09-13T14:55:27.980 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000101679 2024-09-13T14:55:27.980 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.980 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.980 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:55:27.980 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.158782+0000", 2024-09-13T14:55:27.980 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00033148900000000002 2024-09-13T14:55:27.980 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.980 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.980 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:55:27.980 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.160769+0000", 2024-09-13T14:55:27.980 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0019864119999999999 2024-09-13T14:55:27.981 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.981 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.981 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:27.981 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.161676+0000", 2024-09-13T14:55:27.981 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00090709000000000004 2024-09-13T14:55:27.981 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.981 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.981 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:27.981 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.162206+0000", 2024-09-13T14:55:27.981 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00053007 2024-09-13T14:55:27.981 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.981 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.981 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:27.981 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.162249+0000", 2024-09-13T14:55:27.982 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3250000000000001e-05 2024-09-13T14:55:27.982 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.982 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.982 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:27.982 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.162281+0000", 2024-09-13T14:55:27.982 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1241000000000001e-05 2024-09-13T14:55:27.982 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:27.982 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:27.982 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:27.982 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.982 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.982 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156618 6.8 6:13ebd03e:::benchmark_data_smithi096_167087_object39390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:55:27.982 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.040084+0000", 2024-09-13T14:55:27.982 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 535.90321607199996, 2024-09-13T14:55:27.983 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.023429146, 2024-09-13T14:55:27.983 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:27.983 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:27.983 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:55:27.983 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:55:27.983 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:27.983 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156618 2024-09-13T14:55:27.983 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.983 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:27.983 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.983 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:27.983 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040084+0000", 2024-09-13T14:55:27.983 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.983 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.984 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.984 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:27.984 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040084+0000", 2024-09-13T14:55:27.984 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.984 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.984 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.984 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:27.984 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040084+0000", 2024-09-13T14:55:27.984 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:55:27.984 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.984 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.984 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:27.984 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040087+0000", 2024-09-13T14:55:27.984 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0809999999999998e-06 2024-09-13T14:55:27.985 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.985 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.985 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:27.985 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040095+0000", 2024-09-13T14:55:27.985 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9529999999999999e-06 2024-09-13T14:55:27.985 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.985 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.985 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:27.985 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040098+0000", 2024-09-13T14:55:27.985 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4599999999999999e-06 2024-09-13T14:55:27.985 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.985 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.985 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:27.986 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.304957+0000", 2024-09-13T14:55:27.986 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.26485901699999997 2024-09-13T14:55:27.986 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.986 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.986 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:55:27.986 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.304969+0000", 2024-09-13T14:55:27.986 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1610999999999999e-05 2024-09-13T14:55:27.986 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.986 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.986 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:27.986 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.046503+0000", 2024-09-13T14:55:27.986 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.74153483200000003 2024-09-13T14:55:27.986 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.987 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.987 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:27.987 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.046619+0000", 2024-09-13T14:55:27.987 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000115473 2024-09-13T14:55:27.987 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.987 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.987 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:55:27.987 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.047071+0000", 2024-09-13T14:55:27.987 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00045168200000000002 2024-09-13T14:55:27.987 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.987 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.987 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:55:27.987 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.049454+0000", 2024-09-13T14:55:27.987 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002383399 2024-09-13T14:55:27.988 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.988 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.988 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:27.988 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.057281+0000", 2024-09-13T14:55:27.988 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0078272889999999994 2024-09-13T14:55:27.988 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.988 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.988 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:27.988 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.063435+0000", 2024-09-13T14:55:27.988 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0061532130000000003 2024-09-13T14:55:27.988 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.988 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.988 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:27.989 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.063479+0000", 2024-09-13T14:55:27.989 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4530000000000002e-05 2024-09-13T14:55:27.989 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.989 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.989 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:27.989 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.063513+0000", 2024-09-13T14:55:27.989 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4041999999999997e-05 2024-09-13T14:55:27.989 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:27.989 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:27.989 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:27.989 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.989 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.989 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45167.0:156614 6.1 e173/169)", 2024-09-13T14:55:27.989 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.040126+0000", 2024-09-13T14:55:27.990 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 535.90317410299997, 2024-09-13T14:55:27.990 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.24246019299999999, 2024-09-13T14:55:27.990 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:27.990 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:55:27.990 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:27.990 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.990 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:27.990 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040126+0000", 2024-09-13T14:55:27.990 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.990 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.991 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.991 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:27.991 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040126+0000", 2024-09-13T14:55:27.991 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.991 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.991 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.991 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:27.991 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040125+0000", 2024-09-13T14:55:27.991 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:55:27.992 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.992 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.992 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:27.992 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040134+0000", 2024-09-13T14:55:27.992 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.3610000000000007e-06 2024-09-13T14:55:27.992 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.992 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.992 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:27.992 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040144+0000", 2024-09-13T14:55:27.992 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0173000000000001e-05 2024-09-13T14:55:27.992 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.992 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.992 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:27.993 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.040148+0000", 2024-09-13T14:55:27.993 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6169999999999998e-06 2024-09-13T14:55:27.993 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.993 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.993 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:27.993 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282579+0000", 2024-09-13T14:55:27.993 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.242431116 2024-09-13T14:55:27.993 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.993 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.993 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:27.993 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.282586+0000", 2024-09-13T14:55:27.993 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.216e-06 2024-09-13T14:55:27.994 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:27.994 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:27.994 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:27.994 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.994 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.994 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:156624 6.13 6:c852d4ec:::benchmark_data_smithi096_167087_object39396:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:55:27.994 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:46:32.372301+0000", 2024-09-13T14:55:27.994 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 535.57099942499997, 2024-09-13T14:55:27.994 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.7720032, 2024-09-13T14:55:27.994 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:27.994 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:27.994 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:55:27.994 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:55:27.995 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:27.995 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 156624 2024-09-13T14:55:27.995 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.995 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:27.995 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.995 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:27.995 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372301+0000", 2024-09-13T14:55:27.995 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.995 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.995 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.995 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:27.995 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372301+0000", 2024-09-13T14:55:27.995 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:27.995 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.996 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.996 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:27.996 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372299+0000", 2024-09-13T14:55:27.996 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:55:27.996 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.996 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.996 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:27.996 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372306+0000", 2024-09-13T14:55:27.996 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.221e-06 2024-09-13T14:55:27.996 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.996 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.996 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:27.996 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372320+0000", 2024-09-13T14:55:27.997 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3293e-05 2024-09-13T14:55:27.997 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.997 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.997 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:27.997 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372325+0000", 2024-09-13T14:55:27.997 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7089999999999999e-06 2024-09-13T14:55:27.997 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.997 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.997 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:27.997 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372366+0000", 2024-09-13T14:55:27.997 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0308000000000001e-05 2024-09-13T14:55:27.997 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.997 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.998 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:55:27.998 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:32.372372+0000", 2024-09-13T14:55:27.998 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8179999999999998e-06 2024-09-13T14:55:27.998 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.998 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.998 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:27.998 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.070086+0000", 2024-09-13T14:55:27.998 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.69771321799999997 2024-09-13T14:55:27.998 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.998 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.998 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:27.998 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.070229+0000", 2024-09-13T14:55:27.998 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014322 2024-09-13T14:55:27.998 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.999 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.999 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:55:27.999 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.070517+0000", 2024-09-13T14:55:27.999 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00028809400000000002 2024-09-13T14:55:27.999 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.999 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.999 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:55:27.999 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.072169+0000", 2024-09-13T14:55:27.999 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0016519390000000001 2024-09-13T14:55:27.999 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:27.999 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:27.999 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:27.999 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.073034+0000", 2024-09-13T14:55:28.000 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00086496100000000003 2024-09-13T14:55:28.000 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.000 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.000 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:28.000 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.144218+0000", 2024-09-13T14:55:28.000 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.071183739999999995 2024-09-13T14:55:28.000 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.000 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.000 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:28.000 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.144268+0000", 2024-09-13T14:55:28.000 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0076e-05 2024-09-13T14:55:28.000 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.000 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.001 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:28.001 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:46:33.144304+0000", 2024-09-13T14:55:28.001 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6125999999999999e-05 2024-09-13T14:55:28.001 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.001 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:28.001 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.001 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.001 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.001 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:182590 6.15 6:aa32eba5:::benchmark_data_smithi096_167087_object65362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2024-09-13T14:55:28.001 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:19.162570+0000", 2024-09-13T14:55:28.001 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 488.78073020400001, 2024-09-13T14:55:28.001 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95580658399999996, 2024-09-13T14:55:28.001 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:28.002 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:28.002 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:55:28.002 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:55:28.002 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:28.002 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 182590 2024-09-13T14:55:28.002 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.002 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:28.002 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.002 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:28.002 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162570+0000", 2024-09-13T14:55:28.002 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.002 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.002 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.003 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:28.003 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162570+0000", 2024-09-13T14:55:28.003 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.003 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.003 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.003 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:28.003 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162569+0000", 2024-09-13T14:55:28.003 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:55:28.003 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.003 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.003 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:28.003 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162574+0000", 2024-09-13T14:55:28.003 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3739999999999997e-06 2024-09-13T14:55:28.004 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.004 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.004 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:28.004 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162587+0000", 2024-09-13T14:55:28.004 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2448e-05 2024-09-13T14:55:28.004 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.004 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.004 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:28.004 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162593+0000", 2024-09-13T14:55:28.004 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.669e-06 2024-09-13T14:55:28.004 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.004 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.004 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:28.005 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162634+0000", 2024-09-13T14:55:28.005 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0627000000000003e-05 2024-09-13T14:55:28.005 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.005 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.005 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:55:28.005 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.162642+0000", 2024-09-13T14:55:28.005 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2460000000000003e-06 2024-09-13T14:55:28.005 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.005 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.005 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:28.005 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.113644+0000", 2024-09-13T14:55:28.005 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95100172500000002 2024-09-13T14:55:28.005 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.005 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.006 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:28.006 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.113728+0000", 2024-09-13T14:55:28.006 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4036000000000007e-05 2024-09-13T14:55:28.006 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.006 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.006 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:55:28.006 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.113982+0000", 2024-09-13T14:55:28.006 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00025374799999999998 2024-09-13T14:55:28.006 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.006 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.006 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:28.006 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.116230+0000", 2024-09-13T14:55:28.006 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002248613 2024-09-13T14:55:28.006 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.007 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.007 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:55:28.007 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118234+0000", 2024-09-13T14:55:28.007 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0020033669999999998 2024-09-13T14:55:28.007 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.007 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.007 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:28.007 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118305+0000", 2024-09-13T14:55:28.007 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1443e-05 2024-09-13T14:55:28.007 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.007 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.007 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:28.007 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118344+0000", 2024-09-13T14:55:28.008 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8488999999999997e-05 2024-09-13T14:55:28.008 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.008 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.008 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:28.008 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118376+0000", 2024-09-13T14:55:28.008 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2659e-05 2024-09-13T14:55:28.008 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.008 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:28.008 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.008 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.008 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.008 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45167.0:182611 6.15 6:ae0b3e38:::benchmark_data_smithi096_167087_object65383:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2024-09-13T14:55:28.008 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:47:19.183753+0000", 2024-09-13T14:55:28.009 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 488.75954684099997, 2024-09-13T14:55:28.009 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93487196400000006, 2024-09-13T14:55:28.009 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:28.009 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:28.009 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:55:28.009 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45167", 2024-09-13T14:55:28.009 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:28.009 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 182611 2024-09-13T14:55:28.009 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.009 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:28.009 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.009 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:28.009 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183753+0000", 2024-09-13T14:55:28.009 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.010 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.010 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.010 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:28.010 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183753+0000", 2024-09-13T14:55:28.010 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.010 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.010 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.010 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:28.010 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183752+0000", 2024-09-13T14:55:28.010 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:55:28.010 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.010 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.010 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:28.011 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183757+0000", 2024-09-13T14:55:28.011 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0479999999999998e-06 2024-09-13T14:55:28.011 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.011 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.011 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:28.011 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183767+0000", 2024-09-13T14:55:28.011 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7410000000000004e-06 2024-09-13T14:55:28.011 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.011 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.011 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:28.011 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183771+0000", 2024-09-13T14:55:28.011 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1799999999999998e-06 2024-09-13T14:55:28.011 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.011 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.012 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:28.012 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183805+0000", 2024-09-13T14:55:28.012 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4193000000000003e-05 2024-09-13T14:55:28.012 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.012 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.012 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:55:28.012 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:19.183812+0000", 2024-09-13T14:55:28.012 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1349999999999997e-06 2024-09-13T14:55:28.012 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.012 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.012 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:28.012 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.116254+0000", 2024-09-13T14:55:28.012 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93244249599999995 2024-09-13T14:55:28.013 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.013 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.013 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:28.013 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.116358+0000", 2024-09-13T14:55:28.013 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000104342 2024-09-13T14:55:28.013 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.013 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.013 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:55:28.013 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.116545+0000", 2024-09-13T14:55:28.013 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000186152 2024-09-13T14:55:28.013 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.013 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.013 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:28.013 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118435+0000", 2024-09-13T14:55:28.014 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0018904519999999999 2024-09-13T14:55:28.014 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.014 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.014 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:55:28.014 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118536+0000", 2024-09-13T14:55:28.014 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000101067 2024-09-13T14:55:28.014 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.014 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.014 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:28.014 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:28.016 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118571+0000", 2024-09-13T14:55:28.016 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4502000000000003e-05 2024-09-13T14:55:28.016 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.016 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.016 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:28.016 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118608+0000", 2024-09-13T14:55:28.016 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7051999999999997e-05 2024-09-13T14:55:28.016 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.016 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.016 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:28.016 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:47:20.118625+0000", 2024-09-13T14:55:28.017 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7555999999999999e-05 2024-09-13T14:55:28.017 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.017 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:28.017 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.017 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.017 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.017 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:125782 7.5 7:a83dc0eb:::benchmark_data_smithi096_196313_object5651:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2024-09-13T14:55:28.017 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:50:16.277859+0000", 2024-09-13T14:55:28.017 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 311.66544136900001, 2024-09-13T14:55:28.017 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.34631596799999997, 2024-09-13T14:55:28.017 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:28.017 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:28.018 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:55:28.018 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:55:28.018 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:55:28.018 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 125782 2024-09-13T14:55:28.018 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.018 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:28.018 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.018 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:28.018 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277859+0000", 2024-09-13T14:55:28.018 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.018 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.018 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.018 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:28.019 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277859+0000", 2024-09-13T14:55:28.019 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.019 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.019 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.019 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:28.019 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277858+0000", 2024-09-13T14:55:28.019 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:55:28.019 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.019 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.019 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:28.019 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277862+0000", 2024-09-13T14:55:28.019 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6859999999999997e-06 2024-09-13T14:55:28.019 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.020 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.020 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:28.020 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277872+0000", 2024-09-13T14:55:28.020 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6989999999999998e-06 2024-09-13T14:55:28.020 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.020 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.020 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:28.020 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277876+0000", 2024-09-13T14:55:28.020 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.391e-06 2024-09-13T14:55:28.020 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.020 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.020 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:28.020 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.278833+0000", 2024-09-13T14:55:28.021 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00095691899999999998 2024-09-13T14:55:28.021 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.021 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.021 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:28.021 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.278950+0000", 2024-09-13T14:55:28.021 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000116446 2024-09-13T14:55:28.021 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.021 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.021 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:55:28.021 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.279046+0000", 2024-09-13T14:55:28.021 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6119999999999995e-05 2024-09-13T14:55:28.021 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.021 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.022 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:55:28.022 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.282554+0000", 2024-09-13T14:55:28.022 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035082289999999999 2024-09-13T14:55:28.022 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.022 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.022 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:28.022 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287518+0000", 2024-09-13T14:55:28.022 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0049643049999999996 2024-09-13T14:55:28.022 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.022 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.022 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:28.022 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624079+0000", 2024-09-13T14:55:28.022 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.33656060599999998 2024-09-13T14:55:28.022 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.023 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.023 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:28.023 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624136+0000", 2024-09-13T14:55:28.023 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6737000000000002e-05 2024-09-13T14:55:28.023 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.023 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.023 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:28.023 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624175+0000", 2024-09-13T14:55:28.023 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8862000000000002e-05 2024-09-13T14:55:28.023 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.023 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:28.023 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.023 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.023 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.024 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:125792 7.5 7:a137e5c2:::benchmark_data_smithi096_196313_object5661:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2024-09-13T14:55:28.024 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:50:16.287604+0000", 2024-09-13T14:55:28.024 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 311.65569594499999, 2024-09-13T14:55:28.024 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.33670364200000003, 2024-09-13T14:55:28.024 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:28.024 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:28.024 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:55:28.024 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:55:28.024 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:55:28.024 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 125792 2024-09-13T14:55:28.024 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.024 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:28.024 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.025 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:28.025 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287604+0000", 2024-09-13T14:55:28.025 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.025 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.025 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.025 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:28.025 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287604+0000", 2024-09-13T14:55:28.025 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.025 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.025 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.025 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:28.025 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287603+0000", 2024-09-13T14:55:28.025 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:55:28.025 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.026 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.026 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:28.026 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287609+0000", 2024-09-13T14:55:28.026 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6489999999999996e-06 2024-09-13T14:55:28.026 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.026 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.026 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:28.026 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287618+0000", 2024-09-13T14:55:28.026 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5820000000000008e-06 2024-09-13T14:55:28.026 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.026 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.026 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:28.026 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287626+0000", 2024-09-13T14:55:28.027 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1780000000000003e-06 2024-09-13T14:55:28.027 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.027 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.027 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:28.027 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287652+0000", 2024-09-13T14:55:28.027 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5395e-05 2024-09-13T14:55:28.027 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.027 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.027 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:28.027 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287735+0000", 2024-09-13T14:55:28.027 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2979000000000006e-05 2024-09-13T14:55:28.027 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.027 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.027 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:55:28.028 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287800+0000", 2024-09-13T14:55:28.028 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5335999999999999e-05 2024-09-13T14:55:28.028 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.028 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.028 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:55:28.028 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.296078+0000", 2024-09-13T14:55:28.028 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0082774900000000002 2024-09-13T14:55:28.028 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.028 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.028 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:28.028 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.299594+0000", 2024-09-13T14:55:28.028 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035163949999999999 2024-09-13T14:55:28.028 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.028 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.029 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:28.029 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624220+0000", 2024-09-13T14:55:28.029 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.32462581499999998 2024-09-13T14:55:28.029 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.029 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.029 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:28.029 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624269+0000", 2024-09-13T14:55:28.029 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9626000000000002e-05 2024-09-13T14:55:28.029 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.029 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.029 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:28.029 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624308+0000", 2024-09-13T14:55:28.029 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8374999999999998e-05 2024-09-13T14:55:28.029 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.030 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:28.030 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.030 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.030 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.030 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:125795 7.5 7:acb77273:::benchmark_data_smithi096_196313_object5664:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2024-09-13T14:55:28.030 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:50:16.290753+0000", 2024-09-13T14:55:28.030 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 311.65254749500002, 2024-09-13T14:55:28.030 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.33410231899999998, 2024-09-13T14:55:28.030 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:28.030 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:28.030 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:55:28.030 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:55:28.030 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:55:28.030 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 125795 2024-09-13T14:55:28.031 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.031 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:28.031 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.031 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:28.031 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290753+0000", 2024-09-13T14:55:28.031 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.031 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.031 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.031 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:28.031 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290753+0000", 2024-09-13T14:55:28.031 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.031 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.032 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.032 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:28.032 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290751+0000", 2024-09-13T14:55:28.032 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:55:28.032 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.032 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.032 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:28.032 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290758+0000", 2024-09-13T14:55:28.032 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.238e-06 2024-09-13T14:55:28.032 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.032 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.032 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:28.032 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290778+0000", 2024-09-13T14:55:28.032 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9643000000000001e-05 2024-09-13T14:55:28.033 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.033 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.033 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:28.033 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290786+0000", 2024-09-13T14:55:28.033 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5220000000000001e-06 2024-09-13T14:55:28.033 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.033 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.033 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:28.033 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.291631+0000", 2024-09-13T14:55:28.033 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000845617 2024-09-13T14:55:28.033 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.033 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.033 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:28.033 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.291741+0000", 2024-09-13T14:55:28.034 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000109487 2024-09-13T14:55:28.034 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.034 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.034 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:55:28.034 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.291842+0000", 2024-09-13T14:55:28.034 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000101823 2024-09-13T14:55:28.034 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.034 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.034 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:55:28.034 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.296685+0000", 2024-09-13T14:55:28.034 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0048421799999999997 2024-09-13T14:55:28.034 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.034 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.034 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:28.035 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.299630+0000", 2024-09-13T14:55:28.035 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002945544 2024-09-13T14:55:28.035 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.035 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.035 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:28.035 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624773+0000", 2024-09-13T14:55:28.035 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.32514234600000003 2024-09-13T14:55:28.035 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.035 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.035 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:28.035 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624815+0000", 2024-09-13T14:55:28.035 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2979999999999998e-05 2024-09-13T14:55:28.035 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.035 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.036 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:28.036 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624855+0000", 2024-09-13T14:55:28.036 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9347000000000001e-05 2024-09-13T14:55:28.036 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.036 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:28.036 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.036 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.036 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.036 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:172721 7.5 7:a4933ecd:::benchmark_data_smithi096_196313_object52590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-09-13T14:55:28.036 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:16.905298+0000", 2024-09-13T14:55:28.036 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 251.03800168000001, 2024-09-13T14:55:28.036 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18517154299999999, 2024-09-13T14:55:28.036 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:28.037 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:28.037 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:55:28.037 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:55:28.037 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:55:28.037 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 172721 2024-09-13T14:55:28.037 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.037 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:28.037 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.037 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:28.037 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905298+0000", 2024-09-13T14:55:28.037 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.037 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.037 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.037 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:28.037 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905298+0000", 2024-09-13T14:55:28.038 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.038 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.038 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.038 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:28.038 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905297+0000", 2024-09-13T14:55:28.038 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:55:28.038 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.038 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.038 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:28.038 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905304+0000", 2024-09-13T14:55:28.038 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5290000000000003e-06 2024-09-13T14:55:28.038 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.038 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.038 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:28.038 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905320+0000", 2024-09-13T14:55:28.039 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5852e-05 2024-09-13T14:55:28.039 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.039 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.039 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:28.039 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905327+0000", 2024-09-13T14:55:28.039 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5889999999999999e-06 2024-09-13T14:55:28.039 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.039 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.039 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:28.039 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905375+0000", 2024-09-13T14:55:28.039 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8362000000000002e-05 2024-09-13T14:55:28.039 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.039 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.039 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:28.039 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905529+0000", 2024-09-13T14:55:28.040 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015435700000000001 2024-09-13T14:55:28.040 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.040 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.040 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:55:28.040 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905659+0000", 2024-09-13T14:55:28.040 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000129231 2024-09-13T14:55:28.040 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.040 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.040 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:55:28.040 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.909873+0000", 2024-09-13T14:55:28.040 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.004214158 2024-09-13T14:55:28.040 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.040 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.040 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:28.041 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.910053+0000", 2024-09-13T14:55:28.041 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018025800000000001 2024-09-13T14:55:28.041 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.041 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.041 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:28.041 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:17.090381+0000", 2024-09-13T14:55:28.041 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18032772599999999 2024-09-13T14:55:28.041 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.041 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.041 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:28.041 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:17.090429+0000", 2024-09-13T14:55:28.041 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8335999999999999e-05 2024-09-13T14:55:28.041 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.041 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.042 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:28.042 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:17.090470+0000", 2024-09-13T14:55:28.042 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0713000000000001e-05 2024-09-13T14:55:28.042 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.042 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:28.042 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.042 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.042 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.042 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45749.0:110613 8.6 e278/270 8:6b60846c:::benchmark_data_smithi096_227542_object110612:head v 278'7071, mlcod=278'7068)", 2024-09-13T14:55:28.042 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:54:36.972975+0000", 2024-09-13T14:55:28.042 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 50.970325209000002, 2024-09-13T14:55:28.042 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.036324177999999999, 2024-09-13T14:55:28.042 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:28.043 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:28.043 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:28.043 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.043 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:28.043 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:36.972975+0000", 2024-09-13T14:55:28.043 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.043 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.043 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.043 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:28.043 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:36.972975+0000", 2024-09-13T14:55:28.043 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.043 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.043 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.044 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:28.044 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:36.972974+0000", 2024-09-13T14:55:28.044 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:55:28.044 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.044 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.044 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:28.044 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:37.002669+0000", 2024-09-13T14:55:28.044 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.029695272000000002 2024-09-13T14:55:28.044 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.045 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.045 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:28.045 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:37.002685+0000", 2024-09-13T14:55:28.045 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5079000000000001e-05 2024-09-13T14:55:28.045 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.045 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.045 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:28.045 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:37.002690+0000", 2024-09-13T14:55:28.045 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4469999999999997e-06 2024-09-13T14:55:28.045 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.045 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.045 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:28.045 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:37.002751+0000", 2024-09-13T14:55:28.046 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1260999999999995e-05 2024-09-13T14:55:28.046 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.046 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.046 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:28.046 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:37.002771+0000", 2024-09-13T14:55:28.046 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9604000000000001e-05 2024-09-13T14:55:28.046 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.046 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.046 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:28.046 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:37.009260+0000", 2024-09-13T14:55:28.046 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0064888200000000002 2024-09-13T14:55:28.046 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.046 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.046 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:28.047 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:37.009299+0000", 2024-09-13T14:55:28.047 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9332000000000002e-05 2024-09-13T14:55:28.047 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.047 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:28.047 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.047 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.047 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.047 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45749.0:128348 8.c e280/270 8:3c4f237c:::benchmark_data_smithi096_227542_object2953:head v 280'8035, mlcod=280'8032)", 2024-09-13T14:55:28.047 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:54:56.881710+0000", 2024-09-13T14:55:28.047 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 31.061589779999998, 2024-09-13T14:55:28.047 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.033350102, 2024-09-13T14:55:28.047 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:28.048 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:28.048 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:28.048 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.048 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:28.048 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.881710+0000", 2024-09-13T14:55:28.048 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.048 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.048 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.048 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:28.048 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.881710+0000", 2024-09-13T14:55:28.048 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.048 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.048 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.049 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:28.049 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.881710+0000", 2024-09-13T14:55:28.049 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:55:28.049 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.049 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.049 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:28.049 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.881714+0000", 2024-09-13T14:55:28.049 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9979999999999997e-06 2024-09-13T14:55:28.049 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.049 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.049 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:28.049 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.881719+0000", 2024-09-13T14:55:28.049 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4929999999999998e-06 2024-09-13T14:55:28.049 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.050 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.050 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:28.050 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.881723+0000", 2024-09-13T14:55:28.050 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3610000000000001e-06 2024-09-13T14:55:28.050 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.050 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.050 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:28.050 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.889561+0000", 2024-09-13T14:55:28.050 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0078386510000000003 2024-09-13T14:55:28.050 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.050 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.050 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:28.050 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.889578+0000", 2024-09-13T14:55:28.050 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6455000000000001e-05 2024-09-13T14:55:28.051 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.051 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.051 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:28.051 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.914880+0000", 2024-09-13T14:55:28.051 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.025302325000000001 2024-09-13T14:55:28.051 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.051 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.051 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:28.051 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.915060+0000", 2024-09-13T14:55:28.051 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000180077 2024-09-13T14:55:28.051 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.051 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:28.051 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.051 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.052 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.052 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45749.0:128353 8.c e280/270 8:37357f61:::benchmark_data_smithi096_227542_object2958:head v 280'8036, mlcod=280'8032)", 2024-09-13T14:55:28.052 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:54:56.881750+0000", 2024-09-13T14:55:28.052 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 31.061549741, 2024-09-13T14:55:28.052 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.033458133000000001, 2024-09-13T14:55:28.052 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:28.052 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:28.052 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:28.052 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.052 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:28.052 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.881750+0000", 2024-09-13T14:55:28.052 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.052 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.053 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.053 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:28.053 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.881750+0000", 2024-09-13T14:55:28.053 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.053 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.053 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.053 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:28.053 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.881750+0000", 2024-09-13T14:55:28.053 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967296 2024-09-13T14:55:28.053 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.053 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.053 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:28.053 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.881754+0000", 2024-09-13T14:55:28.053 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4230000000000001e-06 2024-09-13T14:55:28.054 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.054 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.054 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:28.054 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.881765+0000", 2024-09-13T14:55:28.054 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1286e-05 2024-09-13T14:55:28.054 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.054 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.054 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:28.054 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.881768+0000", 2024-09-13T14:55:28.054 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0730000000000001e-06 2024-09-13T14:55:28.054 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.054 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.054 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:28.054 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.893815+0000", 2024-09-13T14:55:28.055 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012046873 2024-09-13T14:55:28.055 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.055 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.055 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:28.055 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.893856+0000", 2024-09-13T14:55:28.055 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1047999999999997e-05 2024-09-13T14:55:28.055 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.055 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.055 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:28.055 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.915064+0000", 2024-09-13T14:55:28.055 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.021207837 2024-09-13T14:55:28.055 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.055 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.055 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:28.055 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:54:56.915208+0000", 2024-09-13T14:55:28.056 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014478200000000001 2024-09-13T14:55:28.056 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.056 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:28.056 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.056 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.056 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.056 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45749.0:137700 8.5 e281/270 8:afdacf3c:::benchmark_data_smithi096_227542_object12305:head v 281'8751, mlcod=281'8748)", 2024-09-13T14:55:28.056 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:55:07.883551+0000", 2024-09-13T14:55:28.056 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 20.059748718000002, 2024-09-13T14:55:28.056 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.037662424, 2024-09-13T14:55:28.056 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:28.056 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:28.056 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:28.056 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.056 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:28.057 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.883551+0000", 2024-09-13T14:55:28.057 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.057 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.057 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.057 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:28.057 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.883551+0000", 2024-09-13T14:55:28.057 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.057 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.057 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.057 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:28.057 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.883551+0000", 2024-09-13T14:55:28.057 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:55:28.057 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.057 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.883558+0000", 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2069999999999998e-06 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.883571+0000", 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2821e-05 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.883576+0000", 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3499999999999996e-06 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.058 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:28.059 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.893940+0000", 2024-09-13T14:55:28.059 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010363703 2024-09-13T14:55:28.059 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.059 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.059 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:28.059 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.893993+0000", 2024-09-13T14:55:28.059 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3483000000000003e-05 2024-09-13T14:55:28.059 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.059 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.059 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:28.059 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.920910+0000", 2024-09-13T14:55:28.059 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.026916426 2024-09-13T14:55:28.059 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.059 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.060 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:28.060 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:07.921214+0000", 2024-09-13T14:55:28.060 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00030415800000000001 2024-09-13T14:55:28.060 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.060 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:28.060 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.060 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.060 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.060 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45749.0:142218 8.5 e282/270 8:af4ed74e:::benchmark_data_smithi096_227542_object16823:head v 282'9041, mlcod=282'9039)", 2024-09-13T14:55:28.060 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:55:12.885750+0000", 2024-09-13T14:55:28.060 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 15.057550039000001, 2024-09-13T14:55:28.060 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.032911849, 2024-09-13T14:55:28.060 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:28.060 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:28.061 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:28.061 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.061 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:28.061 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:12.885750+0000", 2024-09-13T14:55:28.061 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.061 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.061 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.061 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:28.061 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:12.885750+0000", 2024-09-13T14:55:28.061 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.061 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.061 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.061 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:28.062 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:12.885749+0000", 2024-09-13T14:55:28.062 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:55:28.062 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.062 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.062 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:28.062 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:12.885757+0000", 2024-09-13T14:55:28.062 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6510000000000001e-06 2024-09-13T14:55:28.062 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.062 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.062 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:28.062 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:12.885771+0000", 2024-09-13T14:55:28.062 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4514e-05 2024-09-13T14:55:28.062 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.063 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.063 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:28.063 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:12.885777+0000", 2024-09-13T14:55:28.063 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5060000000000003e-06 2024-09-13T14:55:28.063 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.063 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.063 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:28.063 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:12.885813+0000", 2024-09-13T14:55:28.063 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6659000000000003e-05 2024-09-13T14:55:28.063 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.063 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.063 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:28.063 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:12.885831+0000", 2024-09-13T14:55:28.063 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7405e-05 2024-09-13T14:55:28.064 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.064 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.064 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:28.064 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:12.918546+0000", 2024-09-13T14:55:28.064 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.032715714 2024-09-13T14:55:28.064 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.064 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.064 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:28.064 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:12.918662+0000", 2024-09-13T14:55:28.064 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000115331 2024-09-13T14:55:28.064 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.064 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:28.064 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.065 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.065 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.065 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.45749.0:153702 8.5 e283/270 8:a53cbada:::benchmark_data_smithi096_227542_object28307:head v 283'9780, mlcod=283'9778)", 2024-09-13T14:55:28.065 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:55:26.523806+0000", 2024-09-13T14:55:28.065 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 1.419493643, 2024-09-13T14:55:28.065 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.031194824999999999, 2024-09-13T14:55:28.065 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:55:28.065 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:28.065 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:55:28.065 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.065 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:55:28.065 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:26.523806+0000", 2024-09-13T14:55:28.065 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.065 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.066 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.066 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:55:28.066 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:26.523806+0000", 2024-09-13T14:55:28.066 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:55:28.066 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.066 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.066 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:55:28.066 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:26.523805+0000", 2024-09-13T14:55:28.066 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:55:28.066 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.066 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.066 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:55:28.066 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:26.523813+0000", 2024-09-13T14:55:28.067 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1600000000000001e-06 2024-09-13T14:55:28.067 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.067 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.067 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:55:28.067 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:26.523824+0000", 2024-09-13T14:55:28.067 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1569000000000001e-05 2024-09-13T14:55:28.067 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.067 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.067 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:55:28.067 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:26.523830+0000", 2024-09-13T14:55:28.067 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6919999999999999e-06 2024-09-13T14:55:28.067 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.067 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.067 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:55:28.068 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:26.523963+0000", 2024-09-13T14:55:28.068 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013340199999999999 2024-09-13T14:55:28.068 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.068 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.068 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:55:28.068 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:26.523983+0000", 2024-09-13T14:55:28.068 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9354000000000001e-05 2024-09-13T14:55:28.068 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.068 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.068 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:55:28.068 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:26.554935+0000", 2024-09-13T14:55:28.068 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.030952805 2024-09-13T14:55:28.068 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:55:28.068 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:55:28.069 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:55:28.069 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:55:26.555001+0000", 2024-09-13T14:55:28.069 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5758000000000003e-05 2024-09-13T14:55:28.069 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.069 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:28.069 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.069 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:55:28.069 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:55:28.069 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:55:28.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.110 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:28.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:28.179 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:55:28.203 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:28 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/462305818' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:28.203 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:28 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:28.203 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:28 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/462305818' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:28.204 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:28 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:28.204 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:55:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:55:27] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.51.0" 2024-09-13T14:55:28.210 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:28.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:28 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/462305818' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:28.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:28 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:28.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.246 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:28.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:28.346 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:28.375 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:28.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:28.476 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:28.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.510 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:28.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:28.611 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:28.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.646 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:28.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:28.747 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:28.780 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:28.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:28.882 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:28.885 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:55:28.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.915 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:28.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:29.016 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:29.045 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:29.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:29.146 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:29.174 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:29.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:29.276 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:29.319 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:29.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:29.421 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:29.455 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:29.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:29.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:29 smithi204 ceph-mon[87071]: pgmap v890: 99 pgs: 99 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2024-09-13T14:55:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:29 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:29 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:55:29.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:29 smithi204 ceph-mon[87071]: osdmap e284: 8 total, 8 up, 8 in 2024-09-13T14:55:29.557 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:29 smithi096 ceph-mon[98901]: pgmap v890: 99 pgs: 99 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2024-09-13T14:55:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:29 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:29 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:55:29.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:29 smithi096 ceph-mon[98901]: osdmap e284: 8 total, 8 up, 8 in 2024-09-13T14:55:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:29 smithi096 ceph-mon[102928]: pgmap v890: 99 pgs: 99 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2024-09-13T14:55:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:29 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:29 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:55:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:29 smithi096 ceph-mon[102928]: osdmap e284: 8 total, 8 up, 8 in 2024-09-13T14:55:29.599 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:29.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:29.701 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:29.747 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:29.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:29.848 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:29.875 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:29.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:29.977 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:30.002 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:30.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:30.103 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:30.144 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:30.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:30.246 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:30.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.280 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:30.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:30.381 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:30.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.423 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:30.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:55:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:55:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:30.524 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:30.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.558 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:30.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:30.659 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:30.703 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:30.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:30.772 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:55:30.773 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:55:30.773 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-13T14:55:30.773 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:55:30.804 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:30.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.831 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:30.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:30.932 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:30.976 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:30.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:31.078 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:31.119 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:31.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:31.222 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:31.251 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:31.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:31.352 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:31.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.385 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:31.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:31.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:31 smithi204 ceph-mon[87071]: pgmap v892: 99 pgs: 99 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-13T14:55:31.486 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:31.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.524 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:31.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:31.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:31 smithi096 ceph-mon[98901]: pgmap v892: 99 pgs: 99 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-13T14:55:31.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:31 smithi096 ceph-mon[102928]: pgmap v892: 99 pgs: 99 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-13T14:55:31.625 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:31.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.662 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:31.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:31.763 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:31.810 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:31.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:31.912 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:31.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.958 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:31.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:32.059 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:32.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.098 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:32.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:32.200 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:32.232 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:32.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:32.334 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:32.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.370 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:32.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:32.454 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:32.462 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:55:32.462 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:32.467 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:32.470 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:32.498 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:55:32.499 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:32.527 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:32.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:32.544 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:55:32.544 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:32.578 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:55:32.578 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:32.609 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:55:32.609 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:32.630 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:32.641 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:55:32.641 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:32.659 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:32.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:32.677 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:55:32.677 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:32.713 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:55:32.713 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:32.761 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:32.795 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:32.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:32.896 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:32.926 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:32.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:33.028 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:33.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.064 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:33.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:33.165 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:33.202 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:33.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:33.304 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:33.339 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:33.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:33.441 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:33.458 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:33 smithi096 ceph-mon[98901]: pgmap v893: 99 pgs: 99 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2024-09-13T14:55:33.458 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:33 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1740577274' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:33.459 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:33 smithi096 ceph-mon[102928]: pgmap v893: 99 pgs: 99 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2024-09-13T14:55:33.459 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:33 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1740577274' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:33 smithi204 ceph-mon[87071]: pgmap v893: 99 pgs: 99 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2024-09-13T14:55:33.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:33 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1740577274' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:33.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.477 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:33.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:33.578 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:33.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.610 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:33.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:33.711 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:33.745 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:33.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:33.846 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:33.852 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:55:33.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.885 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:33.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:33.987 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:34.025 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:34.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:34.127 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:34.152 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:34.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:34.254 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:34.287 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:34.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:34.389 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:34.419 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:34.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:34.521 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:34.547 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:34.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:34.649 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:34.678 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:34.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:34.780 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:34.815 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:34.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:34.917 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:34.948 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:34.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:35.050 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:35.087 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:35.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:35.189 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:35.213 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:35.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:35.316 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:35.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.348 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:35.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:35.449 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:35.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:35 smithi204 ceph-mon[87071]: pgmap v894: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-09-13T14:55:35.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.481 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:35.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:35.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:35 smithi096 ceph-mon[98901]: pgmap v894: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-09-13T14:55:35.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:35 smithi096 ceph-mon[102928]: pgmap v894: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-09-13T14:55:35.582 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:35.624 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:35.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:35.726 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:35.761 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:35.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:35.862 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:35.897 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:35.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:35.999 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:36.031 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:36.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:36.133 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:36.160 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:36.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:36.242 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:55:36.263 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:36.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.293 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:36.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:36.394 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:36.448 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:36.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:36.550 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:36.593 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:36.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:36.695 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:36.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.731 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:36.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:36.832 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:36.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.863 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:36.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:36.964 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:36.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.999 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:37.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:37.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:55:37.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:37.075 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:36 smithi096 ceph-mon[98901]: pgmap v895: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-09-13T14:55:37.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:55:37.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:37.075 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:36 smithi096 ceph-mon[102928]: pgmap v895: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-09-13T14:55:37.101 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:37.143 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:55:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:36 smithi204 ceph-mon[87071]: pgmap v895: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-09-13T14:55:37.244 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:37.288 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:37.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:37.392 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:37.434 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:37.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:37.536 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:37.568 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:37.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:37.670 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:37.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.705 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:37.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:37.806 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:37.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.845 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:37.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:37.871 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:37.871 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:37.946 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:37.984 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:37.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:38.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:55:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:55:37] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.51.0" 2024-09-13T14:55:38.115 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:38.119 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:55:38.121 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:55:38.201 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:38.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:38.303 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:38.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.349 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:38.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:38.451 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:38.499 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:38.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:38.601 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:38.633 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:38.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:38.735 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:38.742 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:55:38.766 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:38.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:38.868 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:38.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.897 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:38.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:38.946 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:38.998 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:39.026 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:39.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:39.129 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:39.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.167 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:39.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:39.268 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:39.307 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:39.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:39.399 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:39 smithi096 ceph-mon[98901]: pgmap v896: 99 pgs: 99 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-09-13T14:55:39.400 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:39 smithi096 ceph-mon[98901]: from='client.55426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:39.400 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:39 smithi096 ceph-mon[102928]: pgmap v896: 99 pgs: 99 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-09-13T14:55:39.400 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:39 smithi096 ceph-mon[102928]: from='client.55426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:39.409 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:39.432 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:39.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:39.446 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":896,"stamp":"2024-09-13T14:55:38.016538+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5870655461,"num_objects":89535,"num_object_clones":0,"num_object_copies":268605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89535,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":162186,"num_write_kb":8037712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":8404818,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130911,"ondisk_log_size":130911,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22297748,"kb_used_data":17286868,"kb_used_omap":35,"kb_used_meta":5010716,"kb_avail":727663468,"statfs":{"total":767960285184,"available":745127391232,"internally_reserved":0,"allocated":17701752832,"data_stored":17686342744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36830,"internal_metadata":5130973218},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,2,3,6,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-595263488,"num_objects":-9083,"num_object_clones":0,"num_object_copies":-27249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9083,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004927"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":438,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.191655+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:55:28.191655+0000","last_peered":"2024-09-13T14:55:28.191655+0000","last_clean":"2024-09-13T14:55:28.191655+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:55:28.191655+0000","last_undegraded":"2024-09-13T14:55:28.191655+0000","last_fullsized":"2024-09-13T14:55:28.191655+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":250,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.144878+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:55:20.144878+0000","last_peered":"2024-09-13T14:55:20.144878+0000","last_clean":"2024-09-13T14:55:20.144878+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:55:20.144878+0000","last_undegraded":"2024-09-13T14:55:20.144878+0000","last_fullsized":"2024-09-13T14:55:20.144878+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":247,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.142276+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:55:20.142276+0000","last_peered":"2024-09-13T14:55:20.142276+0000","last_clean":"2024-09-13T14:55:20.142276+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:55:20.142276+0000","last_undegraded":"2024-09-13T14:55:20.142276+0000","last_fullsized":"2024-09-13T14:55:20.142276+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":452,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.183938+0000","last_change":"2024-09-13T14:51:44.196384+0000","last_active":"2024-09-13T14:55:28.183938+0000","last_peered":"2024-09-13T14:55:28.183938+0000","last_clean":"2024-09-13T14:55:28.183938+0000","last_became_active":"2024-09-13T14:51:44.196082+0000","last_became_peered":"2024-09-13T14:51:44.196082+0000","last_unstale":"2024-09-13T14:55:28.183938+0000","last_undegraded":"2024-09-13T14:55:28.183938+0000","last_fullsized":"2024-09-13T14:55:28.183938+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:11:03.757803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":446,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.184187+0000","last_change":"2024-09-13T14:51:44.199134+0000","last_active":"2024-09-13T14:55:28.184187+0000","last_peered":"2024-09-13T14:55:28.184187+0000","last_clean":"2024-09-13T14:55:28.184187+0000","last_became_active":"2024-09-13T14:51:44.198525+0000","last_became_peered":"2024-09-13T14:51:44.198525+0000","last_unstale":"2024-09-13T14:55:28.184187+0000","last_undegraded":"2024-09-13T14:55:28.184187+0000","last_fullsized":"2024-09-13T14:55:28.184187+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:44.009127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":251,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.145103+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T14:55:20.145103+0000","last_peered":"2024-09-13T14:55:20.145103+0000","last_clean":"2024-09-13T14:55:20.145103+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T14:55:20.145103+0000","last_undegraded":"2024-09-13T14:55:20.145103+0000","last_fullsized":"2024-09-13T14:55:20.145103+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":98,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.192139+0000","last_change":"2024-09-13T14:51:44.207246+0000","last_active":"2024-09-13T14:55:28.192139+0000","last_peered":"2024-09-13T14:55:28.192139+0000","last_clean":"2024-09-13T14:55:28.192139+0000","last_became_active":"2024-09-13T14:51:44.206637+0000","last_became_peered":"2024-09-13T14:51:44.206637+0000","last_unstale":"2024-09-13T14:55:28.192139+0000","last_undegraded":"2024-09-13T14:55:28.192139+0000","last_fullsized":"2024-09-13T14:55:28.192139+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:42.020301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":78,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.134484+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T14:55:20.134484+0000","last_peered":"2024-09-13T14:55:20.134484+0000","last_clean":"2024-09-13T14:55:20.134484+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T14:55:20.134484+0000","last_undegraded":"2024-09-13T14:55:20.134484+0000","last_fullsized":"2024-09-13T14:55:20.134484+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"284'10146","reported_seq":10168,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:37.114787+0000","last_change":"2024-09-13T14:53:13.573293+0000","last_active":"2024-09-13T14:55:37.114787+0000","last_peered":"2024-09-13T14:55:37.114787+0000","last_clean":"2024-09-13T14:55:37.114787+0000","last_became_active":"2024-09-13T14:53:13.572925+0000","last_became_peered":"2024-09-13T14:53:13.572925+0000","last_unstale":"2024-09-13T14:55:37.114787+0000","last_undegraded":"2024-09-13T14:55:37.114787+0000","last_fullsized":"2024-09-13T14:55:37.114787+0000","mapping_epoch":270,"log_start":"275'1300","ondisk_log_start":"275'1300","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":8846,"log_dups_size":0,"ondisk_log_size":8846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:13.459532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354811904,"num_objects":5414,"num_object_clones":0,"num_object_copies":16242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10146,"num_write_kb":497920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":422,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.189618+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:55:28.189618+0000","last_peered":"2024-09-13T14:55:28.189618+0000","last_clean":"2024-09-13T14:55:28.189618+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:55:28.189618+0000","last_undegraded":"2024-09-13T14:55:28.189618+0000","last_fullsized":"2024-09-13T14:55:28.189618+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":270,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.141044+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T14:55:20.141044+0000","last_peered":"2024-09-13T14:55:20.141044+0000","last_clean":"2024-09-13T14:55:20.141044+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T14:55:20.141044+0000","last_undegraded":"2024-09-13T14:55:20.141044+0000","last_fullsized":"2024-09-13T14:55:20.141044+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"284'10356","reported_seq":10378,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:36.733577+0000","last_change":"2024-09-13T14:53:13.865500+0000","last_active":"2024-09-13T14:55:36.733577+0000","last_peered":"2024-09-13T14:55:36.733577+0000","last_clean":"2024-09-13T14:55:36.733577+0000","last_became_active":"2024-09-13T14:53:13.864783+0000","last_became_peered":"2024-09-13T14:53:13.864783+0000","last_unstale":"2024-09-13T14:55:36.733577+0000","last_undegraded":"2024-09-13T14:55:36.733577+0000","last_fullsized":"2024-09-13T14:55:36.733577+0000","mapping_epoch":270,"log_start":"275'2200","ondisk_log_start":"275'2200","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":8156,"log_dups_size":0,"ondisk_log_size":8156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:08:47.267226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363724800,"num_objects":5550,"num_object_clones":0,"num_object_copies":16650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10356,"num_write_kb":508992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":250,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.145135+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:55:20.145135+0000","last_peered":"2024-09-13T14:55:20.145135+0000","last_clean":"2024-09-13T14:55:20.145135+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:55:20.145135+0000","last_undegraded":"2024-09-13T14:55:20.145135+0000","last_fullsized":"2024-09-13T14:55:20.145135+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":305,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.142100+0000","last_change":"2024-09-13T14:51:11.059483+0000","last_active":"2024-09-13T14:55:20.142100+0000","last_peered":"2024-09-13T14:55:20.142100+0000","last_clean":"2024-09-13T14:55:20.142100+0000","last_became_active":"2024-09-13T14:51:11.059262+0000","last_became_peered":"2024-09-13T14:51:11.059262+0000","last_unstale":"2024-09-13T14:55:20.142100+0000","last_undegraded":"2024-09-13T14:55:20.142100+0000","last_fullsized":"2024-09-13T14:55:20.142100+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:45:35.152631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"284'10217","reported_seq":10257,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.709517+0000","last_change":"2024-09-13T14:53:13.864168+0000","last_active":"2024-09-13T14:55:33.709517+0000","last_peered":"2024-09-13T14:55:33.709517+0000","last_clean":"2024-09-13T14:55:33.709517+0000","last_became_active":"2024-09-13T14:53:13.863816+0000","last_became_peered":"2024-09-13T14:53:13.863816+0000","last_unstale":"2024-09-13T14:55:33.709517+0000","last_undegraded":"2024-09-13T14:55:33.709517+0000","last_fullsized":"2024-09-13T14:55:33.709517+0000","mapping_epoch":270,"log_start":"275'2500","ondisk_log_start":"275'2500","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7717,"log_dups_size":0,"ondisk_log_size":7717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:08.763079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375717888,"num_objects":5733,"num_object_clones":0,"num_object_copies":17199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10217,"num_write_kb":510400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":260,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.144971+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T14:55:20.144971+0000","last_peered":"2024-09-13T14:55:20.144971+0000","last_clean":"2024-09-13T14:55:20.144971+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T14:55:20.144971+0000","last_undegraded":"2024-09-13T14:55:20.144971+0000","last_fullsized":"2024-09-13T14:55:20.144971+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":78,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.134608+0000","last_change":"2024-09-13T14:51:44.190879+0000","last_active":"2024-09-13T14:55:20.134608+0000","last_peered":"2024-09-13T14:55:20.134608+0000","last_clean":"2024-09-13T14:55:20.134608+0000","last_became_active":"2024-09-13T14:51:44.190617+0000","last_became_peered":"2024-09-13T14:51:44.190617+0000","last_unstale":"2024-09-13T14:55:20.134608+0000","last_undegraded":"2024-09-13T14:55:20.134608+0000","last_fullsized":"2024-09-13T14:55:20.134608+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:19.536298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"284'9736","reported_seq":9776,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.289590+0000","last_change":"2024-09-13T14:53:13.573599+0000","last_active":"2024-09-13T14:55:33.289590+0000","last_peered":"2024-09-13T14:55:33.289590+0000","last_clean":"2024-09-13T14:55:33.289590+0000","last_became_active":"2024-09-13T14:53:13.572977+0000","last_became_peered":"2024-09-13T14:53:13.572977+0000","last_unstale":"2024-09-13T14:55:33.289590+0000","last_undegraded":"2024-09-13T14:55:33.289590+0000","last_fullsized":"2024-09-13T14:55:33.289590+0000","mapping_epoch":270,"log_start":"275'1800","ondisk_log_start":"275'1800","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7936,"log_dups_size":0,"ondisk_log_size":7936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:54.554732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365559808,"num_objects":5578,"num_object_clones":0,"num_object_copies":16734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9736,"num_write_kb":490048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":268,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.146811+0000","last_change":"2024-09-13T14:48:32.206148+0000","last_active":"2024-09-13T14:55:20.146811+0000","last_peered":"2024-09-13T14:55:20.146811+0000","last_clean":"2024-09-13T14:55:20.146811+0000","last_became_active":"2024-09-13T14:48:32.205832+0000","last_became_peered":"2024-09-13T14:48:32.205832+0000","last_unstale":"2024-09-13T14:55:20.146811+0000","last_undegraded":"2024-09-13T14:55:20.146811+0000","last_fullsized":"2024-09-13T14:55:20.146811+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:55.026094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":432,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.182612+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:55:28.182612+0000","last_peered":"2024-09-13T14:55:28.182612+0000","last_clean":"2024-09-13T14:55:28.182612+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:55:28.182612+0000","last_undegraded":"2024-09-13T14:55:28.182612+0000","last_fullsized":"2024-09-13T14:55:28.182612+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"284'10131","reported_seq":10171,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.614900+0000","last_change":"2024-09-13T14:53:13.574753+0000","last_active":"2024-09-13T14:55:33.614900+0000","last_peered":"2024-09-13T14:55:33.614900+0000","last_clean":"2024-09-13T14:55:33.614900+0000","last_became_active":"2024-09-13T14:53:13.574453+0000","last_became_peered":"2024-09-13T14:53:13.574453+0000","last_unstale":"2024-09-13T14:55:33.614900+0000","last_undegraded":"2024-09-13T14:55:33.614900+0000","last_fullsized":"2024-09-13T14:55:33.614900+0000","mapping_epoch":270,"log_start":"275'2600","ondisk_log_start":"275'2600","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7531,"log_dups_size":0,"ondisk_log_size":7531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:56.710721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374800384,"num_objects":5719,"num_object_clones":0,"num_object_copies":17157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10131,"num_write_kb":507200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":431,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.144608+0000","last_change":"2024-09-13T14:51:01.996634+0000","last_active":"2024-09-13T14:55:20.144608+0000","last_peered":"2024-09-13T14:55:20.144608+0000","last_clean":"2024-09-13T14:55:20.144608+0000","last_became_active":"2024-09-13T14:51:01.996264+0000","last_became_peered":"2024-09-13T14:51:01.996264+0000","last_unstale":"2024-09-13T14:55:20.144608+0000","last_undegraded":"2024-09-13T14:55:20.144608+0000","last_fullsized":"2024-09-13T14:55:20.144608+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:30.083115+0000","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":435,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.191828+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:55:28.191828+0000","last_peered":"2024-09-13T14:55:28.191828+0000","last_clean":"2024-09-13T14:55:28.191828+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:55:28.191828+0000","last_undegraded":"2024-09-13T14:55:28.191828+0000","last_fullsized":"2024-09-13T14:55:28.191828+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":268,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.143083+0000","last_change":"2024-09-13T14:48:32.196993+0000","last_active":"2024-09-13T14:55:20.143083+0000","last_peered":"2024-09-13T14:55:20.143083+0000","last_clean":"2024-09-13T14:55:20.143083+0000","last_became_active":"2024-09-13T14:48:32.196681+0000","last_became_peered":"2024-09-13T14:48:32.196681+0000","last_unstale":"2024-09-13T14:55:20.143083+0000","last_undegraded":"2024-09-13T14:55:20.143083+0000","last_fullsized":"2024-09-13T14:55:20.143083+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:56.040494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"284'10027","reported_seq":10067,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.260952+0000","last_change":"2024-09-13T14:53:13.572344+0000","last_active":"2024-09-13T14:55:33.260952+0000","last_peered":"2024-09-13T14:55:33.260952+0000","last_clean":"2024-09-13T14:55:33.260952+0000","last_became_active":"2024-09-13T14:53:13.572080+0000","last_became_peered":"2024-09-13T14:53:13.572080+0000","last_unstale":"2024-09-13T14:55:33.260952+0000","last_undegraded":"2024-09-13T14:55:33.260952+0000","last_fullsized":"2024-09-13T14:55:33.260952+0000","mapping_epoch":270,"log_start":"275'2100","ondisk_log_start":"275'2100","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7927,"log_dups_size":0,"ondisk_log_size":7927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:37:22.683509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370737152,"num_objects":5657,"num_object_clones":0,"num_object_copies":16971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10027,"num_write_kb":501888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":191,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.140916+0000","last_change":"2024-09-13T14:40:27.868402+0000","last_active":"2024-09-13T14:55:20.140916+0000","last_peered":"2024-09-13T14:55:20.140916+0000","last_clean":"2024-09-13T14:55:20.140916+0000","last_became_active":"2024-09-13T14:40:22.966301+0000","last_became_peered":"2024-09-13T14:40:22.966301+0000","last_unstale":"2024-09-13T14:55:20.140916+0000","last_undegraded":"2024-09-13T14:55:20.140916+0000","last_fullsized":"2024-09-13T14:55:20.140916+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:18.660186+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":68,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.134691+0000","last_change":"2024-09-13T14:48:32.200521+0000","last_active":"2024-09-13T14:55:20.134691+0000","last_peered":"2024-09-13T14:55:20.134691+0000","last_clean":"2024-09-13T14:55:20.134691+0000","last_became_active":"2024-09-13T14:48:32.200256+0000","last_became_peered":"2024-09-13T14:48:32.200256+0000","last_unstale":"2024-09-13T14:55:20.134691+0000","last_undegraded":"2024-09-13T14:55:20.134691+0000","last_fullsized":"2024-09-13T14:55:20.134691+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:38.899629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":265,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.143145+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:55:20.143145+0000","last_peered":"2024-09-13T14:55:20.143145+0000","last_clean":"2024-09-13T14:55:20.143145+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:55:20.143145+0000","last_undegraded":"2024-09-13T14:55:20.143145+0000","last_fullsized":"2024-09-13T14:55:20.143145+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"284'10246","reported_seq":10268,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:36.977669+0000","last_change":"2024-09-13T14:53:13.572602+0000","last_active":"2024-09-13T14:55:36.977669+0000","last_peered":"2024-09-13T14:55:36.977669+0000","last_clean":"2024-09-13T14:55:36.977669+0000","last_became_active":"2024-09-13T14:53:13.572101+0000","last_became_peered":"2024-09-13T14:53:13.572101+0000","last_unstale":"2024-09-13T14:55:36.977669+0000","last_undegraded":"2024-09-13T14:55:36.977669+0000","last_fullsized":"2024-09-13T14:55:36.977669+0000","mapping_epoch":270,"log_start":"275'1900","ondisk_log_start":"275'1900","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":8346,"log_dups_size":0,"ondisk_log_size":8346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:49.928419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360841216,"num_objects":5506,"num_object_clones":0,"num_object_copies":16518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10246,"num_write_kb":504064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":419,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.182430+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:55:28.182430+0000","last_peered":"2024-09-13T14:55:28.182430+0000","last_clean":"2024-09-13T14:55:28.182430+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:55:28.182430+0000","last_undegraded":"2024-09-13T14:55:28.182430+0000","last_fullsized":"2024-09-13T14:55:28.182430+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":268,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.142561+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T14:55:20.142561+0000","last_peered":"2024-09-13T14:55:20.142561+0000","last_clean":"2024-09-13T14:55:20.142561+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T14:55:20.142561+0000","last_undegraded":"2024-09-13T14:55:20.142561+0000","last_fullsized":"2024-09-13T14:55:20.142561+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"284'9927","reported_seq":9967,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.314892+0000","last_change":"2024-09-13T14:53:13.571521+0000","last_active":"2024-09-13T14:55:33.314892+0000","last_peered":"2024-09-13T14:55:33.314892+0000","last_clean":"2024-09-13T14:55:33.314892+0000","last_became_active":"2024-09-13T14:53:13.571234+0000","last_became_peered":"2024-09-13T14:53:13.571234+0000","last_unstale":"2024-09-13T14:55:33.314892+0000","last_undegraded":"2024-09-13T14:55:33.314892+0000","last_fullsized":"2024-09-13T14:55:33.314892+0000","mapping_epoch":270,"log_start":"275'2000","ondisk_log_start":"275'2000","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7927,"log_dups_size":0,"ondisk_log_size":7927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:15.825916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371785728,"num_objects":5673,"num_object_clones":0,"num_object_copies":17019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9927,"num_write_kb":499200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":194,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.141445+0000","last_change":"2024-09-13T14:48:32.205288+0000","last_active":"2024-09-13T14:55:20.141445+0000","last_peered":"2024-09-13T14:55:20.141445+0000","last_clean":"2024-09-13T14:55:20.141445+0000","last_became_active":"2024-09-13T14:48:32.204657+0000","last_became_peered":"2024-09-13T14:48:32.204657+0000","last_unstale":"2024-09-13T14:55:20.141445+0000","last_undegraded":"2024-09-13T14:55:20.141445+0000","last_fullsized":"2024-09-13T14:55:20.141445+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:52.513329+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":92,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.193455+0000","last_change":"2024-09-13T14:51:11.059644+0000","last_active":"2024-09-13T14:55:28.193455+0000","last_peered":"2024-09-13T14:55:28.193455+0000","last_clean":"2024-09-13T14:55:28.193455+0000","last_became_active":"2024-09-13T14:51:11.058874+0000","last_became_peered":"2024-09-13T14:51:11.058874+0000","last_unstale":"2024-09-13T14:55:28.193455+0000","last_undegraded":"2024-09-13T14:55:28.193455+0000","last_fullsized":"2024-09-13T14:55:28.193455+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:15:14.736476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":420,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.187310+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:55:28.187310+0000","last_peered":"2024-09-13T14:55:28.187310+0000","last_clean":"2024-09-13T14:55:28.187310+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:55:28.187310+0000","last_undegraded":"2024-09-13T14:55:28.187310+0000","last_fullsized":"2024-09-13T14:55:28.187310+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"284'10184","reported_seq":10206,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:36.781154+0000","last_change":"2024-09-13T14:53:13.575349+0000","last_active":"2024-09-13T14:55:36.781154+0000","last_peered":"2024-09-13T14:55:36.781154+0000","last_clean":"2024-09-13T14:55:36.781154+0000","last_became_active":"2024-09-13T14:53:13.575026+0000","last_became_peered":"2024-09-13T14:53:13.575026+0000","last_unstale":"2024-09-13T14:55:36.781154+0000","last_undegraded":"2024-09-13T14:55:36.781154+0000","last_fullsized":"2024-09-13T14:55:36.781154+0000","mapping_epoch":270,"log_start":"275'2000","ondisk_log_start":"275'2000","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":8184,"log_dups_size":0,"ondisk_log_size":8184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:42:27.458209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363331584,"num_objects":5544,"num_object_clones":0,"num_object_copies":16632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10184,"num_write_kb":503296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":412,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.144748+0000","last_change":"2024-09-13T14:48:32.204164+0000","last_active":"2024-09-13T14:55:20.144748+0000","last_peered":"2024-09-13T14:55:20.144748+0000","last_clean":"2024-09-13T14:55:20.144748+0000","last_became_active":"2024-09-13T14:48:32.203815+0000","last_became_peered":"2024-09-13T14:48:32.203815+0000","last_unstale":"2024-09-13T14:55:20.144748+0000","last_undegraded":"2024-09-13T14:55:20.144748+0000","last_fullsized":"2024-09-13T14:55:20.144748+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:22.639760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:22.639760+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:21.985178+0000","scrub_duration":0.0051120610000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":810,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.140882+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:55:20.140882+0000","last_peered":"2024-09-13T14:55:20.140882+0000","last_clean":"2024-09-13T14:55:20.140882+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:55:20.140882+0000","last_undegraded":"2024-09-13T14:55:20.140882+0000","last_fullsized":"2024-09-13T14:55:20.140882+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":92,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.190985+0000","last_change":"2024-09-13T14:51:11.059953+0000","last_active":"2024-09-13T14:55:28.190985+0000","last_peered":"2024-09-13T14:55:28.190985+0000","last_clean":"2024-09-13T14:55:28.190985+0000","last_became_active":"2024-09-13T14:51:11.059396+0000","last_became_peered":"2024-09-13T14:51:11.059396+0000","last_unstale":"2024-09-13T14:55:28.190985+0000","last_undegraded":"2024-09-13T14:55:28.190985+0000","last_fullsized":"2024-09-13T14:55:28.190985+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:11:51.119142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":68,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.140015+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T14:55:20.140015+0000","last_peered":"2024-09-13T14:55:20.140015+0000","last_clean":"2024-09-13T14:55:20.140015+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T14:55:20.140015+0000","last_undegraded":"2024-09-13T14:55:20.140015+0000","last_fullsized":"2024-09-13T14:55:20.140015+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"284'9955","reported_seq":9995,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.304118+0000","last_change":"2024-09-13T14:53:13.569268+0000","last_active":"2024-09-13T14:55:33.304118+0000","last_peered":"2024-09-13T14:55:33.304118+0000","last_clean":"2024-09-13T14:55:33.304118+0000","last_became_active":"2024-09-13T14:53:13.568997+0000","last_became_peered":"2024-09-13T14:53:13.568997+0000","last_unstale":"2024-09-13T14:55:33.304118+0000","last_undegraded":"2024-09-13T14:55:33.304118+0000","last_fullsized":"2024-09-13T14:55:33.304118+0000","mapping_epoch":270,"log_start":"275'2000","ondisk_log_start":"275'2000","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7955,"log_dups_size":0,"ondisk_log_size":7955,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:20:18.821269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369557504,"num_objects":5639,"num_object_clones":0,"num_object_copies":16917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9955,"num_write_kb":499008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":559,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.185733+0000","last_change":"2024-09-13T14:48:32.204587+0000","last_active":"2024-09-13T14:55:28.185733+0000","last_peered":"2024-09-13T14:55:28.185733+0000","last_clean":"2024-09-13T14:55:28.185733+0000","last_became_active":"2024-09-13T14:48:32.204153+0000","last_became_peered":"2024-09-13T14:48:32.204153+0000","last_unstale":"2024-09-13T14:55:28.185733+0000","last_undegraded":"2024-09-13T14:55:28.185733+0000","last_fullsized":"2024-09-13T14:55:28.185733+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.784263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.784263+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:11:12.651038+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":495,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.144665+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:55:20.144665+0000","last_peered":"2024-09-13T14:55:20.144665+0000","last_clean":"2024-09-13T14:55:20.144665+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:55:20.144665+0000","last_undegraded":"2024-09-13T14:55:20.144665+0000","last_fullsized":"2024-09-13T14:55:20.144665+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":454,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.182530+0000","last_change":"2024-09-13T14:51:11.061677+0000","last_active":"2024-09-13T14:55:28.182530+0000","last_peered":"2024-09-13T14:55:28.182530+0000","last_clean":"2024-09-13T14:55:28.182530+0000","last_became_active":"2024-09-13T14:51:11.061205+0000","last_became_peered":"2024-09-13T14:51:11.061205+0000","last_unstale":"2024-09-13T14:55:28.182530+0000","last_undegraded":"2024-09-13T14:55:28.182530+0000","last_fullsized":"2024-09-13T14:55:28.182530+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:38.521784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":458,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.184356+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T14:55:28.184356+0000","last_peered":"2024-09-13T14:55:28.184356+0000","last_clean":"2024-09-13T14:55:28.184356+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T14:55:28.184356+0000","last_undegraded":"2024-09-13T14:55:28.184356+0000","last_fullsized":"2024-09-13T14:55:28.184356+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"284'10181","reported_seq":10221,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.664594+0000","last_change":"2024-09-13T14:53:13.571649+0000","last_active":"2024-09-13T14:55:33.664594+0000","last_peered":"2024-09-13T14:55:33.664594+0000","last_clean":"2024-09-13T14:55:33.664594+0000","last_became_active":"2024-09-13T14:53:13.571377+0000","last_became_peered":"2024-09-13T14:53:13.571377+0000","last_unstale":"2024-09-13T14:55:33.664594+0000","last_undegraded":"2024-09-13T14:55:33.664594+0000","last_fullsized":"2024-09-13T14:55:33.664594+0000","mapping_epoch":270,"log_start":"275'2600","ondisk_log_start":"275'2600","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7581,"log_dups_size":0,"ondisk_log_size":7581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:34:54.743293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375848960,"num_objects":5735,"num_object_clones":0,"num_object_copies":17205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10181,"num_write_kb":509312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":41,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.144772+0000","last_change":"2024-09-13T14:51:44.190446+0000","last_active":"2024-09-13T14:55:20.144772+0000","last_peered":"2024-09-13T14:55:20.144772+0000","last_clean":"2024-09-13T14:55:20.144772+0000","last_became_active":"2024-09-13T14:51:44.190115+0000","last_became_peered":"2024-09-13T14:51:44.190115+0000","last_unstale":"2024-09-13T14:55:20.144772+0000","last_undegraded":"2024-09-13T14:55:20.144772+0000","last_fullsized":"2024-09-13T14:55:20.144772+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:45:49.425578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:45:49.425578+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:43.724060+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":906,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.190230+0000","last_change":"2024-09-13T14:51:11.072066+0000","last_active":"2024-09-13T14:55:28.190230+0000","last_peered":"2024-09-13T14:55:28.190230+0000","last_clean":"2024-09-13T14:55:28.190230+0000","last_became_active":"2024-09-13T14:51:11.061307+0000","last_became_peered":"2024-09-13T14:51:11.061307+0000","last_unstale":"2024-09-13T14:55:28.190230+0000","last_undegraded":"2024-09-13T14:55:28.190230+0000","last_fullsized":"2024-09-13T14:55:28.190230+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:06.774106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":435,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.186980+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T14:55:28.186980+0000","last_peered":"2024-09-13T14:55:28.186980+0000","last_clean":"2024-09-13T14:55:28.186980+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T14:55:28.186980+0000","last_undegraded":"2024-09-13T14:55:28.186980+0000","last_fullsized":"2024-09-13T14:55:28.186980+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":453,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.187241+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:55:28.187241+0000","last_peered":"2024-09-13T14:55:28.187241+0000","last_clean":"2024-09-13T14:55:28.187241+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:55:28.187241+0000","last_undegraded":"2024-09-13T14:55:28.187241+0000","last_fullsized":"2024-09-13T14:55:28.187241+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"284'9972","reported_seq":9994,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:37.080045+0000","last_change":"2024-09-13T14:53:13.864463+0000","last_active":"2024-09-13T14:55:37.080045+0000","last_peered":"2024-09-13T14:55:37.080045+0000","last_clean":"2024-09-13T14:55:37.080045+0000","last_became_active":"2024-09-13T14:53:13.864138+0000","last_became_peered":"2024-09-13T14:53:13.864138+0000","last_unstale":"2024-09-13T14:55:37.080045+0000","last_undegraded":"2024-09-13T14:55:37.080045+0000","last_fullsized":"2024-09-13T14:55:37.080045+0000","mapping_epoch":270,"log_start":"274'1100","ondisk_log_start":"274'1100","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":8872,"log_dups_size":0,"ondisk_log_size":8872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:40.441931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354811904,"num_objects":5414,"num_object_clones":0,"num_object_copies":16242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9972,"num_write_kb":492352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":184,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.140684+0000","last_change":"2024-09-13T14:40:26.896691+0000","last_active":"2024-09-13T14:55:20.140684+0000","last_peered":"2024-09-13T14:55:20.140684+0000","last_clean":"2024-09-13T14:55:20.140684+0000","last_became_active":"2024-09-13T14:40:22.949772+0000","last_became_peered":"2024-09-13T14:40:22.949772+0000","last_unstale":"2024-09-13T14:55:20.140684+0000","last_undegraded":"2024-09-13T14:55:20.140684+0000","last_fullsized":"2024-09-13T14:55:20.140684+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:26.896613+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:26.896613+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:01.615433+0000","scrub_duration":0.004120828,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":910,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.141580+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T14:55:20.141580+0000","last_peered":"2024-09-13T14:55:20.141580+0000","last_clean":"2024-09-13T14:55:20.141580+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T14:55:20.141580+0000","last_undegraded":"2024-09-13T14:55:20.141580+0000","last_fullsized":"2024-09-13T14:55:20.141580+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":269,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.142653+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T14:55:20.142653+0000","last_peered":"2024-09-13T14:55:20.142653+0000","last_clean":"2024-09-13T14:55:20.142653+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T14:55:20.142653+0000","last_undegraded":"2024-09-13T14:55:20.142653+0000","last_fullsized":"2024-09-13T14:55:20.142653+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":452,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.185073+0000","last_change":"2024-09-13T14:51:11.060994+0000","last_active":"2024-09-13T14:55:28.185073+0000","last_peered":"2024-09-13T14:55:28.185073+0000","last_clean":"2024-09-13T14:55:28.185073+0000","last_became_active":"2024-09-13T14:51:11.060666+0000","last_became_peered":"2024-09-13T14:51:11.060666+0000","last_unstale":"2024-09-13T14:55:28.185073+0000","last_undegraded":"2024-09-13T14:55:28.185073+0000","last_fullsized":"2024-09-13T14:55:28.185073+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:40:15.828766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"284'10049","reported_seq":10089,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.738905+0000","last_change":"2024-09-13T14:53:13.572932+0000","last_active":"2024-09-13T14:55:33.738905+0000","last_peered":"2024-09-13T14:55:33.738905+0000","last_clean":"2024-09-13T14:55:33.738905+0000","last_became_active":"2024-09-13T14:53:13.572464+0000","last_became_peered":"2024-09-13T14:53:13.572464+0000","last_unstale":"2024-09-13T14:55:33.738905+0000","last_undegraded":"2024-09-13T14:55:33.738905+0000","last_fullsized":"2024-09-13T14:55:33.738905+0000","mapping_epoch":270,"log_start":"275'2300","ondisk_log_start":"275'2300","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7749,"log_dups_size":0,"ondisk_log_size":7749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:47:06.440193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372310016,"num_objects":5681,"num_object_clones":0,"num_object_copies":17043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10049,"num_write_kb":503360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":88,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.135567+0000","last_change":"2024-09-13T14:51:11.061598+0000","last_active":"2024-09-13T14:55:20.135567+0000","last_peered":"2024-09-13T14:55:20.135567+0000","last_clean":"2024-09-13T14:55:20.135567+0000","last_became_active":"2024-09-13T14:51:11.061164+0000","last_became_peered":"2024-09-13T14:51:11.061164+0000","last_unstale":"2024-09-13T14:55:20.135567+0000","last_undegraded":"2024-09-13T14:55:20.135567+0000","last_fullsized":"2024-09-13T14:55:20.135567+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:48:34.095505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:48:34.095505+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:03.076025+0000","scrub_duration":0.001382681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":265,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.142498+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:55:20.142498+0000","last_peered":"2024-09-13T14:55:20.142498+0000","last_clean":"2024-09-13T14:55:20.142498+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:55:20.142498+0000","last_undegraded":"2024-09-13T14:55:20.142498+0000","last_fullsized":"2024-09-13T14:55:20.142498+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":937,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.186351+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:55:28.186351+0000","last_peered":"2024-09-13T14:55:28.186351+0000","last_clean":"2024-09-13T14:55:28.186351+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:55:28.186351+0000","last_undegraded":"2024-09-13T14:55:28.186351+0000","last_fullsized":"2024-09-13T14:55:28.186351+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":409,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.186815+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:55:28.186815+0000","last_peered":"2024-09-13T14:55:28.186815+0000","last_clean":"2024-09-13T14:55:28.186815+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:55:28.186815+0000","last_undegraded":"2024-09-13T14:55:28.186815+0000","last_fullsized":"2024-09-13T14:55:28.186815+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"284'10022","reported_seq":10062,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.683760+0000","last_change":"2024-09-13T14:53:13.575181+0000","last_active":"2024-09-13T14:55:33.683760+0000","last_peered":"2024-09-13T14:55:33.683760+0000","last_clean":"2024-09-13T14:55:33.683760+0000","last_became_active":"2024-09-13T14:53:13.574893+0000","last_became_peered":"2024-09-13T14:53:13.574893+0000","last_unstale":"2024-09-13T14:55:33.683760+0000","last_undegraded":"2024-09-13T14:55:33.683760+0000","last_fullsized":"2024-09-13T14:55:33.683760+0000","mapping_epoch":270,"log_start":"275'2500","ondisk_log_start":"275'2500","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7522,"log_dups_size":0,"ondisk_log_size":7522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:38:28.331899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372375552,"num_objects":5682,"num_object_clones":0,"num_object_copies":17046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10022,"num_write_kb":502528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":78,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.135430+0000","last_change":"2024-09-13T14:51:11.060132+0000","last_active":"2024-09-13T14:55:20.135430+0000","last_peered":"2024-09-13T14:55:20.135430+0000","last_clean":"2024-09-13T14:55:20.135430+0000","last_became_active":"2024-09-13T14:51:11.059785+0000","last_became_peered":"2024-09-13T14:51:11.059785+0000","last_unstale":"2024-09-13T14:55:20.135430+0000","last_undegraded":"2024-09-13T14:55:20.135430+0000","last_fullsized":"2024-09-13T14:55:20.135430+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:23.334379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:23.334379+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:44.298431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":434,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.185173+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T14:55:28.185173+0000","last_peered":"2024-09-13T14:55:28.185173+0000","last_clean":"2024-09-13T14:55:28.185173+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T14:55:28.185173+0000","last_undegraded":"2024-09-13T14:55:28.185173+0000","last_fullsized":"2024-09-13T14:55:28.185173+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":497,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.145049+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:55:20.145049+0000","last_peered":"2024-09-13T14:55:20.145049+0000","last_clean":"2024-09-13T14:55:20.145049+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:55:20.145049+0000","last_undegraded":"2024-09-13T14:55:20.145049+0000","last_fullsized":"2024-09-13T14:55:20.145049+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":422,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.182529+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:55:28.182529+0000","last_peered":"2024-09-13T14:55:28.182529+0000","last_clean":"2024-09-13T14:55:28.182529+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:55:28.182529+0000","last_undegraded":"2024-09-13T14:55:28.182529+0000","last_fullsized":"2024-09-13T14:55:28.182529+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"284'10105","reported_seq":10145,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.698061+0000","last_change":"2024-09-13T14:53:13.864099+0000","last_active":"2024-09-13T14:55:33.698061+0000","last_peered":"2024-09-13T14:55:33.698061+0000","last_clean":"2024-09-13T14:55:33.698061+0000","last_became_active":"2024-09-13T14:53:13.863777+0000","last_became_peered":"2024-09-13T14:53:13.863777+0000","last_unstale":"2024-09-13T14:55:33.698061+0000","last_undegraded":"2024-09-13T14:55:33.698061+0000","last_fullsized":"2024-09-13T14:55:33.698061+0000","mapping_epoch":270,"log_start":"275'2400","ondisk_log_start":"275'2400","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7705,"log_dups_size":0,"ondisk_log_size":7705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:33.743908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367198208,"num_objects":5603,"num_object_clones":0,"num_object_copies":16809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10105,"num_write_kb":502656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":215,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.143018+0000","last_change":"2024-09-13T14:51:01.998101+0000","last_active":"2024-09-13T14:55:20.143018+0000","last_peered":"2024-09-13T14:55:20.143018+0000","last_clean":"2024-09-13T14:55:20.143018+0000","last_became_active":"2024-09-13T14:51:01.997688+0000","last_became_peered":"2024-09-13T14:51:01.997688+0000","last_unstale":"2024-09-13T14:55:20.143018+0000","last_undegraded":"2024-09-13T14:55:20.143018+0000","last_fullsized":"2024-09-13T14:55:20.143018+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:40:30.500046+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:02:51.444049+0000","scrub_duration":0.00248832,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":436,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.192254+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:55:28.192254+0000","last_peered":"2024-09-13T14:55:28.192254+0000","last_clean":"2024-09-13T14:55:28.192254+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:55:28.192254+0000","last_undegraded":"2024-09-13T14:55:28.192254+0000","last_fullsized":"2024-09-13T14:55:28.192254+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":792,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.186196+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T14:55:28.186196+0000","last_peered":"2024-09-13T14:55:28.186196+0000","last_clean":"2024-09-13T14:55:28.186196+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T14:55:28.186196+0000","last_undegraded":"2024-09-13T14:55:28.186196+0000","last_fullsized":"2024-09-13T14:55:28.186196+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":99,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.193067+0000","last_change":"2024-09-13T14:51:44.203186+0000","last_active":"2024-09-13T14:55:28.193067+0000","last_peered":"2024-09-13T14:55:28.193067+0000","last_clean":"2024-09-13T14:55:28.193067+0000","last_became_active":"2024-09-13T14:51:44.202851+0000","last_became_peered":"2024-09-13T14:51:44.202851+0000","last_unstale":"2024-09-13T14:55:28.193067+0000","last_undegraded":"2024-09-13T14:55:28.193067+0000","last_fullsized":"2024-09-13T14:55:28.193067+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:05:49.173340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"284'10025","reported_seq":10048,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:37.101481+0000","last_change":"2024-09-13T14:53:13.864419+0000","last_active":"2024-09-13T14:55:37.101481+0000","last_peered":"2024-09-13T14:55:37.101481+0000","last_clean":"2024-09-13T14:55:37.101481+0000","last_became_active":"2024-09-13T14:53:13.864057+0000","last_became_peered":"2024-09-13T14:53:13.864057+0000","last_unstale":"2024-09-13T14:55:37.101481+0000","last_undegraded":"2024-09-13T14:55:37.101481+0000","last_fullsized":"2024-09-13T14:55:37.101481+0000","mapping_epoch":270,"log_start":"274'1200","ondisk_log_start":"274'1200","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":8825,"log_dups_size":0,"ondisk_log_size":8825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:24.873156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352452632,"num_objects":5379,"num_object_clones":0,"num_object_copies":16137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5379,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10025,"num_write_kb":492865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":366,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.191364+0000","last_change":"2024-09-13T14:40:24.124750+0000","last_active":"2024-09-13T14:55:28.191364+0000","last_peered":"2024-09-13T14:55:28.191364+0000","last_clean":"2024-09-13T14:55:28.191364+0000","last_became_active":"2024-09-13T14:40:22.958820+0000","last_became_peered":"2024-09-13T14:40:22.958820+0000","last_unstale":"2024-09-13T14:55:28.191364+0000","last_undegraded":"2024-09-13T14:55:28.191364+0000","last_fullsized":"2024-09-13T14:55:28.191364+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:24.124531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:24.124531+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:47:30.267886+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":444,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.184091+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:55:28.184091+0000","last_peered":"2024-09-13T14:55:28.184091+0000","last_clean":"2024-09-13T14:55:28.184091+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:55:28.184091+0000","last_undegraded":"2024-09-13T14:55:28.184091+0000","last_fullsized":"2024-09-13T14:55:28.184091+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1097,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.135700+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T14:55:20.135700+0000","last_peered":"2024-09-13T14:55:20.135700+0000","last_clean":"2024-09-13T14:55:20.135700+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T14:55:20.135700+0000","last_undegraded":"2024-09-13T14:55:20.135700+0000","last_fullsized":"2024-09-13T14:55:20.135700+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":251,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.142031+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T14:55:20.142031+0000","last_peered":"2024-09-13T14:55:20.142031+0000","last_clean":"2024-09-13T14:55:20.142031+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T14:55:20.142031+0000","last_undegraded":"2024-09-13T14:55:20.142031+0000","last_fullsized":"2024-09-13T14:55:20.142031+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":268,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.142599+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T14:55:20.142599+0000","last_peered":"2024-09-13T14:55:20.142599+0000","last_clean":"2024-09-13T14:55:20.142599+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T14:55:20.142599+0000","last_undegraded":"2024-09-13T14:55:20.142599+0000","last_fullsized":"2024-09-13T14:55:20.142599+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":451,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.182742+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T14:55:28.182742+0000","last_peered":"2024-09-13T14:55:28.182742+0000","last_clean":"2024-09-13T14:55:28.182742+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T14:55:28.182742+0000","last_undegraded":"2024-09-13T14:55:28.182742+0000","last_fullsized":"2024-09-13T14:55:28.182742+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":433,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.186689+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T14:55:28.186689+0000","last_peered":"2024-09-13T14:55:28.186689+0000","last_clean":"2024-09-13T14:55:28.186689+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T14:55:28.186689+0000","last_undegraded":"2024-09-13T14:55:28.186689+0000","last_fullsized":"2024-09-13T14:55:28.186689+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":264,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.145159+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:55:20.145159+0000","last_peered":"2024-09-13T14:55:20.145159+0000","last_clean":"2024-09-13T14:55:20.145159+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:55:20.145159+0000","last_undegraded":"2024-09-13T14:55:20.145159+0000","last_fullsized":"2024-09-13T14:55:20.145159+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":248,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.140690+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:55:20.140690+0000","last_peered":"2024-09-13T14:55:20.140690+0000","last_clean":"2024-09-13T14:55:20.140690+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:55:20.140690+0000","last_undegraded":"2024-09-13T14:55:20.140690+0000","last_fullsized":"2024-09-13T14:55:20.140690+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":403,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.190818+0000","last_change":"2024-09-13T14:48:32.200953+0000","last_active":"2024-09-13T14:55:28.190818+0000","last_peered":"2024-09-13T14:55:28.190818+0000","last_clean":"2024-09-13T14:55:28.190818+0000","last_became_active":"2024-09-13T14:48:32.200634+0000","last_became_peered":"2024-09-13T14:48:32.200634+0000","last_unstale":"2024-09-13T14:55:28.190818+0000","last_undegraded":"2024-09-13T14:55:28.190818+0000","last_fullsized":"2024-09-13T14:55:28.190818+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:18.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":431,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.183682+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:55:28.183682+0000","last_peered":"2024-09-13T14:55:28.183682+0000","last_clean":"2024-09-13T14:55:28.183682+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:55:28.183682+0000","last_undegraded":"2024-09-13T14:55:28.183682+0000","last_fullsized":"2024-09-13T14:55:28.183682+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":452,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.193294+0000","last_change":"2024-09-13T14:48:32.201812+0000","last_active":"2024-09-13T14:55:28.193294+0000","last_peered":"2024-09-13T14:55:28.193294+0000","last_clean":"2024-09-13T14:55:28.193294+0000","last_became_active":"2024-09-13T14:48:32.201486+0000","last_became_peered":"2024-09-13T14:48:32.201486+0000","last_unstale":"2024-09-13T14:55:28.193294+0000","last_undegraded":"2024-09-13T14:55:28.193294+0000","last_fullsized":"2024-09-13T14:55:28.193294+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:23:29.693906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.135630+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T14:55:20.135630+0000","last_peered":"2024-09-13T14:55:20.135630+0000","last_clean":"2024-09-13T14:55:20.135630+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T14:55:20.135630+0000","last_undegraded":"2024-09-13T14:55:20.135630+0000","last_fullsized":"2024-09-13T14:55:20.135630+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":456,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.193207+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:55:28.193207+0000","last_peered":"2024-09-13T14:55:28.193207+0000","last_clean":"2024-09-13T14:55:28.193207+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:55:28.193207+0000","last_undegraded":"2024-09-13T14:55:28.193207+0000","last_fullsized":"2024-09-13T14:55:28.193207+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":454,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.184264+0000","last_change":"2024-09-13T14:51:44.198771+0000","last_active":"2024-09-13T14:55:28.184264+0000","last_peered":"2024-09-13T14:55:28.184264+0000","last_clean":"2024-09-13T14:55:28.184264+0000","last_became_active":"2024-09-13T14:51:44.198466+0000","last_became_peered":"2024-09-13T14:51:44.198466+0000","last_unstale":"2024-09-13T14:55:28.184264+0000","last_undegraded":"2024-09-13T14:55:28.184264+0000","last_fullsized":"2024-09-13T14:55:28.184264+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:41:16.099954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":419,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.187948+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:55:28.187948+0000","last_peered":"2024-09-13T14:55:28.187948+0000","last_clean":"2024-09-13T14:55:28.187948+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:55:28.187948+0000","last_undegraded":"2024-09-13T14:55:28.187948+0000","last_fullsized":"2024-09-13T14:55:28.187948+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":451,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.191621+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:55:28.191621+0000","last_peered":"2024-09-13T14:55:28.191621+0000","last_clean":"2024-09-13T14:55:28.191621+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:55:28.191621+0000","last_undegraded":"2024-09-13T14:55:28.191621+0000","last_fullsized":"2024-09-13T14:55:28.191621+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":267,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.142471+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:55:20.142471+0000","last_peered":"2024-09-13T14:55:20.142471+0000","last_clean":"2024-09-13T14:55:20.142471+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:55:20.142471+0000","last_undegraded":"2024-09-13T14:55:20.142471+0000","last_fullsized":"2024-09-13T14:55:20.142471+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":95,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.188197+0000","last_change":"2024-09-13T14:51:02.029301+0000","last_active":"2024-09-13T14:55:28.188197+0000","last_peered":"2024-09-13T14:55:28.188197+0000","last_clean":"2024-09-13T14:55:28.188197+0000","last_became_active":"2024-09-13T14:51:02.029032+0000","last_became_peered":"2024-09-13T14:51:02.029032+0000","last_unstale":"2024-09-13T14:55:28.188197+0000","last_undegraded":"2024-09-13T14:55:28.188197+0000","last_fullsized":"2024-09-13T14:55:28.188197+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:40.150320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":423,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.182438+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:55:28.182438+0000","last_peered":"2024-09-13T14:55:28.182438+0000","last_clean":"2024-09-13T14:55:28.182438+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:55:28.182438+0000","last_undegraded":"2024-09-13T14:55:28.182438+0000","last_fullsized":"2024-09-13T14:55:28.182438+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":265,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.142585+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:55:20.142585+0000","last_peered":"2024-09-13T14:55:20.142585+0000","last_clean":"2024-09-13T14:55:20.142585+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:55:20.142585+0000","last_undegraded":"2024-09-13T14:55:20.142585+0000","last_fullsized":"2024-09-13T14:55:20.142585+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":250,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.146815+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:55:20.146815+0000","last_peered":"2024-09-13T14:55:20.146815+0000","last_clean":"2024-09-13T14:55:20.146815+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:55:20.146815+0000","last_undegraded":"2024-09-13T14:55:20.146815+0000","last_fullsized":"2024-09-13T14:55:20.146815+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":265,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.142396+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:55:20.142396+0000","last_peered":"2024-09-13T14:55:20.142396+0000","last_clean":"2024-09-13T14:55:20.142396+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:55:20.142396+0000","last_undegraded":"2024-09-13T14:55:20.142396+0000","last_fullsized":"2024-09-13T14:55:20.142396+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":289,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.142432+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T14:55:20.142432+0000","last_peered":"2024-09-13T14:55:20.142432+0000","last_clean":"2024-09-13T14:55:20.142432+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T14:55:20.142432+0000","last_undegraded":"2024-09-13T14:55:20.142432+0000","last_fullsized":"2024-09-13T14:55:20.142432+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":239,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.140799+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:55:20.140799+0000","last_peered":"2024-09-13T14:55:20.140799+0000","last_clean":"2024-09-13T14:55:20.140799+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:55:20.140799+0000","last_undegraded":"2024-09-13T14:55:20.140799+0000","last_fullsized":"2024-09-13T14:55:20.140799+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":459,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.192377+0000","last_change":"2024-09-13T14:48:32.202404+0000","last_active":"2024-09-13T14:55:28.192377+0000","last_peered":"2024-09-13T14:55:28.192377+0000","last_clean":"2024-09-13T14:55:28.192377+0000","last_became_active":"2024-09-13T14:48:32.202064+0000","last_became_peered":"2024-09-13T14:48:32.202064+0000","last_unstale":"2024-09-13T14:55:28.192377+0000","last_undegraded":"2024-09-13T14:55:28.192377+0000","last_fullsized":"2024-09-13T14:55:28.192377+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:25.674819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":271,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.141949+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T14:55:20.141949+0000","last_peered":"2024-09-13T14:55:20.141949+0000","last_clean":"2024-09-13T14:55:20.141949+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T14:55:20.141949+0000","last_undegraded":"2024-09-13T14:55:20.141949+0000","last_fullsized":"2024-09-13T14:55:20.141949+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":475,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.183560+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T14:55:28.183560+0000","last_peered":"2024-09-13T14:55:28.183560+0000","last_clean":"2024-09-13T14:55:28.183560+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T14:55:28.183560+0000","last_undegraded":"2024-09-13T14:55:28.183560+0000","last_fullsized":"2024-09-13T14:55:28.183560+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5865865240,"num_objects":89507,"num_object_clones":0,"num_object_copies":268521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89507,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":161279,"num_write_kb":8025089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17530302464,"data_stored":17530290248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128779,"ondisk_log_size":128779,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":16172,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397860,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2361728,"kb_used_data":1763512,"kb_used_omap":1,"kb_used_meta":598206,"kb_avail":91383424,"statfs":{"total":95995035648,"available":93576626176,"internally_reserved":0,"allocated":1805836288,"data_stored":1803898995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612563288},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594089,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2088500,"kb_used_data":1778668,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91656652,"statfs":{"total":95995035648,"available":93856411648,"internally_reserved":0,"allocated":1821356032,"data_stored":1819430575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790318,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3704436,"kb_used_data":2819884,"kb_used_omap":1,"kb_used_meta":884542,"kb_avail":90040716,"statfs":{"total":95995035648,"available":92201693184,"internally_reserved":0,"allocated":2887561216,"data_stored":2885632224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":905771352},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986547,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2373680,"kb_used_data":1779624,"kb_used_omap":1,"kb_used_meta":594046,"kb_avail":91371472,"statfs":{"total":95995035648,"available":93564387328,"internally_reserved":0,"allocated":1822334976,"data_stored":1820409243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":608303448},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676598994,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2852516,"kb_used_data":2192732,"kb_used_omap":6,"kb_used_meta":659769,"kb_avail":90892636,"statfs":{"total":95995035648,"available":93074059264,"internally_reserved":0,"allocated":2245357568,"data_stored":2243444305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":675604448},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762521,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2896500,"kb_used_data":2178284,"kb_used_omap":8,"kb_used_meta":718199,"kb_avail":90848652,"statfs":{"total":95995035648,"available":93029019648,"internally_reserved":0,"allocated":2230562816,"data_stored":2228633819,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":735436234},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926047,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3212072,"kb_used_data":2572768,"kb_used_omap":7,"kb_used_meta":639224,"kb_avail":90533080,"statfs":{"total":95995035648,"available":92705873920,"internally_reserved":0,"allocated":2634514432,"data_stored":2632596792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":654566089},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,3,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122287,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2808316,"kb_used_data":2201396,"kb_used_omap":7,"kb_used_meta":606904,"kb_avail":90936836,"statfs":{"total":95995035648,"available":93119320064,"internally_reserved":0,"allocated":2254229504,"data_stored":2252296791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":621470287},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2233204736,"data_stored":2233204736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2604007424,"data_stored":2604007424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2215055360,"data_stored":2215051288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2222456832,"data_stored":2222456832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1801912320,"data_stored":1801912320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2859339776,"data_stored":2859335704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1803943936,"data_stored":1803943936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1790382080,"data_stored":1790378008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:55:39.451 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:55:39.451 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pg_num 21 2024-09-13T14:55:39.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:39 smithi204 ceph-mon[87071]: pgmap v896: 99 pgs: 99 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-09-13T14:55:39.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:39 smithi204 ceph-mon[87071]: from='client.55426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:39.534 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:39.565 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:39.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:39.667 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:39.692 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:39.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:39.794 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:39.821 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:39.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:39.923 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:39.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.977 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:39.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:40.078 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:40.108 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:40.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:40.210 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:40.242 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:40.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:40.247 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:40.257 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:40.343 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:40.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.375 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:40.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:40.454 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:55:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:55:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:40.476 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:40.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.528 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:40.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:40.629 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:40.658 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:40.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:40.760 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:40.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.795 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:40.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:40.896 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:40.935 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:40.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:41.037 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:41.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.077 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:41.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:41.178 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:41.214 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:41.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:41.316 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:41.347 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:41.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:41.369 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:41 smithi204 ceph-mon[87071]: pgmap v897: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:55:41.369 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:55:41.450 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:41.480 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:41.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:41.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:41 smithi096 ceph-mon[98901]: pgmap v897: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:55:41.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:55:41.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:41 smithi096 ceph-mon[102928]: pgmap v897: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:55:41.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:55:41.582 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:41.614 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:41.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:41.717 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:41.751 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:41.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:41.853 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:41.895 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:41.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:41.997 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:42.022 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:42.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:42.124 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:42.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.160 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:42.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:42.261 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:42.286 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:42.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:42.388 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:42.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.419 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:42.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:42.520 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:42.570 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:42.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:42.674 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:42.709 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:42.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:42.812 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:42.854 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:42.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:42.955 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:42.997 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:42.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:43.099 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:43.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.126 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:43.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:43.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:43 smithi204 ceph-mon[87071]: pgmap v898: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2024-09-13T14:55:43.227 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:43.274 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:43.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:43.377 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:43 smithi096 ceph-mon[98901]: pgmap v898: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2024-09-13T14:55:43.377 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:43 smithi096 ceph-mon[102928]: pgmap v898: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2024-09-13T14:55:43.378 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:43.411 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:43.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:43.513 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:43.551 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:43.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:43.653 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:43.686 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:43.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:43.788 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:43.836 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:43.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:43.938 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:43.986 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:43.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:44.090 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:44.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.139 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:44.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:44.240 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:44.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.274 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:44.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:44.375 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:44.412 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:44.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:44.514 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:44.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.558 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:44.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:44.659 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:44.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.691 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:44.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:44.792 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:44.839 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:44.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:44.941 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:44.979 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:44.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:45.081 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:45.102 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:55:45.119 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:45.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:45.221 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:45.255 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:45.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:45.357 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:45.379 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:45 smithi204 ceph-mon[87071]: pgmap v899: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:55:45.380 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:45 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1465094755' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:45.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.408 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:45.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:45.509 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:45.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.541 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:45.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:45.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:45 smithi096 ceph-mon[98901]: pgmap v899: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:55:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:45 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1465094755' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:45.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:45 smithi096 ceph-mon[102928]: pgmap v899: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:55:45.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:45 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1465094755' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:45.642 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:45.690 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:45.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:45.792 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:45.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.826 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:45.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:45.927 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:45.974 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:45.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:46.076 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:46.140 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:46.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:46.242 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:46.278 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:46.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:46 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:46.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:46 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1465094755' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:46.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:46 smithi096 ceph-mon[98901]: osdmap e285: 8 total, 8 up, 8 in 2024-09-13T14:55:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:46 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:46 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1465094755' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:46.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:46 smithi096 ceph-mon[102928]: osdmap e285: 8 total, 8 up, 8 in 2024-09-13T14:55:46.380 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:46.403 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:46 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:46.404 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:46 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1465094755' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:46.404 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:46 smithi204 ceph-mon[87071]: osdmap e285: 8 total, 8 up, 8 in 2024-09-13T14:55:46.415 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:46.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:46.517 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:46.538 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:46.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:46.640 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:46.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.665 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:46.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:46.766 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:46.799 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:46.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:46.901 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:46.922 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:46.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:47.024 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:47.056 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:47.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:47.158 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:47.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.206 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:47.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:47.302 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:47 smithi096 ceph-mon[98901]: pgmap v901: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2024-09-13T14:55:47.302 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:47 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:55:47.302 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:47 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:55:47.302 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:47 smithi096 ceph-mon[102928]: pgmap v901: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2024-09-13T14:55:47.302 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:47 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:55:47.302 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:47 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:55:47.308 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:47.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.354 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:47.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:47.455 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:47.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:47 smithi204 ceph-mon[87071]: pgmap v901: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2024-09-13T14:55:47.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:47 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:55:47.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:47 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:55:47.484 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:47.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:47.586 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:47.615 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:47.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:47.717 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:47.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.741 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:47.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:47.842 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:47.869 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:47.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:47.971 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:48.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.029 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:48.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:48.120 INFO:teuthology.orchestra.run.smithi096.stderr:set pool 1 pg_num to 21 2024-09-13T14:55:48.130 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:48.172 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:48.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:48.274 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:48.306 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:48 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/238239788' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-13T14:55:48.306 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:48 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/238239788' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-13T14:55:48.307 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:55:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:55:47] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.51.0" 2024-09-13T14:55:48.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.308 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:48.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:48.409 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:48.453 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:48.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:48.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:48 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/238239788' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-13T14:55:48.556 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:48.598 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:48.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:48.700 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:48.740 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:48.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:48.841 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:48.870 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:48.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:48.972 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:49.011 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:49.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:49.113 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:49.143 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:49.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:49.212 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:49 smithi096 ceph-mon[98901]: pgmap v902: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2024-09-13T14:55:49.212 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:49 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/238239788' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-13T14:55:49.212 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:49 smithi096 ceph-mon[98901]: osdmap e286: 8 total, 8 up, 8 in 2024-09-13T14:55:49.213 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:49 smithi096 ceph-mon[102928]: pgmap v902: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2024-09-13T14:55:49.213 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:49 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/238239788' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-13T14:55:49.213 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:49 smithi096 ceph-mon[102928]: osdmap e286: 8 total, 8 up, 8 in 2024-09-13T14:55:49.246 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:49.270 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:49.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:49.372 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:49.400 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:49.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:49 smithi204 ceph-mon[87071]: pgmap v902: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2024-09-13T14:55:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:49 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/238239788' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-13T14:55:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:49 smithi204 ceph-mon[87071]: osdmap e286: 8 total, 8 up, 8 in 2024-09-13T14:55:49.502 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:49.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.532 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:49.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:49.634 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:49.661 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:49.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:49.762 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:49.785 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:49.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:49.888 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:49.912 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:49.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:50.014 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:50.047 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:50.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:50.149 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:50.167 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:50 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-13T14:55:50.168 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:50 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-13T14:55:50.177 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:50.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:50.279 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:50.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.322 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:50.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:50.358 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:55:50.386 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:50 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-13T14:55:50.387 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:55:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:55:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:50.423 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:50.465 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:55:50.466 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:50.488 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:50.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:50.503 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:55:50.503 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:50.554 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:55:50.554 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:50.590 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:55:50.590 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:50.592 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:50.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.632 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:50.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:50.632 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:55:50.633 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:50.665 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:55:50.665 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:55:50.665 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:55:50.665 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:55:50.665 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:55:50.666 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:55:50.667 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:50.694 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:55:50.694 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:50.726 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:55:50.726 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:50.733 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:50.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.779 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:50.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:50.880 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:50.922 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:50.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:51.023 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:51.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.072 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:51.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:51.173 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:51.210 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:51.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:51.312 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:51.349 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:51.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:51.450 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:51.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:51 smithi204 ceph-mon[87071]: pgmap v904: 99 pgs: 99 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:55:51.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-13T14:55:51.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:51 smithi204 ceph-mon[87071]: osdmap e287: 8 total, 8 up, 8 in 2024-09-13T14:55:51.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:51 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/136311821' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:51.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:51.491 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:51.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:51.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:51 smithi096 ceph-mon[98901]: pgmap v904: 99 pgs: 99 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:55:51.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-13T14:55:51.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:51 smithi096 ceph-mon[98901]: osdmap e287: 8 total, 8 up, 8 in 2024-09-13T14:55:51.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:51 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/136311821' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:51.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:51.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:51 smithi096 ceph-mon[102928]: pgmap v904: 99 pgs: 99 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:55:51.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-13T14:55:51.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:51 smithi096 ceph-mon[102928]: osdmap e287: 8 total, 8 up, 8 in 2024-09-13T14:55:51.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:51 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/136311821' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:51.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:51.593 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:51.637 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:51.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:51.739 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:51.764 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:51.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:51.866 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:51.898 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:51.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:52.000 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:52.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.039 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:52.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:52.140 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:52.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.177 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:52.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:52.278 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:52.316 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:52.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:52.418 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:52.422 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:52 smithi096 ceph-mon[98901]: osdmap e288: 8 total, 8 up, 8 in 2024-09-13T14:55:52.422 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:52 smithi096 ceph-mon[98901]: 1.4 scrub starts 2024-09-13T14:55:52.422 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:52 smithi096 ceph-mon[102928]: osdmap e288: 8 total, 8 up, 8 in 2024-09-13T14:55:52.423 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:52 smithi096 ceph-mon[102928]: 1.4 scrub starts 2024-09-13T14:55:52.452 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:52.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:52.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:52 smithi204 ceph-mon[87071]: osdmap e288: 8 total, 8 up, 8 in 2024-09-13T14:55:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:52 smithi204 ceph-mon[87071]: 1.4 scrub starts 2024-09-13T14:55:52.554 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:52.594 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:52.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:52.697 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:52.733 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:52.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:52.835 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:52.872 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:52.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:52.974 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:53.000 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:55:53.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.015 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:53.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:53.018 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:53.116 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:53.152 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:53.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:53.216 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:53 smithi096 ceph-mon[98901]: 1.3 scrub starts 2024-09-13T14:55:53.216 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:53 smithi096 ceph-mon[98901]: pgmap v907: 109 pgs: 6 unknown, 103 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T14:55:53.216 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:53 smithi096 ceph-mon[98901]: 1.14 scrub starts 2024-09-13T14:55:53.217 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:53 smithi096 ceph-mon[102928]: 1.3 scrub starts 2024-09-13T14:55:53.217 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:53 smithi096 ceph-mon[102928]: pgmap v907: 109 pgs: 6 unknown, 103 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T14:55:53.217 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:53 smithi096 ceph-mon[102928]: 1.14 scrub starts 2024-09-13T14:55:53.254 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:53.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.308 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:53.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:53.410 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:53.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.440 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:53.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:53 smithi204 ceph-mon[87071]: 1.3 scrub starts 2024-09-13T14:55:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:53 smithi204 ceph-mon[87071]: pgmap v907: 109 pgs: 6 unknown, 103 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T14:55:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:53 smithi204 ceph-mon[87071]: 1.14 scrub starts 2024-09-13T14:55:53.542 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:53.580 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:53.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:53.615 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:55:53.682 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:53.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.724 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:53.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:53.828 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:53.860 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:53.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:53.962 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:53.985 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:53.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:54.052 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:55:54.052 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:55:54.052 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T14:55:54.053 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:55:54.087 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:54.133 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:54.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:54.234 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:54.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.281 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:54.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:54.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:54 smithi096 ceph-mon[102928]: 1.13 scrub starts 2024-09-13T14:55:54.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:54 smithi096 ceph-mon[102928]: 1.c scrub starts 2024-09-13T14:55:54.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:54 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:55:54.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:54 smithi096 ceph-mon[98901]: 1.13 scrub starts 2024-09-13T14:55:54.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:54 smithi096 ceph-mon[98901]: 1.c scrub starts 2024-09-13T14:55:54.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:54 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:55:54.382 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:54.416 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:54.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:54 smithi204 ceph-mon[87071]: 1.13 scrub starts 2024-09-13T14:55:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:54 smithi204 ceph-mon[87071]: 1.c scrub starts 2024-09-13T14:55:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:54 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:55:54.519 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:54.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.547 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:54.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:54.648 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:54.686 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:54.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:54.788 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:54.817 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:54.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:54.919 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:54.928 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:54.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.957 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:54.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:55.058 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:55.093 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:55.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:55.195 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:55.221 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:55.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:55.324 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:55.354 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:55.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:55.457 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:55.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:55 smithi204 ceph-mon[87071]: 1.b scrub starts 2024-09-13T14:55:55.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:55 smithi204 ceph-mon[87071]: pgmap v908: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-13T14:55:55.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:55 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:55:55.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:55 smithi204 ceph-mon[87071]: osdmap e289: 8 total, 8 up, 8 in 2024-09-13T14:55:55.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:55 smithi204 ceph-mon[87071]: 1.3 scrub starts 2024-09-13T14:55:55.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:55 smithi204 ceph-mon[87071]: 1.4 scrub starts 2024-09-13T14:55:55.486 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:55.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:55.489 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:55 smithi096 ceph-mon[98901]: 1.b scrub starts 2024-09-13T14:55:55.489 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:55 smithi096 ceph-mon[98901]: pgmap v908: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-13T14:55:55.489 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:55 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:55:55.489 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:55 smithi096 ceph-mon[98901]: osdmap e289: 8 total, 8 up, 8 in 2024-09-13T14:55:55.489 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:55 smithi096 ceph-mon[98901]: 1.3 scrub starts 2024-09-13T14:55:55.489 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:55 smithi096 ceph-mon[98901]: 1.4 scrub starts 2024-09-13T14:55:55.489 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:55 smithi096 ceph-mon[102928]: 1.b scrub starts 2024-09-13T14:55:55.489 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:55 smithi096 ceph-mon[102928]: pgmap v908: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-13T14:55:55.489 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:55 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:55:55.489 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:55 smithi096 ceph-mon[102928]: osdmap e289: 8 total, 8 up, 8 in 2024-09-13T14:55:55.490 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:55 smithi096 ceph-mon[102928]: 1.3 scrub starts 2024-09-13T14:55:55.490 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:55 smithi096 ceph-mon[102928]: 1.4 scrub starts 2024-09-13T14:55:55.588 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:55.626 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:55.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:55.728 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:55.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.760 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:55.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:55.861 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:55.890 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:55.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:55.992 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:56.030 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:56.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:56.132 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:56.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.170 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:56.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:56.271 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:56.298 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:56.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:56.400 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:56.440 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:56.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:56 smithi204 ceph-mon[87071]: osdmap e290: 8 total, 8 up, 8 in 2024-09-13T14:55:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:56 smithi204 ceph-mon[87071]: 1.14 scrub starts 2024-09-13T14:55:56.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:56 smithi204 ceph-mon[87071]: 1.c scrub starts 2024-09-13T14:55:56.527 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:56 smithi096 ceph-mon[98901]: osdmap e290: 8 total, 8 up, 8 in 2024-09-13T14:55:56.527 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:56 smithi096 ceph-mon[98901]: 1.14 scrub starts 2024-09-13T14:55:56.527 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:56 smithi096 ceph-mon[98901]: 1.c scrub starts 2024-09-13T14:55:56.527 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:56 smithi096 ceph-mon[102928]: osdmap e290: 8 total, 8 up, 8 in 2024-09-13T14:55:56.527 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:56 smithi096 ceph-mon[102928]: 1.14 scrub starts 2024-09-13T14:55:56.527 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:56 smithi096 ceph-mon[102928]: 1.c scrub starts 2024-09-13T14:55:56.542 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:56.578 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:56.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:56.679 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:56.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.726 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:56.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:56.827 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:56.859 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:56.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:56.961 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:57.004 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:57.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:57.105 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:57.156 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:57.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:57.270 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:57.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.311 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:57.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:57.412 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:57.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.454 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:57.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:57.455 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:57 smithi096 ceph-mon[98901]: pgmap v911: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-09-13T14:55:57.455 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:57 smithi096 ceph-mon[98901]: 1.11 scrub starts 2024-09-13T14:55:57.455 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:57 smithi096 ceph-mon[98901]: 1.b scrub starts 2024-09-13T14:55:57.455 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:57 smithi096 ceph-mon[102928]: pgmap v911: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-09-13T14:55:57.455 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:57 smithi096 ceph-mon[102928]: 1.11 scrub starts 2024-09-13T14:55:57.456 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:57 smithi096 ceph-mon[102928]: 1.b scrub starts 2024-09-13T14:55:57.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:57 smithi204 ceph-mon[87071]: pgmap v911: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-09-13T14:55:57.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:57 smithi204 ceph-mon[87071]: 1.11 scrub starts 2024-09-13T14:55:57.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:57 smithi204 ceph-mon[87071]: 1.b scrub starts 2024-09-13T14:55:57.555 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:57.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.582 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:57.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:57.683 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:57.734 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:57.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:57.744 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:57.787 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:55:57.787 INFO:teuthology.orchestra.run.smithi096.stdout: "size": 20, 2024-09-13T14:55:57.787 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 600, 2024-09-13T14:55:57.787 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:55:57.787 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.787 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:136922 6.9 6:9ddaeceb:::benchmark_data_smithi096_167087_object19694:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:55:57.787 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:02.138249+0000", 2024-09-13T14:55:57.787 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 595.64240870699996, 2024-09-13T14:55:57.787 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1346435320000001, 2024-09-13T14:55:57.788 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.788 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.788 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.788 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.788 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.788 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.788 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 136922 2024-09-13T14:55:57.788 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.788 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.788 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.788 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.788 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.138249+0000", 2024-09-13T14:55:57.788 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.789 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.789 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.789 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.789 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.138249+0000", 2024-09-13T14:55:57.789 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.789 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.789 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.789 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.789 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.138250+0000", 2024-09-13T14:55:57.789 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.635e-06 2024-09-13T14:55:57.789 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.789 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.789 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.790 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.138257+0000", 2024-09-13T14:55:57.790 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3080000000000004e-06 2024-09-13T14:55:57.790 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.790 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.790 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.790 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.138272+0000", 2024-09-13T14:55:57.790 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4837e-05 2024-09-13T14:55:57.790 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.790 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.790 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.790 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.138279+0000", 2024-09-13T14:55:57.790 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.8450000000000003e-06 2024-09-13T14:55:57.790 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.791 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.791 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.791 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.138355+0000", 2024-09-13T14:55:57.791 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.5227000000000003e-05 2024-09-13T14:55:57.791 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.791 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.791 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.791 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:02.138365+0000", 2024-09-13T14:55:57.791 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0861e-05 2024-09-13T14:55:57.791 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.791 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.791 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.791 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:03.090388+0000", 2024-09-13T14:55:57.792 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.95202227500000003 2024-09-13T14:55:57.792 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.792 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.792 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.792 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:03.090400+0000", 2024-09-13T14:55:57.792 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2686e-05 2024-09-13T14:55:57.792 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.792 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.792 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.792 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.236709+0000", 2024-09-13T14:55:57.792 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.14630904 2024-09-13T14:55:57.792 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.792 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.793 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.793 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.236843+0000", 2024-09-13T14:55:57.793 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000133311 2024-09-13T14:55:57.793 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.793 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.793 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T14:55:57.793 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.237559+0000", 2024-09-13T14:55:57.793 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00071639900000000005 2024-09-13T14:55:57.793 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.793 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.793 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.793 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.251453+0000", 2024-09-13T14:55:57.793 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.013893453 2024-09-13T14:55:57.794 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.794 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.794 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.794 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.252093+0000", 2024-09-13T14:55:57.794 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00064017399999999997 2024-09-13T14:55:57.794 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.794 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.794 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.794 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.252136+0000", 2024-09-13T14:55:57.794 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2945999999999998e-05 2024-09-13T14:55:57.794 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.794 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.794 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.794 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.272708+0000", 2024-09-13T14:55:57.795 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.02057256 2024-09-13T14:55:57.795 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.795 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.795 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.795 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.272772+0000", 2024-09-13T14:55:57.795 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3999999999999997e-05 2024-09-13T14:55:57.795 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.795 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.795 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.795 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:04.272892+0000", 2024-09-13T14:55:57.795 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000119975 2024-09-13T14:55:57.795 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.795 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.795 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.796 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.796 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.796 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:156592 6.19 6:9866403e:::benchmark_data_smithi096_167087_object39364:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:55:57.796 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:32.018114+0000", 2024-09-13T14:55:57.796 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 565.76254396100001, 2024-09-13T14:55:57.796 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.098294267, 2024-09-13T14:55:57.796 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.796 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.796 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "reached pg", 2024-09-13T14:55:57.796 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.796 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.796 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.796 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 156592 2024-09-13T14:55:57.796 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.797 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.797 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.797 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.797 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018114+0000", 2024-09-13T14:55:57.797 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.797 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.797 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.797 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.797 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018114+0000", 2024-09-13T14:55:57.797 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.797 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.797 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.797 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.798 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018115+0000", 2024-09-13T14:55:57.798 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6220000000000001e-06 2024-09-13T14:55:57.798 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.798 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.798 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.798 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018120+0000", 2024-09-13T14:55:57.798 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6569999999999996e-06 2024-09-13T14:55:57.798 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.798 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.798 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.799 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018136+0000", 2024-09-13T14:55:57.799 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6291e-05 2024-09-13T14:55:57.799 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.799 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.799 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.799 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018144+0000", 2024-09-13T14:55:57.799 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.943e-06 2024-09-13T14:55:57.799 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.799 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.799 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.799 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.257132+0000", 2024-09-13T14:55:57.799 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.238987757 2024-09-13T14:55:57.799 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.800 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.800 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.800 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.257158+0000", 2024-09-13T14:55:57.800 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6673e-05 2024-09-13T14:55:57.800 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.800 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.800 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.800 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.024753+0000", 2024-09-13T14:55:57.800 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7675942060000001 2024-09-13T14:55:57.800 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.800 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.800 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.800 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.024761+0000", 2024-09-13T14:55:57.800 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.3990000000000004e-06 2024-09-13T14:55:57.801 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.801 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.801 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.801 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.116340+0000", 2024-09-13T14:55:57.801 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0915788550000001 2024-09-13T14:55:57.801 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.801 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.801 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.801 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.116408+0000", 2024-09-13T14:55:57.801 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7863999999999999e-05 2024-09-13T14:55:57.801 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.801 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.802 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.802 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.802 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.802 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:156604 6.19 6:9dff4ae2:::benchmark_data_smithi096_167087_object39376:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:55:57.802 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:32.018158+0000", 2024-09-13T14:55:57.802 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 565.762499865, 2024-09-13T14:55:57.802 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.114865462, 2024-09-13T14:55:57.802 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.802 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.802 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "reached pg", 2024-09-13T14:55:57.802 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.802 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.803 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.803 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 156604 2024-09-13T14:55:57.803 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.803 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.803 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.803 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.803 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018158+0000", 2024-09-13T14:55:57.803 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.803 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.803 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.803 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.803 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018158+0000", 2024-09-13T14:55:57.803 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.804 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.804 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.804 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.804 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018158+0000", 2024-09-13T14:55:57.804 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.46e-07 2024-09-13T14:55:57.804 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.804 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.804 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.804 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018161+0000", 2024-09-13T14:55:57.804 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.5579999999999999e-06 2024-09-13T14:55:57.804 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.804 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.804 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.805 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018166+0000", 2024-09-13T14:55:57.805 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7379999999999999e-06 2024-09-13T14:55:57.805 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.805 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.805 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.805 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018171+0000", 2024-09-13T14:55:57.805 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6020000000000002e-06 2024-09-13T14:55:57.805 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.805 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.805 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.805 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.257287+0000", 2024-09-13T14:55:57.805 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.239115684 2024-09-13T14:55:57.805 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.805 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.806 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.806 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.257307+0000", 2024-09-13T14:55:57.806 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0441999999999998e-05 2024-09-13T14:55:57.806 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.806 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.806 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.806 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.024785+0000", 2024-09-13T14:55:57.806 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.767477411 2024-09-13T14:55:57.806 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.806 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.806 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.806 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.024791+0000", 2024-09-13T14:55:57.806 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6830000000000002e-06 2024-09-13T14:55:57.807 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.807 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.807 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.807 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.116828+0000", 2024-09-13T14:55:57.807 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0920370180000001 2024-09-13T14:55:57.807 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.807 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.807 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for missing object", 2024-09-13T14:55:57.807 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.116941+0000", 2024-09-13T14:55:57.807 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011244299999999999 2024-09-13T14:55:57.807 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.807 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.807 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.808 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.132950+0000", 2024-09-13T14:55:57.808 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.016009417000000001 2024-09-13T14:55:57.808 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.808 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.808 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.808 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.133023+0000", 2024-09-13T14:55:57.808 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3020000000000002e-05 2024-09-13T14:55:57.808 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.808 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.808 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.808 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.808 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.809 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:156612 6.19 6:9a2df5a2:::benchmark_data_smithi096_167087_object39384:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:55:57.809 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:32.018177+0000", 2024-09-13T14:55:57.809 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 565.76248055099995, 2024-09-13T14:55:57.809 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.11289354, 2024-09-13T14:55:57.809 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.809 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.809 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "reached pg", 2024-09-13T14:55:57.809 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.809 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.809 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.809 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 156612 2024-09-13T14:55:57.809 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.809 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.809 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.810 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.810 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018177+0000", 2024-09-13T14:55:57.810 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.810 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.810 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.810 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.810 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018177+0000", 2024-09-13T14:55:57.810 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.810 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.810 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.810 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.811 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018177+0000", 2024-09-13T14:55:57.811 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7799999999999997e-07 2024-09-13T14:55:57.811 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.811 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.811 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.811 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018179+0000", 2024-09-13T14:55:57.811 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6649999999999999e-06 2024-09-13T14:55:57.811 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.811 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.811 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.811 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018184+0000", 2024-09-13T14:55:57.811 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8829999999999997e-06 2024-09-13T14:55:57.811 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.811 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.812 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.812 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.018188+0000", 2024-09-13T14:55:57.812 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9670000000000003e-06 2024-09-13T14:55:57.812 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.812 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.812 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.812 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.257436+0000", 2024-09-13T14:55:57.812 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.239247927 2024-09-13T14:55:57.812 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.812 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.812 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.812 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.257455+0000", 2024-09-13T14:55:57.812 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9686000000000001e-05 2024-09-13T14:55:57.813 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.813 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.813 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.813 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.024815+0000", 2024-09-13T14:55:57.813 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.767360067 2024-09-13T14:55:57.813 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.813 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.813 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.813 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.024821+0000", 2024-09-13T14:55:57.813 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5539999999999997e-06 2024-09-13T14:55:57.813 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.813 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.813 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.814 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.118293+0000", 2024-09-13T14:55:57.814 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.093472394 2024-09-13T14:55:57.814 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.814 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.814 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for missing object", 2024-09-13T14:55:57.814 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.118405+0000", 2024-09-13T14:55:57.814 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000111156 2024-09-13T14:55:57.814 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.814 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.814 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.814 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.130989+0000", 2024-09-13T14:55:57.814 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.012584863 2024-09-13T14:55:57.814 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.814 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.815 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.815 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.131070+0000", 2024-09-13T14:55:57.815 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1100000000000006e-05 2024-09-13T14:55:57.815 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.815 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.815 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.815 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.815 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.815 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:156617 6.19 6:982f2767:::benchmark_data_smithi096_167087_object39389:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2024-09-13T14:55:57.815 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:32.022634+0000", 2024-09-13T14:55:57.815 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 565.75802308599998, 2024-09-13T14:55:57.815 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.1087744380000002, 2024-09-13T14:55:57.815 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.816 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.816 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.816 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.816 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.816 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.816 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 156617 2024-09-13T14:55:57.816 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.816 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.816 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.816 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.816 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.022634+0000", 2024-09-13T14:55:57.817 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.817 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.817 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.817 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.818 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.022634+0000", 2024-09-13T14:55:57.818 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.818 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.818 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.818 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.818 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.022636+0000", 2024-09-13T14:55:57.818 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1650000000000001e-06 2024-09-13T14:55:57.818 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.818 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.818 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.818 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.022641+0000", 2024-09-13T14:55:57.818 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.412e-06 2024-09-13T14:55:57.818 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.818 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.819 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.819 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.022657+0000", 2024-09-13T14:55:57.819 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6093999999999999e-05 2024-09-13T14:55:57.819 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.819 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.819 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.819 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.022665+0000", 2024-09-13T14:55:57.819 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9740000000000002e-06 2024-09-13T14:55:57.819 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.819 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.819 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.819 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.257655+0000", 2024-09-13T14:55:57.819 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.23498961900000001 2024-09-13T14:55:57.820 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.820 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.820 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.820 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:32.257675+0000", 2024-09-13T14:55:57.820 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0540999999999999e-05 2024-09-13T14:55:57.820 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.820 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.820 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.820 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.024842+0000", 2024-09-13T14:55:57.820 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7671669300000001 2024-09-13T14:55:57.820 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.820 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.820 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.821 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.024848+0000", 2024-09-13T14:55:57.821 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0480000000000004e-06 2024-09-13T14:55:57.821 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.821 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.821 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.821 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.118429+0000", 2024-09-13T14:55:57.821 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.093580564 2024-09-13T14:55:57.821 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.821 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.821 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.821 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.118531+0000", 2024-09-13T14:55:57.821 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000102128 2024-09-13T14:55:57.821 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.821 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.822 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T14:55:57.822 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.118769+0000", 2024-09-13T14:55:57.822 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00023827899999999999 2024-09-13T14:55:57.822 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.822 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.822 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.822 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.122422+0000", 2024-09-13T14:55:57.822 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0036528620000000002 2024-09-13T14:55:57.822 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.822 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.822 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.822 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.124288+0000", 2024-09-13T14:55:57.822 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0018657820000000001 2024-09-13T14:55:57.822 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.823 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.823 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.823 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.130112+0000", 2024-09-13T14:55:57.823 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0058238669999999999 2024-09-13T14:55:57.823 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.823 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.823 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.823 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.131236+0000", 2024-09-13T14:55:57.823 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001124711 2024-09-13T14:55:57.823 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.823 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.823 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.823 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.131295+0000", 2024-09-13T14:55:57.823 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8610000000000003e-05 2024-09-13T14:55:57.824 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.824 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.824 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.824 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.131409+0000", 2024-09-13T14:55:57.824 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011385199999999999 2024-09-13T14:55:57.824 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.824 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.824 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.824 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.824 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.824 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:156627 6.10 6:0965516b:::benchmark_data_smithi096_167087_object39399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2024-09-13T14:55:57.824 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:33.054315+0000", 2024-09-13T14:55:57.824 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 564.72634237800003, 2024-09-13T14:55:57.824 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.991792864, 2024-09-13T14:55:57.825 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.825 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.825 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.825 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.825 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.825 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.825 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 156627 2024-09-13T14:55:57.825 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.825 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.825 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.825 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.825 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.054315+0000", 2024-09-13T14:55:57.825 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.826 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.826 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.826 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.826 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.054315+0000", 2024-09-13T14:55:57.826 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.826 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.826 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.826 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.826 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.054317+0000", 2024-09-13T14:55:57.826 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.951e-06 2024-09-13T14:55:57.826 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.826 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.826 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.827 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.054325+0000", 2024-09-13T14:55:57.827 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.7479999999999993e-06 2024-09-13T14:55:57.827 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.827 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.827 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.827 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.054346+0000", 2024-09-13T14:55:57.827 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1636000000000001e-05 2024-09-13T14:55:57.827 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.827 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.827 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.827 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.054357+0000", 2024-09-13T14:55:57.827 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0156000000000001e-05 2024-09-13T14:55:57.827 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.827 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.828 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.828 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.054424+0000", 2024-09-13T14:55:57.828 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7475e-05 2024-09-13T14:55:57.828 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.828 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.828 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.828 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.054448+0000", 2024-09-13T14:55:57.828 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3601999999999999e-05 2024-09-13T14:55:57.828 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.828 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.828 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.828 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.023011+0000", 2024-09-13T14:55:57.828 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.968562955 2024-09-13T14:55:57.828 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.829 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.829 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.829 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.023021+0000", 2024-09-13T14:55:57.829 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0151e-05 2024-09-13T14:55:57.829 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.829 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.829 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.829 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.041905+0000", 2024-09-13T14:55:57.829 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0188843379999999 2024-09-13T14:55:57.829 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.829 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.829 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.829 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.042026+0000", 2024-09-13T14:55:57.829 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000120584 2024-09-13T14:55:57.830 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.830 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.830 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:55:57.830 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.042291+0000", 2024-09-13T14:55:57.830 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00026559499999999998 2024-09-13T14:55:57.830 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.830 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.830 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.830 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.045736+0000", 2024-09-13T14:55:57.830 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0034450700000000002 2024-09-13T14:55:57.830 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.830 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.830 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.831 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.045778+0000", 2024-09-13T14:55:57.831 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2001e-05 2024-09-13T14:55:57.831 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.831 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.831 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.831 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.045829+0000", 2024-09-13T14:55:57.831 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0526999999999999e-05 2024-09-13T14:55:57.831 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.831 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.831 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.831 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.045875+0000", 2024-09-13T14:55:57.831 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5719000000000001e-05 2024-09-13T14:55:57.831 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.831 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.832 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.832 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.045987+0000", 2024-09-13T14:55:57.832 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000112342 2024-09-13T14:55:57.832 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.832 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.832 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.832 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.046108+0000", 2024-09-13T14:55:57.832 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000121014 2024-09-13T14:55:57.832 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.833 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.833 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.833 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.833 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.833 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:156632 6.10 6:09bbad32:::benchmark_data_smithi096_167087_object39404:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2024-09-13T14:55:57.833 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:33.076500+0000", 2024-09-13T14:55:57.833 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 564.704157732, 2024-09-13T14:55:57.833 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.973113702, 2024-09-13T14:55:57.833 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.833 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.833 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.833 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.833 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.834 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.834 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 156632 2024-09-13T14:55:57.834 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.834 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.834 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.834 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.834 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.076500+0000", 2024-09-13T14:55:57.834 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.834 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.834 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.834 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.834 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.076500+0000", 2024-09-13T14:55:57.834 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.835 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.835 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.835 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.835 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.076501+0000", 2024-09-13T14:55:57.835 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.705e-06 2024-09-13T14:55:57.835 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.835 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.835 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.835 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.076507+0000", 2024-09-13T14:55:57.835 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7790000000000001e-06 2024-09-13T14:55:57.835 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.835 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.835 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.835 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.076520+0000", 2024-09-13T14:55:57.836 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2838e-05 2024-09-13T14:55:57.836 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.836 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.836 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.836 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.076529+0000", 2024-09-13T14:55:57.836 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0010000000000007e-06 2024-09-13T14:55:57.836 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.836 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.836 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.836 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.076598+0000", 2024-09-13T14:55:57.836 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9060000000000006e-05 2024-09-13T14:55:57.836 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.836 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.837 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.837 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.076610+0000", 2024-09-13T14:55:57.837 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1916000000000001e-05 2024-09-13T14:55:57.837 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.837 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.837 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.837 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.023050+0000", 2024-09-13T14:55:57.837 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.94643968499999998 2024-09-13T14:55:57.837 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.837 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.837 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.837 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.023057+0000", 2024-09-13T14:55:57.837 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.0720000000000002e-06 2024-09-13T14:55:57.837 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.838 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.838 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.838 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.044036+0000", 2024-09-13T14:55:57.838 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0209790889999999 2024-09-13T14:55:57.838 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.838 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.838 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.838 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.044143+0000", 2024-09-13T14:55:57.838 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000107143 2024-09-13T14:55:57.838 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.838 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.838 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:55:57.838 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.044408+0000", 2024-09-13T14:55:57.838 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00026513900000000001 2024-09-13T14:55:57.839 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.839 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.839 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.839 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.049283+0000", 2024-09-13T14:55:57.839 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0048744829999999998 2024-09-13T14:55:57.839 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.839 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.839 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.839 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.049324+0000", 2024-09-13T14:55:57.839 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1111e-05 2024-09-13T14:55:57.839 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.839 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.839 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.839 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.049368+0000", 2024-09-13T14:55:57.840 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3927999999999997e-05 2024-09-13T14:55:57.840 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.840 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.840 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.840 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.049446+0000", 2024-09-13T14:55:57.840 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.8597000000000007e-05 2024-09-13T14:55:57.840 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.840 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.840 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.840 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.049503+0000", 2024-09-13T14:55:57.840 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6267000000000002e-05 2024-09-13T14:55:57.840 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.840 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.840 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.841 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.049613+0000", 2024-09-13T14:55:57.841 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000110889 2024-09-13T14:55:57.841 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.841 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.841 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.841 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.841 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.841 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:156643 6.10 6:0f5f4f94:::benchmark_data_smithi096_167087_object39415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2024-09-13T14:55:57.841 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:33.150823+0000", 2024-09-13T14:55:57.841 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 564.62983434800003, 2024-09-13T14:55:57.841 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9038776799999999, 2024-09-13T14:55:57.841 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.841 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.842 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.842 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.842 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.842 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.842 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 156643 2024-09-13T14:55:57.842 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.842 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.842 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.842 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.842 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.150823+0000", 2024-09-13T14:55:57.842 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.842 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.842 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.842 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.842 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.150823+0000", 2024-09-13T14:55:57.843 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.843 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.843 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.843 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.843 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.150824+0000", 2024-09-13T14:55:57.843 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.2799999999999995e-07 2024-09-13T14:55:57.843 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.843 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.843 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.843 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.150828+0000", 2024-09-13T14:55:57.843 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4440000000000001e-06 2024-09-13T14:55:57.843 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.843 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.843 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.844 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.150841+0000", 2024-09-13T14:55:57.844 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2153e-05 2024-09-13T14:55:57.844 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.844 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.844 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.844 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.150848+0000", 2024-09-13T14:55:57.844 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.289e-06 2024-09-13T14:55:57.844 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.844 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.844 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.844 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.150904+0000", 2024-09-13T14:55:57.844 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5770000000000003e-05 2024-09-13T14:55:57.844 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.845 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.845 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.845 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.150913+0000", 2024-09-13T14:55:57.845 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.6930000000000002e-06 2024-09-13T14:55:57.845 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.845 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.845 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.845 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.023075+0000", 2024-09-13T14:55:57.845 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.87216211200000004 2024-09-13T14:55:57.845 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.845 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.845 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.845 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.023081+0000", 2024-09-13T14:55:57.845 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3199999999999999e-06 2024-09-13T14:55:57.846 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.846 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.846 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.846 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.046143+0000", 2024-09-13T14:55:57.846 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.023062033 2024-09-13T14:55:57.846 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.846 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.846 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.846 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.046255+0000", 2024-09-13T14:55:57.846 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000112377 2024-09-13T14:55:57.846 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.846 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.846 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:55:57.847 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.046498+0000", 2024-09-13T14:55:57.847 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00024256000000000001 2024-09-13T14:55:57.847 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.847 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.847 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.847 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.049409+0000", 2024-09-13T14:55:57.847 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.002910817 2024-09-13T14:55:57.847 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.847 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.847 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.847 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.049626+0000", 2024-09-13T14:55:57.847 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00021772999999999999 2024-09-13T14:55:57.847 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.847 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.848 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.848 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.049676+0000", 2024-09-13T14:55:57.848 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9756000000000003e-05 2024-09-13T14:55:57.848 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.848 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.848 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.848 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.054534+0000", 2024-09-13T14:55:57.848 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.004858294 2024-09-13T14:55:57.848 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.848 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.848 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.848 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.054593+0000", 2024-09-13T14:55:57.848 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8850000000000001e-05 2024-09-13T14:55:57.849 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.849 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.849 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.849 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.054701+0000", 2024-09-13T14:55:57.849 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000107654 2024-09-13T14:55:57.849 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.849 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.849 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.849 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.849 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.850 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:156677 6.10 6:0b630985:::benchmark_data_smithi096_167087_object39449:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2024-09-13T14:55:57.850 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:33.227401+0000", 2024-09-13T14:55:57.850 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 564.55325630699997, 2024-09-13T14:55:57.850 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.835988803, 2024-09-13T14:55:57.850 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.850 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.850 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.850 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.850 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.850 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.850 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 156677 2024-09-13T14:55:57.850 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.850 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.851 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.851 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.851 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.227401+0000", 2024-09-13T14:55:57.851 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.851 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.851 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.851 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.851 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.227401+0000", 2024-09-13T14:55:57.851 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.851 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.851 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.851 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.851 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.227403+0000", 2024-09-13T14:55:57.851 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.745e-06 2024-09-13T14:55:57.851 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.852 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.852 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.852 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.227407+0000", 2024-09-13T14:55:57.852 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9509999999999999e-06 2024-09-13T14:55:57.852 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.852 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.852 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.852 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.227418+0000", 2024-09-13T14:55:57.852 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0801e-05 2024-09-13T14:55:57.852 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.852 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.852 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.852 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.227425+0000", 2024-09-13T14:55:57.853 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9290000000000001e-06 2024-09-13T14:55:57.853 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.853 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.853 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.853 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.227470+0000", 2024-09-13T14:55:57.853 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4946e-05 2024-09-13T14:55:57.853 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.853 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.853 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.853 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.227475+0000", 2024-09-13T14:55:57.853 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8769999999999998e-06 2024-09-13T14:55:57.853 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.853 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.853 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.854 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.023099+0000", 2024-09-13T14:55:57.854 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.79562335399999995 2024-09-13T14:55:57.854 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.854 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.854 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.854 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.023104+0000", 2024-09-13T14:55:57.854 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3990000000000003e-06 2024-09-13T14:55:57.854 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.854 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.854 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.854 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.049720+0000", 2024-09-13T14:55:57.854 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.026616046 2024-09-13T14:55:57.854 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.855 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.855 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.855 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.049856+0000", 2024-09-13T14:55:57.855 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013544300000000001 2024-09-13T14:55:57.855 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.855 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.855 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:55:57.855 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.050138+0000", 2024-09-13T14:55:57.855 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00028252000000000003 2024-09-13T14:55:57.855 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.855 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.855 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.855 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.054740+0000", 2024-09-13T14:55:57.856 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.004602269 2024-09-13T14:55:57.856 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.856 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.856 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.856 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.054780+0000", 2024-09-13T14:55:57.856 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9407999999999999e-05 2024-09-13T14:55:57.856 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.856 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.856 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.856 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.056531+0000", 2024-09-13T14:55:57.856 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001751072 2024-09-13T14:55:57.856 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.856 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.857 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.857 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.063206+0000", 2024-09-13T14:55:57.857 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0066745830000000004 2024-09-13T14:55:57.857 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.857 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.857 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.857 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.063268+0000", 2024-09-13T14:55:57.857 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2800999999999997e-05 2024-09-13T14:55:57.857 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.857 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.857 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.857 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.063390+0000", 2024-09-13T14:55:57.857 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000121659 2024-09-13T14:55:57.858 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.858 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.858 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.858 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.860 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.860 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:156695 6.10 6:0aeb2f81:::benchmark_data_smithi096_167087_object39467:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2024-09-13T14:55:57.860 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:46:33.256668+0000", 2024-09-13T14:55:57.861 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 564.52398974100004, 2024-09-13T14:55:57.861 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8083695930000001, 2024-09-13T14:55:57.861 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.861 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.861 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.861 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.861 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.861 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.861 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 156695 2024-09-13T14:55:57.861 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.861 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.861 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.861 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.861 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.256668+0000", 2024-09-13T14:55:57.862 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.862 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.862 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.862 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.862 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.256668+0000", 2024-09-13T14:55:57.862 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.862 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.862 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.862 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.862 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.256669+0000", 2024-09-13T14:55:57.862 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.449e-06 2024-09-13T14:55:57.862 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.862 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.863 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:57.864 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.864 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.256675+0000", 2024-09-13T14:55:57.864 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2299999999999996e-06 2024-09-13T14:55:57.864 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.864 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.864 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.864 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.256688+0000", 2024-09-13T14:55:57.864 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2819999999999999e-05 2024-09-13T14:55:57.864 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.865 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.865 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.865 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.256695+0000", 2024-09-13T14:55:57.865 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4280000000000001e-06 2024-09-13T14:55:57.865 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.865 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.865 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.865 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.256742+0000", 2024-09-13T14:55:57.865 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7222000000000003e-05 2024-09-13T14:55:57.865 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.865 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.865 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.865 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:33.256748+0000", 2024-09-13T14:55:57.865 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.421e-06 2024-09-13T14:55:57.866 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.866 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.866 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.866 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.023126+0000", 2024-09-13T14:55:57.866 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.76637816199999997 2024-09-13T14:55:57.866 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.866 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.866 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.866 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:34.023133+0000", 2024-09-13T14:55:57.866 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9829999999999999e-06 2024-09-13T14:55:57.866 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.866 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.866 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.866 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.051450+0000", 2024-09-13T14:55:57.867 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0283161249999999 2024-09-13T14:55:57.867 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.867 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.867 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.867 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.051598+0000", 2024-09-13T14:55:57.867 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000148539 2024-09-13T14:55:57.867 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.867 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.867 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:55:57.867 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.051795+0000", 2024-09-13T14:55:57.867 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000196405 2024-09-13T14:55:57.867 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.867 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.868 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.868 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.056537+0000", 2024-09-13T14:55:57.868 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0047421190000000004 2024-09-13T14:55:57.868 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.868 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.868 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.868 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.058123+0000", 2024-09-13T14:55:57.868 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0015863170000000001 2024-09-13T14:55:57.868 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.868 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.868 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.868 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.058166+0000", 2024-09-13T14:55:57.868 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2957000000000002e-05 2024-09-13T14:55:57.869 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.869 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.869 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.869 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.064869+0000", 2024-09-13T14:55:57.869 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0067028000000000001 2024-09-13T14:55:57.869 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.869 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.869 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.869 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.064932+0000", 2024-09-13T14:55:57.869 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2854999999999994e-05 2024-09-13T14:55:57.869 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.869 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.869 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.870 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:46:35.065037+0000", 2024-09-13T14:55:57.870 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010576100000000001 2024-09-13T14:55:57.870 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.870 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.870 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.870 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.870 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.870 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182427 6.19 6:9d06ec9c:::benchmark_data_smithi096_167087_object65199:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e209)", 2024-09-13T14:55:57.870 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:19.100397+0000", 2024-09-13T14:55:57.870 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 518.68026050499998, 2024-09-13T14:55:57.870 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.146521852, 2024-09-13T14:55:57.870 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.870 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.870 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.871 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.871 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.871 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.871 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182427 2024-09-13T14:55:57.871 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.871 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.871 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.871 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.871 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100397+0000", 2024-09-13T14:55:57.871 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.871 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.871 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.871 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.871 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100397+0000", 2024-09-13T14:55:57.872 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.872 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.872 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.872 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.872 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100397+0000", 2024-09-13T14:55:57.872 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9500000000000003e-07 2024-09-13T14:55:57.872 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.872 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.872 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.872 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100400+0000", 2024-09-13T14:55:57.872 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3209999999999999e-06 2024-09-13T14:55:57.872 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.873 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.873 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.873 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100407+0000", 2024-09-13T14:55:57.873 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2420000000000003e-06 2024-09-13T14:55:57.873 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.873 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.873 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.873 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100412+0000", 2024-09-13T14:55:57.873 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4160000000000003e-06 2024-09-13T14:55:57.873 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.873 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.873 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.873 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100501+0000", 2024-09-13T14:55:57.873 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8399000000000005e-05 2024-09-13T14:55:57.874 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.874 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.874 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.874 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100506+0000", 2024-09-13T14:55:57.874 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4179999999999996e-06 2024-09-13T14:55:57.874 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.874 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.874 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.874 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.239323+0000", 2024-09-13T14:55:57.874 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1388168460000001 2024-09-13T14:55:57.875 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.875 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.875 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.875 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.239483+0000", 2024-09-13T14:55:57.875 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00016014499999999999 2024-09-13T14:55:57.875 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.875 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.875 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:55:57.875 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.239748+0000", 2024-09-13T14:55:57.875 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00026440699999999998 2024-09-13T14:55:57.875 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.875 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.875 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.875 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.245850+0000", 2024-09-13T14:55:57.876 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0061026370000000002 2024-09-13T14:55:57.876 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.876 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.876 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.876 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.246095+0000", 2024-09-13T14:55:57.876 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00024516799999999998 2024-09-13T14:55:57.876 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.876 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.876 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.876 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.246435+0000", 2024-09-13T14:55:57.876 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00033959300000000001 2024-09-13T14:55:57.876 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.876 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.877 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.877 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.246513+0000", 2024-09-13T14:55:57.877 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.7824999999999994e-05 2024-09-13T14:55:57.877 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.877 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.877 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.877 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.246568+0000", 2024-09-13T14:55:57.877 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5520999999999999e-05 2024-09-13T14:55:57.877 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.877 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.877 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.877 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.246675+0000", 2024-09-13T14:55:57.877 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010686 2024-09-13T14:55:57.878 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.878 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.878 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.878 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.246747+0000", 2024-09-13T14:55:57.878 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2112000000000003e-05 2024-09-13T14:55:57.878 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.878 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.878 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.878 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.246919+0000", 2024-09-13T14:55:57.878 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000171447 2024-09-13T14:55:57.878 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.878 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.878 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.878 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.879 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.879 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182430 6.19 6:9d5298cb:::benchmark_data_smithi096_167087_object65202:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e209)", 2024-09-13T14:55:57.879 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:19.100453+0000", 2024-09-13T14:55:57.879 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 518.68020475200001, 2024-09-13T14:55:57.879 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1468226379999999, 2024-09-13T14:55:57.879 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.879 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.879 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.879 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.879 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.879 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.879 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182430 2024-09-13T14:55:57.879 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.880 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.880 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.880 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.880 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100453+0000", 2024-09-13T14:55:57.880 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.880 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.880 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.880 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.880 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100453+0000", 2024-09-13T14:55:57.880 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.880 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.880 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.880 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.881 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100453+0000", 2024-09-13T14:55:57.881 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6300000000000005e-07 2024-09-13T14:55:57.881 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.881 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.881 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.881 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100457+0000", 2024-09-13T14:55:57.881 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.3890000000000001e-06 2024-09-13T14:55:57.881 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.881 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.881 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.881 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100466+0000", 2024-09-13T14:55:57.881 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.1800000000000002e-06 2024-09-13T14:55:57.881 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.882 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.882 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.882 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100473+0000", 2024-09-13T14:55:57.882 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.323e-06 2024-09-13T14:55:57.882 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.882 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.882 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.882 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100574+0000", 2024-09-13T14:55:57.882 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010044599999999999 2024-09-13T14:55:57.882 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.882 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.882 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.882 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100580+0000", 2024-09-13T14:55:57.882 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5019999999999998e-06 2024-09-13T14:55:57.883 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.883 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.883 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.883 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.242425+0000", 2024-09-13T14:55:57.883 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.141844447 2024-09-13T14:55:57.883 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.883 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.883 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.883 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.242572+0000", 2024-09-13T14:55:57.883 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00014693900000000001 2024-09-13T14:55:57.883 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.883 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.883 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:55:57.883 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.242829+0000", 2024-09-13T14:55:57.884 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00025752000000000001 2024-09-13T14:55:57.884 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.884 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.884 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.884 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.246031+0000", 2024-09-13T14:55:57.884 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0032019150000000001 2024-09-13T14:55:57.884 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.884 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.884 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.884 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.246623+0000", 2024-09-13T14:55:57.884 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00059230399999999999 2024-09-13T14:55:57.884 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.884 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.884 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.885 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.246962+0000", 2024-09-13T14:55:57.885 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00033892200000000002 2024-09-13T14:55:57.885 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.885 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.885 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.885 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.247004+0000", 2024-09-13T14:55:57.885 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1664e-05 2024-09-13T14:55:57.885 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.885 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.885 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.885 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.247059+0000", 2024-09-13T14:55:57.885 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4963999999999998e-05 2024-09-13T14:55:57.885 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.885 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.886 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.886 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.247112+0000", 2024-09-13T14:55:57.886 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3088999999999999e-05 2024-09-13T14:55:57.886 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.886 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.886 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.886 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.247145+0000", 2024-09-13T14:55:57.886 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.2651999999999998e-05 2024-09-13T14:55:57.886 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.886 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.886 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.887 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.247275+0000", 2024-09-13T14:55:57.887 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013071900000000001 2024-09-13T14:55:57.887 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.887 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.887 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.887 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.887 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.887 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182474 6.19 6:9a2905ad:::benchmark_data_smithi096_167087_object65246:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e209)", 2024-09-13T14:55:57.887 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:19.100496+0000", 2024-09-13T14:55:57.887 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 518.68016193100004, 2024-09-13T14:55:57.887 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1651367370000001, 2024-09-13T14:55:57.887 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.888 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.888 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.888 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.888 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.888 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.888 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182474 2024-09-13T14:55:57.888 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.888 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.888 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.888 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.888 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100496+0000", 2024-09-13T14:55:57.889 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.889 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.889 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.889 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.889 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100496+0000", 2024-09-13T14:55:57.889 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.889 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.889 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.889 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.889 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100496+0000", 2024-09-13T14:55:57.889 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3000000000000001e-07 2024-09-13T14:55:57.889 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.889 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.890 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.890 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100499+0000", 2024-09-13T14:55:57.890 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6589999999999999e-06 2024-09-13T14:55:57.890 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.890 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.890 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.890 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100504+0000", 2024-09-13T14:55:57.890 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0839999999999999e-06 2024-09-13T14:55:57.890 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.890 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.890 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.890 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100507+0000", 2024-09-13T14:55:57.890 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4740000000000001e-06 2024-09-13T14:55:57.890 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.891 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.891 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.891 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100610+0000", 2024-09-13T14:55:57.891 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000102221 2024-09-13T14:55:57.891 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.891 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.891 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.891 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100616+0000", 2024-09-13T14:55:57.891 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3609999999999997e-06 2024-09-13T14:55:57.891 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.891 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.891 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.891 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.247309+0000", 2024-09-13T14:55:57.891 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.146693424 2024-09-13T14:55:57.892 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.892 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.892 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.892 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.247419+0000", 2024-09-13T14:55:57.892 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000109676 2024-09-13T14:55:57.892 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.892 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.892 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:55:57.892 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.247735+0000", 2024-09-13T14:55:57.892 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00031607500000000003 2024-09-13T14:55:57.892 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.892 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.892 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.892 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.254464+0000", 2024-09-13T14:55:57.893 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0067288820000000003 2024-09-13T14:55:57.893 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.893 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.893 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.893 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.254477+0000", 2024-09-13T14:55:57.893 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3152999999999999e-05 2024-09-13T14:55:57.893 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.893 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.893 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.893 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.254535+0000", 2024-09-13T14:55:57.893 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7640999999999999e-05 2024-09-13T14:55:57.893 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.893 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.893 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.894 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.254578+0000", 2024-09-13T14:55:57.894 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3176999999999997e-05 2024-09-13T14:55:57.894 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.894 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.894 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.894 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.254621+0000", 2024-09-13T14:55:57.894 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2988000000000002e-05 2024-09-13T14:55:57.894 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.894 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.894 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.894 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.265429+0000", 2024-09-13T14:55:57.894 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.010807845 2024-09-13T14:55:57.894 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.895 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.895 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.895 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.265494+0000", 2024-09-13T14:55:57.895 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5154999999999995e-05 2024-09-13T14:55:57.895 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.895 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.895 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.895 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.265632+0000", 2024-09-13T14:55:57.895 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013839200000000001 2024-09-13T14:55:57.895 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.895 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.895 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.895 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.895 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.896 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182488 6.19 6:9cad88d1:::benchmark_data_smithi096_167087_object65260:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e209)", 2024-09-13T14:55:57.896 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:19.100520+0000", 2024-09-13T14:55:57.896 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 518.68013764800003, 2024-09-13T14:55:57.896 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1655562449999999, 2024-09-13T14:55:57.896 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.896 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.896 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.896 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.896 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.896 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.896 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182488 2024-09-13T14:55:57.896 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.896 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.897 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.897 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.897 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100520+0000", 2024-09-13T14:55:57.897 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.897 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.897 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.897 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.897 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100520+0000", 2024-09-13T14:55:57.897 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.897 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.897 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.897 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.897 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100520+0000", 2024-09-13T14:55:57.897 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.1899999999999998e-07 2024-09-13T14:55:57.898 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.898 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.898 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.898 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100522+0000", 2024-09-13T14:55:57.898 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7260000000000001e-06 2024-09-13T14:55:57.898 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.898 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.898 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.898 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100526+0000", 2024-09-13T14:55:57.898 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.8850000000000001e-06 2024-09-13T14:55:57.898 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.898 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.898 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.898 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100529+0000", 2024-09-13T14:55:57.899 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7999999999999999e-06 2024-09-13T14:55:57.899 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.899 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.899 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.899 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100640+0000", 2024-09-13T14:55:57.899 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000110963 2024-09-13T14:55:57.899 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.899 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.899 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.899 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100646+0000", 2024-09-13T14:55:57.899 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2210000000000002e-06 2024-09-13T14:55:57.899 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.899 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.899 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.900 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.249343+0000", 2024-09-13T14:55:57.900 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.148697522 2024-09-13T14:55:57.900 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.900 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.900 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.900 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.249461+0000", 2024-09-13T14:55:57.900 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000117618 2024-09-13T14:55:57.900 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.900 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.900 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:55:57.900 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.249725+0000", 2024-09-13T14:55:57.900 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00026408099999999999 2024-09-13T14:55:57.900 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.900 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.900 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.901 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.254483+0000", 2024-09-13T14:55:57.901 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0047576490000000001 2024-09-13T14:55:57.901 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.901 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.901 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.901 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.254678+0000", 2024-09-13T14:55:57.901 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000195658 2024-09-13T14:55:57.901 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.901 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.901 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.901 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.254729+0000", 2024-09-13T14:55:57.901 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0288999999999999e-05 2024-09-13T14:55:57.901 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.901 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.902 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.902 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.254768+0000", 2024-09-13T14:55:57.902 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9768000000000003e-05 2024-09-13T14:55:57.902 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.902 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.902 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.902 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.254809+0000", 2024-09-13T14:55:57.902 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0661999999999998e-05 2024-09-13T14:55:57.902 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.902 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.902 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.902 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.265871+0000", 2024-09-13T14:55:57.902 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.01106178 2024-09-13T14:55:57.902 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.903 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.903 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.903 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.265938+0000", 2024-09-13T14:55:57.903 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7115999999999999e-05 2024-09-13T14:55:57.903 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.903 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.903 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.903 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.266076+0000", 2024-09-13T14:55:57.903 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000138188 2024-09-13T14:55:57.903 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.903 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.903 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.903 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.903 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.904 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182497 6.19 6:9a4c3a6b:::benchmark_data_smithi096_167087_object65269:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e209)", 2024-09-13T14:55:57.904 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:19.100532+0000", 2024-09-13T14:55:57.904 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 518.68012525100005, 2024-09-13T14:55:57.904 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.170443218, 2024-09-13T14:55:57.904 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.904 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.904 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.904 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.904 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.904 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.904 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182497 2024-09-13T14:55:57.904 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.904 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.905 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.905 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.905 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100532+0000", 2024-09-13T14:55:57.905 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.905 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.905 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.905 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.905 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100532+0000", 2024-09-13T14:55:57.905 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.905 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.905 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.905 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.905 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100532+0000", 2024-09-13T14:55:57.906 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.17e-07 2024-09-13T14:55:57.906 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.906 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.906 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.906 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100534+0000", 2024-09-13T14:55:57.906 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1149999999999999e-06 2024-09-13T14:55:57.906 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.906 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.906 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.906 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100537+0000", 2024-09-13T14:55:57.906 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.055e-06 2024-09-13T14:55:57.906 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.906 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.906 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.907 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100540+0000", 2024-09-13T14:55:57.907 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.9359999999999999e-06 2024-09-13T14:55:57.907 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.907 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.907 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.907 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100671+0000", 2024-09-13T14:55:57.907 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000131624 2024-09-13T14:55:57.907 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.907 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.907 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.907 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100678+0000", 2024-09-13T14:55:57.907 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5610000000000004e-06 2024-09-13T14:55:57.907 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.907 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.907 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.908 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.252114+0000", 2024-09-13T14:55:57.908 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.151436272 2024-09-13T14:55:57.908 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.908 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.908 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.908 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.252255+0000", 2024-09-13T14:55:57.908 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00014123 2024-09-13T14:55:57.908 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.908 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.908 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:55:57.908 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.252543+0000", 2024-09-13T14:55:57.908 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00028801800000000002 2024-09-13T14:55:57.908 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.909 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.909 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.909 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.254779+0000", 2024-09-13T14:55:57.909 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0022361630000000002 2024-09-13T14:55:57.909 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.909 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.909 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.909 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.254847+0000", 2024-09-13T14:55:57.909 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7401000000000001e-05 2024-09-13T14:55:57.909 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.909 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.909 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.909 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.259088+0000", 2024-09-13T14:55:57.910 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0042410410000000001 2024-09-13T14:55:57.910 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.910 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.910 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.910 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.259128+0000", 2024-09-13T14:55:57.910 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9817000000000002e-05 2024-09-13T14:55:57.910 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.910 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.910 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.910 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.259198+0000", 2024-09-13T14:55:57.910 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.0167999999999996e-05 2024-09-13T14:55:57.911 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.911 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.911 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.911 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.270794+0000", 2024-09-13T14:55:57.911 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011596301 2024-09-13T14:55:57.911 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.911 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.912 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.912 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.270861+0000", 2024-09-13T14:55:57.912 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7260000000000003e-05 2024-09-13T14:55:57.912 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.912 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.913 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.913 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.270975+0000", 2024-09-13T14:55:57.913 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000114039 2024-09-13T14:55:57.913 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.913 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.913 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.913 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.913 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.913 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182506 6.19 6:9e83dc0e:::benchmark_data_smithi096_167087_object65278:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e209)", 2024-09-13T14:55:57.913 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:19.100542+0000", 2024-09-13T14:55:57.913 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 518.68011511099996, 2024-09-13T14:55:57.913 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.171064479, 2024-09-13T14:55:57.913 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.914 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.914 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.914 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.914 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.914 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.914 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182506 2024-09-13T14:55:57.914 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.914 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.914 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.914 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.914 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100542+0000", 2024-09-13T14:55:57.914 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.914 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.914 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.915 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.915 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100542+0000", 2024-09-13T14:55:57.915 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.915 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.915 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.915 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.915 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100543+0000", 2024-09-13T14:55:57.915 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0900000000000001e-07 2024-09-13T14:55:57.915 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.915 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.915 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.915 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100544+0000", 2024-09-13T14:55:57.916 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0899999999999999e-06 2024-09-13T14:55:57.916 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.916 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.916 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.916 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100547+0000", 2024-09-13T14:55:57.916 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.0809999999999998e-06 2024-09-13T14:55:57.916 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.916 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.916 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.916 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100563+0000", 2024-09-13T14:55:57.916 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6640999999999999e-05 2024-09-13T14:55:57.916 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.917 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.917 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.917 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100701+0000", 2024-09-13T14:55:57.917 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013715299999999999 2024-09-13T14:55:57.917 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.917 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.917 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.917 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100707+0000", 2024-09-13T14:55:57.917 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0109999999999997e-06 2024-09-13T14:55:57.917 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.917 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.917 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.918 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.254877+0000", 2024-09-13T14:55:57.918 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.154170675 2024-09-13T14:55:57.918 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.918 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.918 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.918 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.254994+0000", 2024-09-13T14:55:57.918 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000116618 2024-09-13T14:55:57.918 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.918 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.918 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:55:57.918 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.255246+0000", 2024-09-13T14:55:57.918 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00025191699999999999 2024-09-13T14:55:57.918 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.919 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.919 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.919 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.259163+0000", 2024-09-13T14:55:57.919 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0039170899999999998 2024-09-13T14:55:57.919 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.919 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.919 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.919 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.259232+0000", 2024-09-13T14:55:57.919 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8935000000000003e-05 2024-09-13T14:55:57.919 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.919 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.919 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.919 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.259266+0000", 2024-09-13T14:55:57.919 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.3745000000000003e-05 2024-09-13T14:55:57.920 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.920 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.920 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.920 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.261137+0000", 2024-09-13T14:55:57.920 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0018717720000000001 2024-09-13T14:55:57.920 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.920 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.920 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.920 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.263574+0000", 2024-09-13T14:55:57.920 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0024363459999999998 2024-09-13T14:55:57.920 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.920 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.920 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.921 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.271419+0000", 2024-09-13T14:55:57.921 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0078456120000000001 2024-09-13T14:55:57.921 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.921 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.921 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.921 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.271472+0000", 2024-09-13T14:55:57.921 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2269999999999999e-05 2024-09-13T14:55:57.921 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.921 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.921 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.921 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.271607+0000", 2024-09-13T14:55:57.922 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013531400000000001 2024-09-13T14:55:57.922 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.922 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.922 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.922 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.922 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.923 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182555 6.19 6:9b252168:::benchmark_data_smithi096_167087_object65327:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e209)", 2024-09-13T14:55:57.923 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:19.100578+0000", 2024-09-13T14:55:57.923 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 518.68007981300002, 2024-09-13T14:55:57.923 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1714163019999999, 2024-09-13T14:55:57.923 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.924 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.924 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.924 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.924 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:55:57.924 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:55:57.924 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182555 2024-09-13T14:55:57.924 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.924 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.924 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.924 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.924 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100578+0000", 2024-09-13T14:55:57.924 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.924 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.925 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.925 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.925 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100578+0000", 2024-09-13T14:55:57.925 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.925 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.925 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.925 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.925 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100578+0000", 2024-09-13T14:55:57.925 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2800000000000002e-07 2024-09-13T14:55:57.925 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.925 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.925 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.926 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100580+0000", 2024-09-13T14:55:57.926 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1789999999999998e-06 2024-09-13T14:55:57.926 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.926 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.926 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.926 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100586+0000", 2024-09-13T14:55:57.926 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1129999999999997e-06 2024-09-13T14:55:57.926 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.926 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.926 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.926 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100591+0000", 2024-09-13T14:55:57.926 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8600000000000001e-06 2024-09-13T14:55:57.926 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.927 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.927 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.927 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100731+0000", 2024-09-13T14:55:57.927 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013938500000000001 2024-09-13T14:55:57.927 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.927 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.927 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.927 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:19.100737+0000", 2024-09-13T14:55:57.927 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0610000000000001e-06 2024-09-13T14:55:57.927 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.927 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.927 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.927 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.256888+0000", 2024-09-13T14:55:57.928 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1561508300000001 2024-09-13T14:55:57.928 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.928 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.928 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.928 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.257014+0000", 2024-09-13T14:55:57.928 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012679299999999999 2024-09-13T14:55:57.928 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.928 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.928 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:55:57.928 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.257252+0000", 2024-09-13T14:55:57.928 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00023745400000000001 2024-09-13T14:55:57.928 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.928 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.929 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.929 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.261187+0000", 2024-09-13T14:55:57.929 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0039353169999999998 2024-09-13T14:55:57.929 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.929 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.929 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.929 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.261232+0000", 2024-09-13T14:55:57.929 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4682999999999999e-05 2024-09-13T14:55:57.929 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.929 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.929 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.929 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.261275+0000", 2024-09-13T14:55:57.929 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3062999999999997e-05 2024-09-13T14:55:57.930 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.930 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.930 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.930 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.261321+0000", 2024-09-13T14:55:57.930 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6350999999999997e-05 2024-09-13T14:55:57.930 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.930 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.930 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.930 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.265379+0000", 2024-09-13T14:55:57.930 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0040575960000000001 2024-09-13T14:55:57.930 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.930 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.930 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.931 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.271799+0000", 2024-09-13T14:55:57.931 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0064205870000000002 2024-09-13T14:55:57.931 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.931 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.931 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.931 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.271867+0000", 2024-09-13T14:55:57.931 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7274000000000007e-05 2024-09-13T14:55:57.931 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.931 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.931 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.931 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:20.271994+0000", 2024-09-13T14:55:57.931 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012732800000000001 2024-09-13T14:55:57.931 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.932 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.932 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.932 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.932 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.932 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35607.0:169252 7.6 7:6b726cbc:::benchmark_data_smithi096_196313_object49121:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2024-09-13T14:55:57.932 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:51:10.059960+0000", 2024-09-13T14:55:57.932 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 287.72069729200001, 2024-09-13T14:55:57.932 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.2807134439999999, 2024-09-13T14:55:57.932 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.932 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.932 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.932 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.932 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35607", 2024-09-13T14:55:57.933 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:55:57.933 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 169252 2024-09-13T14:55:57.933 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.933 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.933 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.933 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.933 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.059960+0000", 2024-09-13T14:55:57.933 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.933 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.933 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.933 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.933 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.059960+0000", 2024-09-13T14:55:57.933 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.934 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.934 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.934 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.934 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.059962+0000", 2024-09-13T14:55:57.934 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3519999999999999e-06 2024-09-13T14:55:57.934 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.934 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.934 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.934 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.059967+0000", 2024-09-13T14:55:57.934 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4480000000000002e-06 2024-09-13T14:55:57.934 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.934 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.934 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.934 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.059978+0000", 2024-09-13T14:55:57.935 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1222e-05 2024-09-13T14:55:57.935 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.935 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.935 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.935 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.059987+0000", 2024-09-13T14:55:57.935 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0010000000000007e-06 2024-09-13T14:55:57.935 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.935 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.935 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.935 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.067105+0000", 2024-09-13T14:55:57.935 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0071175459999999998 2024-09-13T14:55:57.935 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.935 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.935 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.935 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.067114+0000", 2024-09-13T14:55:57.936 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3360000000000001e-06 2024-09-13T14:55:57.936 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.936 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.936 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.936 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:11.056467+0000", 2024-09-13T14:55:57.936 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.98935249999999997 2024-09-13T14:55:57.936 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.936 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.936 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.936 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:11.056478+0000", 2024-09-13T14:55:57.936 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1704e-05 2024-09-13T14:55:57.936 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.936 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.937 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.937 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.313372+0000", 2024-09-13T14:55:57.937 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2568936040000001 2024-09-13T14:55:57.937 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.937 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.937 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.937 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.313698+0000", 2024-09-13T14:55:57.937 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000326309 2024-09-13T14:55:57.937 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.937 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.937 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 3,5,7", 2024-09-13T14:55:57.937 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.314091+0000", 2024-09-13T14:55:57.938 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00039276900000000001 2024-09-13T14:55:57.938 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.938 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.938 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.938 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.318936+0000", 2024-09-13T14:55:57.939 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0048449039999999997 2024-09-13T14:55:57.939 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.939 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.939 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.939 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.335782+0000", 2024-09-13T14:55:57.940 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.016845632999999999 2024-09-13T14:55:57.940 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.940 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.940 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.940 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.335862+0000", 2024-09-13T14:55:57.940 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0222999999999994e-05 2024-09-13T14:55:57.941 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.941 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.941 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.941 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.340353+0000", 2024-09-13T14:55:57.941 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0044911100000000004 2024-09-13T14:55:57.941 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.941 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.941 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.941 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.340428+0000", 2024-09-13T14:55:57.941 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.4981999999999995e-05 2024-09-13T14:55:57.941 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.941 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.941 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.942 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.340674+0000", 2024-09-13T14:55:57.942 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00024580099999999997 2024-09-13T14:55:57.942 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.942 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.942 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.942 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.942 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.943 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35607.0:169254 7.6 7:6aa85002:::benchmark_data_smithi096_196313_object49123:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T14:55:57.943 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:51:10.060227+0000", 2024-09-13T14:55:57.943 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 287.72043010499999, 2024-09-13T14:55:57.943 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.2810936000000002, 2024-09-13T14:55:57.943 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.944 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.944 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.944 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.944 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35607", 2024-09-13T14:55:57.944 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:55:57.945 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 169254 2024-09-13T14:55:57.945 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.945 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.945 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.945 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.945 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.060227+0000", 2024-09-13T14:55:57.946 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.946 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.946 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.946 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.946 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.060227+0000", 2024-09-13T14:55:57.947 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.947 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.947 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.947 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.947 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.060228+0000", 2024-09-13T14:55:57.947 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9999999999999997e-07 2024-09-13T14:55:57.947 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.947 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.947 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.947 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.060231+0000", 2024-09-13T14:55:57.947 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6410000000000002e-06 2024-09-13T14:55:57.947 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.947 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.948 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.948 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.060239+0000", 2024-09-13T14:55:57.948 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.7169999999999997e-06 2024-09-13T14:55:57.948 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.948 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.948 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.948 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.060246+0000", 2024-09-13T14:55:57.948 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4160000000000001e-06 2024-09-13T14:55:57.948 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.948 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.948 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.948 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.067136+0000", 2024-09-13T14:55:57.948 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0068903330000000002 2024-09-13T14:55:57.949 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.949 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.949 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.949 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.067142+0000", 2024-09-13T14:55:57.949 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2500000000000003e-06 2024-09-13T14:55:57.949 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.949 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.949 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.949 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:11.056811+0000", 2024-09-13T14:55:57.949 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.98966856700000005 2024-09-13T14:55:57.949 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.949 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.949 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.950 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:11.056821+0000", 2024-09-13T14:55:57.950 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.6369999999999994e-06 2024-09-13T14:55:57.950 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.950 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.950 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.950 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.318988+0000", 2024-09-13T14:55:57.950 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.262167746 2024-09-13T14:55:57.951 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.951 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.951 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.951 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.319123+0000", 2024-09-13T14:55:57.951 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013414699999999999 2024-09-13T14:55:57.951 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.952 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.952 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 3,5,7", 2024-09-13T14:55:57.952 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.319268+0000", 2024-09-13T14:55:57.952 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00014591800000000001 2024-09-13T14:55:57.952 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.953 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.953 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.953 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.335849+0000", 2024-09-13T14:55:57.953 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.016580553000000001 2024-09-13T14:55:57.953 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.953 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.954 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.954 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.335867+0000", 2024-09-13T14:55:57.954 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7558999999999999e-05 2024-09-13T14:55:57.954 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.954 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.954 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.955 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.336121+0000", 2024-09-13T14:55:57.955 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00025491499999999998 2024-09-13T14:55:57.955 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.955 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.955 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.955 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.340878+0000", 2024-09-13T14:55:57.955 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0047567679999999998 2024-09-13T14:55:57.955 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.955 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.955 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.955 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.341052+0000", 2024-09-13T14:55:57.955 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00017356899999999999 2024-09-13T14:55:57.955 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.955 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.956 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.956 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.341321+0000", 2024-09-13T14:55:57.956 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00026916400000000001 2024-09-13T14:55:57.956 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.956 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.956 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.956 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.956 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.956 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.35607.0:169256 7.6 7:629787be:::benchmark_data_smithi096_196313_object49125:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T14:55:57.956 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:51:10.062236+0000", 2024-09-13T14:55:57.956 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 287.71842107800001, 2024-09-13T14:55:57.956 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.2792899690000001, 2024-09-13T14:55:57.956 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:55:57.957 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:55:57.957 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:57.957 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:55:57.957 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.35607", 2024-09-13T14:55:57.957 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:55:57.957 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 169256 2024-09-13T14:55:57.957 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.957 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:55:57.957 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.957 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:55:57.957 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.062236+0000", 2024-09-13T14:55:57.957 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.957 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.958 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.958 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:55:57.958 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.062236+0000", 2024-09-13T14:55:57.958 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:55:57.958 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.958 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.958 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:55:57.958 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.062238+0000", 2024-09-13T14:55:57.958 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.136e-06 2024-09-13T14:55:57.958 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.958 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.958 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:55:57.958 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.062242+0000", 2024-09-13T14:55:57.958 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5990000000000004e-06 2024-09-13T14:55:57.959 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.959 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.959 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:55:57.959 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.062251+0000", 2024-09-13T14:55:57.959 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6829999999999997e-06 2024-09-13T14:55:57.959 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.959 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.959 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:55:57.959 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.062258+0000", 2024-09-13T14:55:57.960 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9199999999999998e-06 2024-09-13T14:55:57.960 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.960 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.960 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.960 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.067162+0000", 2024-09-13T14:55:57.960 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0049045060000000003 2024-09-13T14:55:57.961 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.961 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.961 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.961 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:10.067170+0000", 2024-09-13T14:55:57.961 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.6850000000000001e-06 2024-09-13T14:55:57.961 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.961 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.962 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.962 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:11.056878+0000", 2024-09-13T14:55:57.962 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.98970845399999996 2024-09-13T14:55:57.962 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.962 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.962 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:55:57.963 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:11.056886+0000", 2024-09-13T14:55:57.963 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1369999999999996e-06 2024-09-13T14:55:57.963 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.963 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.963 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:55:57.963 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.323682+0000", 2024-09-13T14:55:57.963 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.266796101 2024-09-13T14:55:57.963 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.964 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.964 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:55:57.964 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.324047+0000", 2024-09-13T14:55:57.964 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00036509999999999998 2024-09-13T14:55:57.964 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.964 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.964 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 3,5,7", 2024-09-13T14:55:57.965 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.324504+0000", 2024-09-13T14:55:57.965 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00045752299999999999 2024-09-13T14:55:57.965 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.965 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.965 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:55:57.965 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.335871+0000", 2024-09-13T14:55:57.965 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011367172 2024-09-13T14:55:57.966 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.966 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.966 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.966 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.336173+0000", 2024-09-13T14:55:57.966 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00030173500000000002 2024-09-13T14:55:57.966 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.966 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.967 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.967 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.336299+0000", 2024-09-13T14:55:57.967 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012633100000000001 2024-09-13T14:55:57.967 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.967 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.967 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:57.967 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.341365+0000", 2024-09-13T14:55:57.968 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0050652789999999998 2024-09-13T14:55:57.968 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.968 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.968 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:55:57.968 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.341417+0000", 2024-09-13T14:55:57.968 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2403e-05 2024-09-13T14:55:57.968 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:55:57.969 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:55:57.969 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:55:57.969 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:51:12.341526+0000", 2024-09-13T14:55:57.969 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000109205 2024-09-13T14:55:57.969 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.969 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.969 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.970 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:55:57.970 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:55:57.970 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:55:57.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.972 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:57.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:55:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:55:57] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.51.0" 2024-09-13T14:55:58.074 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:58.125 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:58.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:58.221 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:55:58.227 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:58.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.258 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:58.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:58.359 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:58.390 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:58.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:58.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:58 smithi204 ceph-mon[87071]: 1.13 scrub starts 2024-09-13T14:55:58.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:58 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3946698388' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:58.491 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:58.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.525 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:58.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:58.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:58 smithi096 ceph-mon[98901]: 1.13 scrub starts 2024-09-13T14:55:58.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:58 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3946698388' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:58.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:58 smithi096 ceph-mon[102928]: 1.13 scrub starts 2024-09-13T14:55:58.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:58 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3946698388' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:58.626 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:58.660 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:58.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:58.762 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:58.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.799 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:58.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:58.900 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:58.936 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:58.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:59.038 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:59.064 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:59.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:59.166 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:59.198 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:59.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:59.301 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:55:59.302 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:59.344 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:59.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:59.446 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:59.469 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:59.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:59 smithi204 ceph-mon[87071]: pgmap v912: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 4.9 KiB/s rd, 11 KiB/s wr, 907 op/s; 3.8 KiB/s, 0 objects/s recovering 2024-09-13T14:55:59.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:59 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:59.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:59 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3946698388' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:55:59.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:59 smithi204 ceph-mon[87071]: osdmap e291: 8 total, 8 up, 8 in 2024-09-13T14:55:59.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:59 smithi204 ceph-mon[87071]: 1.5 scrub starts 2024-09-13T14:55:59.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:55:59 smithi204 ceph-mon[87071]: 1.5 scrub ok 2024-09-13T14:55:59.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:59.571 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:59 smithi096 ceph-mon[98901]: pgmap v912: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 4.9 KiB/s rd, 11 KiB/s wr, 907 op/s; 3.8 KiB/s, 0 objects/s recovering 2024-09-13T14:55:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:59 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:59 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3946698388' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:55:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:59 smithi096 ceph-mon[98901]: osdmap e291: 8 total, 8 up, 8 in 2024-09-13T14:55:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:59 smithi096 ceph-mon[98901]: 1.5 scrub starts 2024-09-13T14:55:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:55:59 smithi096 ceph-mon[98901]: 1.5 scrub ok 2024-09-13T14:55:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:59 smithi096 ceph-mon[102928]: pgmap v912: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 4.9 KiB/s rd, 11 KiB/s wr, 907 op/s; 3.8 KiB/s, 0 objects/s recovering 2024-09-13T14:55:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:59 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:59 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3946698388' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:55:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:59 smithi096 ceph-mon[102928]: osdmap e291: 8 total, 8 up, 8 in 2024-09-13T14:55:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:59 smithi096 ceph-mon[102928]: 1.5 scrub starts 2024-09-13T14:55:59.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:55:59 smithi096 ceph-mon[102928]: 1.5 scrub ok 2024-09-13T14:55:59.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.627 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:59.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:59.728 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:59.758 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:55:59.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:59.860 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:59.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.894 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:55:59.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:59.995 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:00.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.032 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:00.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:00.133 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:00.165 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:00.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:00.266 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:00.304 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:00.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:00.406 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:00.440 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:00.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:00 smithi204 ceph-mon[87071]: 1.0 scrub starts 2024-09-13T14:56:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:00 smithi204 ceph-mon[87071]: 1.0 scrub ok 2024-09-13T14:56:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:00 smithi204 ceph-mon[87071]: 1.3 scrub starts 2024-09-13T14:56:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:00 smithi204 ceph-mon[87071]: 1.3 scrub ok 2024-09-13T14:56:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:00 smithi204 ceph-mon[87071]: 1.4 scrub starts 2024-09-13T14:56:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:00 smithi204 ceph-mon[87071]: 1.4 scrub ok 2024-09-13T14:56:00.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:56:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:56:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:00.542 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:00.569 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:00.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:00.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:00 smithi096 ceph-mon[98901]: 1.0 scrub starts 2024-09-13T14:56:00.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:00 smithi096 ceph-mon[98901]: 1.0 scrub ok 2024-09-13T14:56:00.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:00 smithi096 ceph-mon[98901]: 1.3 scrub starts 2024-09-13T14:56:00.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:00 smithi096 ceph-mon[98901]: 1.3 scrub ok 2024-09-13T14:56:00.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:00 smithi096 ceph-mon[98901]: 1.4 scrub starts 2024-09-13T14:56:00.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:00 smithi096 ceph-mon[98901]: 1.4 scrub ok 2024-09-13T14:56:00.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:00 smithi096 ceph-mon[102928]: 1.0 scrub starts 2024-09-13T14:56:00.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:00 smithi096 ceph-mon[102928]: 1.0 scrub ok 2024-09-13T14:56:00.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:00 smithi096 ceph-mon[102928]: 1.3 scrub starts 2024-09-13T14:56:00.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:00 smithi096 ceph-mon[102928]: 1.3 scrub ok 2024-09-13T14:56:00.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:00 smithi096 ceph-mon[102928]: 1.4 scrub starts 2024-09-13T14:56:00.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:00 smithi096 ceph-mon[102928]: 1.4 scrub ok 2024-09-13T14:56:00.671 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:00.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.716 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:00.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:00.817 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:00.849 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:00.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:00.951 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:00.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.991 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:00.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:01.092 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:01.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.128 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:01.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:01.229 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:01.273 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:01.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:01.375 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:01.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.411 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:01.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:01.513 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:01.553 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:01.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:01 smithi096 ceph-mon[98901]: 1.2 scrub starts 2024-09-13T14:56:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:01 smithi096 ceph-mon[98901]: 1.2 scrub ok 2024-09-13T14:56:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:01 smithi096 ceph-mon[98901]: pgmap v914: 109 pgs: 109 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 11 KiB/s rd, 25 KiB/s wr, 826 op/s; 62 KiB/s, 0 objects/s recovering 2024-09-13T14:56:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:01 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:01 smithi096 ceph-mon[102928]: 1.2 scrub starts 2024-09-13T14:56:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:01 smithi096 ceph-mon[102928]: 1.2 scrub ok 2024-09-13T14:56:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:01 smithi096 ceph-mon[102928]: pgmap v914: 109 pgs: 109 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 11 KiB/s rd, 25 KiB/s wr, 826 op/s; 62 KiB/s, 0 objects/s recovering 2024-09-13T14:56:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:01 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:01.655 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:01.691 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:01.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:01.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:01 smithi204 ceph-mon[87071]: 1.2 scrub starts 2024-09-13T14:56:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:01 smithi204 ceph-mon[87071]: 1.2 scrub ok 2024-09-13T14:56:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:01 smithi204 ceph-mon[87071]: pgmap v914: 109 pgs: 109 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 11 KiB/s rd, 25 KiB/s wr, 826 op/s; 62 KiB/s, 0 objects/s recovering 2024-09-13T14:56:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:01 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:01.793 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:01.833 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:01.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:01.935 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:01.969 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:01.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:02.046 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:02.071 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:02.102 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:02.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:02.204 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:02.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.237 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:02.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:02.338 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:02.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.379 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:02.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:02.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[98901]: 1.11 scrub starts 2024-09-13T14:56:02.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[98901]: 1.11 scrub ok 2024-09-13T14:56:02.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[98901]: 1.14 scrub starts 2024-09-13T14:56:02.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[98901]: 1.14 scrub ok 2024-09-13T14:56:02.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[98901]: 1.b scrub starts 2024-09-13T14:56:02.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[98901]: 1.1 scrub starts 2024-09-13T14:56:02.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[98901]: 1.1 scrub ok 2024-09-13T14:56:02.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[98901]: 1.c scrub starts 2024-09-13T14:56:02.427 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[98901]: 1.c scrub ok 2024-09-13T14:56:02.427 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[102928]: 1.11 scrub starts 2024-09-13T14:56:02.428 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[102928]: 1.11 scrub ok 2024-09-13T14:56:02.428 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[102928]: 1.14 scrub starts 2024-09-13T14:56:02.428 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[102928]: 1.14 scrub ok 2024-09-13T14:56:02.428 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[102928]: 1.b scrub starts 2024-09-13T14:56:02.428 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[102928]: 1.1 scrub starts 2024-09-13T14:56:02.428 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[102928]: 1.1 scrub ok 2024-09-13T14:56:02.428 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[102928]: 1.c scrub starts 2024-09-13T14:56:02.428 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:02 smithi096 ceph-mon[102928]: 1.c scrub ok 2024-09-13T14:56:02.480 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:02.501 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:02.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:02.603 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:02.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.644 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:02.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:02.712 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:56:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:02 smithi204 ceph-mon[87071]: 1.11 scrub starts 2024-09-13T14:56:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:02 smithi204 ceph-mon[87071]: 1.11 scrub ok 2024-09-13T14:56:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:02 smithi204 ceph-mon[87071]: 1.14 scrub starts 2024-09-13T14:56:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:02 smithi204 ceph-mon[87071]: 1.14 scrub ok 2024-09-13T14:56:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:02 smithi204 ceph-mon[87071]: 1.b scrub starts 2024-09-13T14:56:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:02 smithi204 ceph-mon[87071]: 1.1 scrub starts 2024-09-13T14:56:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:02 smithi204 ceph-mon[87071]: 1.1 scrub ok 2024-09-13T14:56:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:02 smithi204 ceph-mon[87071]: 1.c scrub starts 2024-09-13T14:56:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:02 smithi204 ceph-mon[87071]: 1.c scrub ok 2024-09-13T14:56:02.745 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:02.781 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:02.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:02.882 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:02.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.911 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:02.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:03.011 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:03.051 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:03.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:03.153 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:03.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.189 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:03.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:03.290 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[98901]: 1.b scrub ok 2024-09-13T14:56:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[98901]: 1.d scrub starts 2024-09-13T14:56:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[98901]: 1.d scrub ok 2024-09-13T14:56:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[98901]: 1.7 scrub starts 2024-09-13T14:56:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[98901]: 1.7 scrub ok 2024-09-13T14:56:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[98901]: pgmap v915: 109 pgs: 109 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 8.7 KiB/s rd, 19 KiB/s wr, 710 op/s; 47 KiB/s, 0 objects/s recovering 2024-09-13T14:56:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[98901]: 1.e scrub starts 2024-09-13T14:56:03.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[98901]: 1.e scrub ok 2024-09-13T14:56:03.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[102928]: 1.b scrub ok 2024-09-13T14:56:03.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[102928]: 1.d scrub starts 2024-09-13T14:56:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[102928]: 1.d scrub ok 2024-09-13T14:56:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[102928]: 1.7 scrub starts 2024-09-13T14:56:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[102928]: 1.7 scrub ok 2024-09-13T14:56:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[102928]: pgmap v915: 109 pgs: 109 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 8.7 KiB/s rd, 19 KiB/s wr, 710 op/s; 47 KiB/s, 0 objects/s recovering 2024-09-13T14:56:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[102928]: 1.e scrub starts 2024-09-13T14:56:03.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:03 smithi096 ceph-mon[102928]: 1.e scrub ok 2024-09-13T14:56:03.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.327 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:03.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:03.428 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:03.467 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:03.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:03.569 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:03.601 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:03.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:03.703 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:03 smithi204 ceph-mon[87071]: 1.b scrub ok 2024-09-13T14:56:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:03 smithi204 ceph-mon[87071]: 1.d scrub starts 2024-09-13T14:56:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:03 smithi204 ceph-mon[87071]: 1.d scrub ok 2024-09-13T14:56:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:03 smithi204 ceph-mon[87071]: 1.7 scrub starts 2024-09-13T14:56:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:03 smithi204 ceph-mon[87071]: 1.7 scrub ok 2024-09-13T14:56:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:03 smithi204 ceph-mon[87071]: pgmap v915: 109 pgs: 109 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 8.7 KiB/s rd, 19 KiB/s wr, 710 op/s; 47 KiB/s, 0 objects/s recovering 2024-09-13T14:56:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:03 smithi204 ceph-mon[87071]: 1.e scrub starts 2024-09-13T14:56:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:03 smithi204 ceph-mon[87071]: 1.e scrub ok 2024-09-13T14:56:03.734 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:03.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:03.835 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:03.874 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:03.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:03.977 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:04.013 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:04.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:04.115 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:04.145 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:04.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:04.246 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:04.285 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:04.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:04.387 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:04.421 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:04.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:04.523 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:04.555 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:04.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:04.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:04 smithi096 ceph-mon[98901]: 1.13 scrub starts 2024-09-13T14:56:04.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:04 smithi096 ceph-mon[98901]: 1.13 scrub ok 2024-09-13T14:56:04.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:04 smithi096 ceph-mon[98901]: 1.12 scrub starts 2024-09-13T14:56:04.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:04 smithi096 ceph-mon[98901]: 1.12 scrub ok 2024-09-13T14:56:04.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:04 smithi096 ceph-mon[102928]: 1.13 scrub starts 2024-09-13T14:56:04.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:04 smithi096 ceph-mon[102928]: 1.13 scrub ok 2024-09-13T14:56:04.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:04 smithi096 ceph-mon[102928]: 1.12 scrub starts 2024-09-13T14:56:04.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:04 smithi096 ceph-mon[102928]: 1.12 scrub ok 2024-09-13T14:56:04.657 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:04.686 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:04.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:04.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:04 smithi204 ceph-mon[87071]: 1.13 scrub starts 2024-09-13T14:56:04.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:04 smithi204 ceph-mon[87071]: 1.13 scrub ok 2024-09-13T14:56:04.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:04 smithi204 ceph-mon[87071]: 1.12 scrub starts 2024-09-13T14:56:04.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:04 smithi204 ceph-mon[87071]: 1.12 scrub ok 2024-09-13T14:56:04.787 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:04.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.822 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:04.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:04.924 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:04.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.959 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:04.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:05.060 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:05.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.100 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:05.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:05.201 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:05.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.238 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:05.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:05.339 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:05.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.378 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:05.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:05.479 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:05.516 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:05.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:05.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:05 smithi096 ceph-mon[102928]: pgmap v916: 109 pgs: 109 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 3.8 KiB/s rd, 8.4 KiB/s wr, 1.12k op/s; 3.0 KiB/s, 0 objects/s recovering 2024-09-13T14:56:05.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:05 smithi096 ceph-mon[102928]: 1.10 scrub starts 2024-09-13T14:56:05.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:05 smithi096 ceph-mon[102928]: 1.10 scrub ok 2024-09-13T14:56:05.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:05 smithi096 ceph-mon[98901]: pgmap v916: 109 pgs: 109 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 3.8 KiB/s rd, 8.4 KiB/s wr, 1.12k op/s; 3.0 KiB/s, 0 objects/s recovering 2024-09-13T14:56:05.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:05 smithi096 ceph-mon[98901]: 1.10 scrub starts 2024-09-13T14:56:05.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:05 smithi096 ceph-mon[98901]: 1.10 scrub ok 2024-09-13T14:56:05.618 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:05.669 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:05.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:05 smithi204 ceph-mon[87071]: pgmap v916: 109 pgs: 109 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 3.8 KiB/s rd, 8.4 KiB/s wr, 1.12k op/s; 3.0 KiB/s, 0 objects/s recovering 2024-09-13T14:56:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:05 smithi204 ceph-mon[87071]: 1.10 scrub starts 2024-09-13T14:56:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:05 smithi204 ceph-mon[87071]: 1.10 scrub ok 2024-09-13T14:56:05.771 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:05.801 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:05.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:05.906 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:05.944 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:05.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:06.046 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:06.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.082 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:06.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:06.126 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:56:06.126 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:56:06.167 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:56:06.167 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:56:06.184 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:06.202 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:56:06.202 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:56:06.213 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:06.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:06.234 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:56:06.234 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:56:06.264 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:56:06.264 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:56:06.298 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:56:06.298 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:56:06.318 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:06.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:06 smithi096 ceph-mon[98901]: 1.6 scrub starts 2024-09-13T14:56:06.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:06 smithi096 ceph-mon[98901]: 1.6 scrub ok 2024-09-13T14:56:06.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:06.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:06 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1307717884' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:06.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:06 smithi096 ceph-mon[102928]: 1.6 scrub starts 2024-09-13T14:56:06.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:06 smithi096 ceph-mon[102928]: 1.6 scrub ok 2024-09-13T14:56:06.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:06.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:06 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1307717884' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:06.329 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:56:06.329 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:56:06.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.346 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:06.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:06.359 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:56:06.359 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:56:06.447 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:06.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.481 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:06.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:06.582 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:06.617 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:06.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:06 smithi204 ceph-mon[87071]: 1.6 scrub starts 2024-09-13T14:56:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:06 smithi204 ceph-mon[87071]: 1.6 scrub ok 2024-09-13T14:56:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:06 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1307717884' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:06.719 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:06.753 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:06.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:06.855 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:06.889 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:06.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:06.991 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:07.028 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:07.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:07.130 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:07.170 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:07.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:07.272 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:07.308 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:07.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:07.335 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:07.410 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:07.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.443 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:07.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:07.462 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:07 smithi096 ceph-mon[102928]: pgmap v917: 109 pgs: 109 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 3.4 KiB/s rd, 7.4 KiB/s wr, 991 op/s; 2.6 KiB/s, 0 objects/s recovering 2024-09-13T14:56:07.463 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:07 smithi096 ceph-mon[98901]: pgmap v917: 109 pgs: 109 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 3.4 KiB/s rd, 7.4 KiB/s wr, 991 op/s; 2.6 KiB/s, 0 objects/s recovering 2024-09-13T14:56:07.544 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:07.578 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:07.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:07.680 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:07.706 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:07.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:07.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:07 smithi204 ceph-mon[87071]: pgmap v917: 109 pgs: 109 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 3.4 KiB/s rd, 7.4 KiB/s wr, 991 op/s; 2.6 KiB/s, 0 objects/s recovering 2024-09-13T14:56:07.808 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:07.835 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:07.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:07.937 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:07.966 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:07.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:08.067 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:08.099 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:08.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:08.200 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:56:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:56:07] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.51.0" 2024-09-13T14:56:08.201 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:08.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.261 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:08.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:08.362 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:08.389 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:08.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:08.491 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:08.522 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:08.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:08.624 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:08.657 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:08.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:08.759 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:08.793 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:08.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:08.945 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:08.977 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:08.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:09.079 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:09.114 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:09.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:09.217 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:09.249 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:09.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:09.293 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:56:09.351 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:09.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.396 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:09.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:09.498 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:09.531 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:09.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:09 smithi096 ceph-mon[102928]: pgmap v918: 109 pgs: 109 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2024-09-13T14:56:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:09 smithi096 ceph-mon[98901]: pgmap v918: 109 pgs: 109 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2024-09-13T14:56:09.633 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:09.660 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:09.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:09 smithi204 ceph-mon[87071]: pgmap v918: 109 pgs: 109 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2024-09-13T14:56:09.758 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:09.762 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:09.784 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:09.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:09.886 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:09.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.921 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:09.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:10.022 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:10.061 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:10.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:10.163 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:10.194 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:10.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:10.295 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:10.328 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:10.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:10.430 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:10 smithi204 ceph-mon[87071]: 1.f scrub starts 2024-09-13T14:56:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:10 smithi204 ceph-mon[87071]: 1.f scrub ok 2024-09-13T14:56:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:56:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:56:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:10.492 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:10.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:10.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:10 smithi096 ceph-mon[98901]: 1.f scrub starts 2024-09-13T14:56:10.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:10 smithi096 ceph-mon[98901]: 1.f scrub ok 2024-09-13T14:56:10.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:10 smithi096 ceph-mon[102928]: 1.f scrub starts 2024-09-13T14:56:10.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:10 smithi096 ceph-mon[102928]: 1.f scrub ok 2024-09-13T14:56:10.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:10.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.638 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:10.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:10.689 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:10.739 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:10.778 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:10.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:10.880 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:10.911 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:10.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:10.945 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:56:10.946 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:56:11.013 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:11.042 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:11.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:11.144 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:11.186 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:11.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:11.281 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:11 smithi096 ceph-mon[98901]: pgmap v919: 109 pgs: 109 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T14:56:11.288 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:11.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.334 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:11.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:11.435 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:11.476 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:11.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:11.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:11 smithi096 ceph-mon[102928]: pgmap v919: 109 pgs: 109 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T14:56:11.578 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:11.607 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:11.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:11.709 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:11.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:11 smithi204 ceph-mon[87071]: pgmap v919: 109 pgs: 109 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T14:56:11.737 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:11.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:11.839 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:11.873 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:11.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:11.974 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:12.011 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:12.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:12.112 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:12.149 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:12.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:12.251 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:12.277 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:12.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:12.379 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:12.400 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":919,"stamp":"2024-09-13T14:56:10.031160+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3862697957,"num_objects":58896,"num_object_clones":0,"num_object_copies":176688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58896,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":192825,"num_write_kb":8037712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":8404818,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119589,"ondisk_log_size":119589,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16847252,"kb_used_data":11494100,"kb_used_omap":35,"kb_used_meta":5352732,"kb_avail":733113964,"statfs":{"total":767960285184,"available":750708699136,"internally_reserved":0,"allocated":11769958400,"data_stored":11754121800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36830,"internal_metadata":5481197602},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,2,7,4,9,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-847866067,"num_objects":-12937,"num_object_clones":0,"num_object_copies":-38811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-12937,"num_whiteouts":0,"num_read":-13,"num_read_kb":-34,"num_write":12930,"num_write_kb":-74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":-26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.807893"},"pg_stats":[{"pgid":"1.11","version":"82'100","reported_seq":171,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:01.088057+0000","last_change":"2024-09-13T14:56:01.088057+0000","last_active":"2024-09-13T14:56:01.088057+0000","last_peered":"2024-09-13T14:56:01.088057+0000","last_clean":"2024-09-13T14:56:01.088057+0000","last_became_active":"2024-09-13T14:55:55.203197+0000","last_became_peered":"2024-09-13T14:55:55.203197+0000","last_unstale":"2024-09-13T14:56:01.088057+0000","last_undegraded":"2024-09-13T14:56:01.088057+0000","last_fullsized":"2024-09-13T14:56:01.088057+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.087994+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.087994+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:35:16.514920+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":452,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.234507+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:55:58.234507+0000","last_peered":"2024-09-13T14:55:58.234507+0000","last_clean":"2024-09-13T14:55:58.234507+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:55:58.234507+0000","last_undegraded":"2024-09-13T14:55:58.234507+0000","last_fullsized":"2024-09-13T14:55:58.234507+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":257,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.180121+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:55:55.180121+0000","last_peered":"2024-09-13T14:55:55.180121+0000","last_clean":"2024-09-13T14:55:55.180121+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:55:55.180121+0000","last_undegraded":"2024-09-13T14:55:55.180121+0000","last_fullsized":"2024-09-13T14:55:55.180121+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":21,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:04.643714+0000","last_change":"2024-09-13T14:56:04.643672+0000","last_active":"2024-09-13T14:56:04.643714+0000","last_peered":"2024-09-13T14:56:04.643714+0000","last_clean":"2024-09-13T14:56:04.643714+0000","last_became_active":"2024-09-13T14:55:55.185995+0000","last_became_peered":"2024-09-13T14:55:55.185995+0000","last_unstale":"2024-09-13T14:56:04.643714+0000","last_undegraded":"2024-09-13T14:56:04.643714+0000","last_fullsized":"2024-09-13T14:56:04.643714+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.643614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.643614+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:36.495546+0000","scrub_duration":0.001441681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":254,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.182115+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:55:55.182115+0000","last_peered":"2024-09-13T14:55:55.182115+0000","last_clean":"2024-09-13T14:55:55.182115+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:55:55.182115+0000","last_undegraded":"2024-09-13T14:55:55.182115+0000","last_fullsized":"2024-09-13T14:55:55.182115+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":466,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.228119+0000","last_change":"2024-09-13T14:51:44.196384+0000","last_active":"2024-09-13T14:55:58.228119+0000","last_peered":"2024-09-13T14:55:58.228119+0000","last_clean":"2024-09-13T14:55:58.228119+0000","last_became_active":"2024-09-13T14:51:44.196082+0000","last_became_peered":"2024-09-13T14:51:44.196082+0000","last_unstale":"2024-09-13T14:55:58.228119+0000","last_undegraded":"2024-09-13T14:55:58.228119+0000","last_fullsized":"2024-09-13T14:55:58.228119+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:11:03.757803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":602,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:03.240962+0000","last_change":"2024-09-13T14:56:03.240962+0000","last_active":"2024-09-13T14:56:03.240962+0000","last_peered":"2024-09-13T14:56:03.240962+0000","last_clean":"2024-09-13T14:56:03.240962+0000","last_became_active":"2024-09-13T14:55:55.229584+0000","last_became_peered":"2024-09-13T14:55:55.229584+0000","last_unstale":"2024-09-13T14:56:03.240962+0000","last_undegraded":"2024-09-13T14:56:03.240962+0000","last_fullsized":"2024-09-13T14:56:03.240962+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.240882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.240882+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:47:27.889463+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":460,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.226681+0000","last_change":"2024-09-13T14:51:44.199134+0000","last_active":"2024-09-13T14:55:58.226681+0000","last_peered":"2024-09-13T14:55:58.226681+0000","last_clean":"2024-09-13T14:55:58.226681+0000","last_became_active":"2024-09-13T14:51:44.198525+0000","last_became_peered":"2024-09-13T14:51:44.198525+0000","last_unstale":"2024-09-13T14:55:58.226681+0000","last_undegraded":"2024-09-13T14:55:58.226681+0000","last_fullsized":"2024-09-13T14:55:58.226681+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:44.009127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":258,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.179704+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T14:55:55.179704+0000","last_peered":"2024-09-13T14:55:55.179704+0000","last_clean":"2024-09-13T14:55:55.179704+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T14:55:55.179704+0000","last_undegraded":"2024-09-13T14:55:55.179704+0000","last_fullsized":"2024-09-13T14:55:55.179704+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":21,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:03.605491+0000","last_change":"2024-09-13T14:56:03.605447+0000","last_active":"2024-09-13T14:56:03.605491+0000","last_peered":"2024-09-13T14:56:03.605491+0000","last_clean":"2024-09-13T14:56:03.605491+0000","last_became_active":"2024-09-13T14:55:55.218939+0000","last_became_peered":"2024-09-13T14:55:55.218939+0000","last_unstale":"2024-09-13T14:56:03.605491+0000","last_undegraded":"2024-09-13T14:56:03.605491+0000","last_fullsized":"2024-09-13T14:56:03.605491+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.605356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.605356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:12.467371+0000","scrub_duration":0.0020269620000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":113,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.234761+0000","last_change":"2024-09-13T14:51:44.207246+0000","last_active":"2024-09-13T14:55:58.234761+0000","last_peered":"2024-09-13T14:55:58.234761+0000","last_clean":"2024-09-13T14:55:58.234761+0000","last_became_active":"2024-09-13T14:51:44.206637+0000","last_became_peered":"2024-09-13T14:51:44.206637+0000","last_unstale":"2024-09-13T14:55:58.234761+0000","last_undegraded":"2024-09-13T14:55:58.234761+0000","last_fullsized":"2024-09-13T14:55:58.234761+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:42.020301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":85,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.181391+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T14:55:55.181391+0000","last_peered":"2024-09-13T14:55:55.181391+0000","last_clean":"2024-09-13T14:55:55.181391+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T14:55:55.181391+0000","last_undegraded":"2024-09-13T14:55:55.181391+0000","last_fullsized":"2024-09-13T14:55:55.181391+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":21,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:01.654887+0000","last_change":"2024-09-13T14:56:01.654803+0000","last_active":"2024-09-13T14:56:01.654887+0000","last_peered":"2024-09-13T14:56:01.654887+0000","last_clean":"2024-09-13T14:56:01.654887+0000","last_became_active":"2024-09-13T14:55:55.188788+0000","last_became_peered":"2024-09-13T14:55:55.188788+0000","last_unstale":"2024-09-13T14:56:01.654887+0000","last_undegraded":"2024-09-13T14:56:01.654887+0000","last_fullsized":"2024-09-13T14:56:01.654887+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.654717+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.654717+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:38.649187+0000","scrub_duration":0.0060902070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"291'11857","reported_seq":11886,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:07.128919+0000","last_change":"2024-09-13T14:53:13.573293+0000","last_active":"2024-09-13T14:56:07.128919+0000","last_peered":"2024-09-13T14:56:07.128919+0000","last_clean":"2024-09-13T14:56:07.128919+0000","last_became_active":"2024-09-13T14:53:13.572925+0000","last_became_peered":"2024-09-13T14:53:13.572925+0000","last_unstale":"2024-09-13T14:56:07.128919+0000","last_undegraded":"2024-09-13T14:56:07.128919+0000","last_fullsized":"2024-09-13T14:56:07.128919+0000","mapping_epoch":270,"log_start":"276'4136","ondisk_log_start":"276'4136","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7721,"log_dups_size":0,"ondisk_log_size":7721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:13.459532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242679808,"num_objects":3703,"num_object_clones":0,"num_object_copies":11109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11857,"num_write_kb":497920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":437,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.248450+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:55:58.248450+0000","last_peered":"2024-09-13T14:55:58.248450+0000","last_clean":"2024-09-13T14:55:58.248450+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:55:58.248450+0000","last_undegraded":"2024-09-13T14:55:58.248450+0000","last_fullsized":"2024-09-13T14:55:58.248450+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":277,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.184289+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T14:55:55.184289+0000","last_peered":"2024-09-13T14:55:55.184289+0000","last_clean":"2024-09-13T14:55:55.184289+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T14:55:55.184289+0000","last_undegraded":"2024-09-13T14:55:55.184289+0000","last_fullsized":"2024-09-13T14:55:55.184289+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":415,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:01.843753+0000","last_change":"2024-09-13T14:56:01.843753+0000","last_active":"2024-09-13T14:56:01.843753+0000","last_peered":"2024-09-13T14:56:01.843753+0000","last_clean":"2024-09-13T14:56:01.843753+0000","last_became_active":"2024-09-13T14:55:55.188079+0000","last_became_peered":"2024-09-13T14:55:55.188079+0000","last_unstale":"2024-09-13T14:56:01.843753+0000","last_undegraded":"2024-09-13T14:56:01.843753+0000","last_fullsized":"2024-09-13T14:56:01.843753+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.843498+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.843498+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:53:22.355050+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.5","version":"291'12066","reported_seq":12093,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:06.815353+0000","last_change":"2024-09-13T14:53:13.865500+0000","last_active":"2024-09-13T14:56:06.815353+0000","last_peered":"2024-09-13T14:56:06.815353+0000","last_clean":"2024-09-13T14:56:06.815353+0000","last_became_active":"2024-09-13T14:53:13.864783+0000","last_became_peered":"2024-09-13T14:53:13.864783+0000","last_unstale":"2024-09-13T14:56:06.815353+0000","last_undegraded":"2024-09-13T14:56:06.815353+0000","last_fullsized":"2024-09-13T14:56:06.815353+0000","mapping_epoch":270,"log_start":"276'4681","ondisk_log_start":"276'4681","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7385,"log_dups_size":0,"ondisk_log_size":7385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:08:47.267226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251658240,"num_objects":3840,"num_object_clones":0,"num_object_copies":11520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12066,"num_write_kb":508992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":257,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.180880+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:55:55.180880+0000","last_peered":"2024-09-13T14:55:55.180880+0000","last_clean":"2024-09-13T14:55:55.180880+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:55:55.180880+0000","last_undegraded":"2024-09-13T14:55:55.180880+0000","last_fullsized":"2024-09-13T14:55:55.180880+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":312,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.176578+0000","last_change":"2024-09-13T14:51:11.059483+0000","last_active":"2024-09-13T14:55:55.176578+0000","last_peered":"2024-09-13T14:55:55.176578+0000","last_clean":"2024-09-13T14:55:55.176578+0000","last_became_active":"2024-09-13T14:51:11.059262+0000","last_became_peered":"2024-09-13T14:51:11.059262+0000","last_unstale":"2024-09-13T14:55:55.176578+0000","last_undegraded":"2024-09-13T14:55:55.176578+0000","last_fullsized":"2024-09-13T14:55:55.176578+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:45:35.152631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":114,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.186542+0000","last_change":"2024-09-13T14:55:55.186230+0000","last_active":"2024-09-13T14:55:55.186542+0000","last_peered":"2024-09-13T14:55:55.186542+0000","last_clean":"2024-09-13T14:55:55.186542+0000","last_became_active":"2024-09-13T14:55:55.185755+0000","last_became_peered":"2024-09-13T14:55:55.185755+0000","last_unstale":"2024-09-13T14:55:55.186542+0000","last_undegraded":"2024-09-13T14:55:55.186542+0000","last_fullsized":"2024-09-13T14:55:55.186542+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:48:34.095505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:48:34.095505+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001382681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.6","version":"291'12295","reported_seq":12349,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.716867+0000","last_change":"2024-09-13T14:53:13.864168+0000","last_active":"2024-09-13T14:56:08.716867+0000","last_peered":"2024-09-13T14:56:08.716867+0000","last_clean":"2024-09-13T14:56:08.716867+0000","last_became_active":"2024-09-13T14:53:13.863816+0000","last_became_peered":"2024-09-13T14:53:13.863816+0000","last_unstale":"2024-09-13T14:56:08.716867+0000","last_undegraded":"2024-09-13T14:56:08.716867+0000","last_fullsized":"2024-09-13T14:56:08.716867+0000","mapping_epoch":270,"log_start":"276'5057","ondisk_log_start":"276'5057","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7238,"log_dups_size":0,"ondisk_log_size":7238,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:08.763079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239534080,"num_objects":3655,"num_object_clones":0,"num_object_copies":10965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12295,"num_write_kb":510400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":267,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.181019+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T14:55:55.181019+0000","last_peered":"2024-09-13T14:55:55.181019+0000","last_clean":"2024-09-13T14:55:55.181019+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T14:55:55.181019+0000","last_undegraded":"2024-09-13T14:55:55.181019+0000","last_fullsized":"2024-09-13T14:55:55.181019+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":85,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.181641+0000","last_change":"2024-09-13T14:51:44.190879+0000","last_active":"2024-09-13T14:55:55.181641+0000","last_peered":"2024-09-13T14:55:55.181641+0000","last_clean":"2024-09-13T14:55:55.181641+0000","last_became_active":"2024-09-13T14:51:44.190617+0000","last_became_peered":"2024-09-13T14:51:44.190617+0000","last_unstale":"2024-09-13T14:55:55.181641+0000","last_undegraded":"2024-09-13T14:55:55.181641+0000","last_fullsized":"2024-09-13T14:55:55.181641+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:19.536298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":21,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:02.325773+0000","last_change":"2024-09-13T14:56:02.325723+0000","last_active":"2024-09-13T14:56:02.325773+0000","last_peered":"2024-09-13T14:56:02.325773+0000","last_clean":"2024-09-13T14:56:02.325773+0000","last_became_active":"2024-09-13T14:55:55.182743+0000","last_became_peered":"2024-09-13T14:55:55.182743+0000","last_unstale":"2024-09-13T14:56:02.325773+0000","last_undegraded":"2024-09-13T14:56:02.325773+0000","last_fullsized":"2024-09-13T14:56:02.325773+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:02.325629+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:02.325629+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:47.868641+0000","scrub_duration":0.0049176200000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.7","version":"291'11721","reported_seq":11775,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.358120+0000","last_change":"2024-09-13T14:53:13.573599+0000","last_active":"2024-09-13T14:56:08.358120+0000","last_peered":"2024-09-13T14:56:08.358120+0000","last_clean":"2024-09-13T14:56:08.358120+0000","last_became_active":"2024-09-13T14:53:13.572977+0000","last_became_peered":"2024-09-13T14:53:13.572977+0000","last_unstale":"2024-09-13T14:56:08.358120+0000","last_undegraded":"2024-09-13T14:56:08.358120+0000","last_fullsized":"2024-09-13T14:56:08.358120+0000","mapping_epoch":270,"log_start":"276'4479","ondisk_log_start":"276'4479","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7242,"log_dups_size":0,"ondisk_log_size":7242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:54.554732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235470848,"num_objects":3593,"num_object_clones":0,"num_object_copies":10779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11721,"num_write_kb":490048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":275,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.184006+0000","last_change":"2024-09-13T14:48:32.206148+0000","last_active":"2024-09-13T14:55:55.184006+0000","last_peered":"2024-09-13T14:55:55.184006+0000","last_clean":"2024-09-13T14:55:55.184006+0000","last_became_active":"2024-09-13T14:48:32.205832+0000","last_became_peered":"2024-09-13T14:48:32.205832+0000","last_unstale":"2024-09-13T14:55:55.184006+0000","last_undegraded":"2024-09-13T14:55:55.184006+0000","last_fullsized":"2024-09-13T14:55:55.184006+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:55.026094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":447,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.226280+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:55:58.226280+0000","last_peered":"2024-09-13T14:55:58.226280+0000","last_clean":"2024-09-13T14:55:58.226280+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:55:58.226280+0000","last_undegraded":"2024-09-13T14:55:58.226280+0000","last_fullsized":"2024-09-13T14:55:58.226280+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"291'12165","reported_seq":12219,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.677650+0000","last_change":"2024-09-13T14:53:13.574753+0000","last_active":"2024-09-13T14:56:08.677650+0000","last_peered":"2024-09-13T14:56:08.677650+0000","last_clean":"2024-09-13T14:56:08.677650+0000","last_became_active":"2024-09-13T14:53:13.574453+0000","last_became_peered":"2024-09-13T14:53:13.574453+0000","last_unstale":"2024-09-13T14:56:08.677650+0000","last_undegraded":"2024-09-13T14:56:08.677650+0000","last_fullsized":"2024-09-13T14:56:08.677650+0000","mapping_epoch":270,"log_start":"277'5273","ondisk_log_start":"277'5273","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":6892,"log_dups_size":0,"ondisk_log_size":6892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:56.710721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241500160,"num_objects":3685,"num_object_clones":0,"num_object_copies":11055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12165,"num_write_kb":507200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":438,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.179513+0000","last_change":"2024-09-13T14:51:01.996634+0000","last_active":"2024-09-13T14:55:55.179513+0000","last_peered":"2024-09-13T14:55:55.179513+0000","last_clean":"2024-09-13T14:55:55.179513+0000","last_became_active":"2024-09-13T14:51:01.996264+0000","last_became_peered":"2024-09-13T14:51:01.996264+0000","last_unstale":"2024-09-13T14:55:55.179513+0000","last_undegraded":"2024-09-13T14:55:55.179513+0000","last_fullsized":"2024-09-13T14:55:55.179513+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:30.083115+0000","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":449,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.233982+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:55:58.233982+0000","last_peered":"2024-09-13T14:55:58.233982+0000","last_clean":"2024-09-13T14:55:58.233982+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:55:58.233982+0000","last_undegraded":"2024-09-13T14:55:58.233982+0000","last_fullsized":"2024-09-13T14:55:58.233982+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":275,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.176911+0000","last_change":"2024-09-13T14:48:32.196993+0000","last_active":"2024-09-13T14:55:55.176911+0000","last_peered":"2024-09-13T14:55:55.176911+0000","last_clean":"2024-09-13T14:55:55.176911+0000","last_became_active":"2024-09-13T14:48:32.196681+0000","last_became_peered":"2024-09-13T14:48:32.196681+0000","last_unstale":"2024-09-13T14:55:55.176911+0000","last_undegraded":"2024-09-13T14:55:55.176911+0000","last_fullsized":"2024-09-13T14:55:55.176911+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:56.040494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"291'12003","reported_seq":12057,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.384824+0000","last_change":"2024-09-13T14:53:13.572344+0000","last_active":"2024-09-13T14:56:08.384824+0000","last_peered":"2024-09-13T14:56:08.384824+0000","last_clean":"2024-09-13T14:56:08.384824+0000","last_became_active":"2024-09-13T14:53:13.572080+0000","last_became_peered":"2024-09-13T14:53:13.572080+0000","last_unstale":"2024-09-13T14:56:08.384824+0000","last_undegraded":"2024-09-13T14:56:08.384824+0000","last_fullsized":"2024-09-13T14:56:08.384824+0000","mapping_epoch":270,"log_start":"276'4848","ondisk_log_start":"276'4848","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7155,"log_dups_size":0,"ondisk_log_size":7155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:37:22.683509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241238016,"num_objects":3681,"num_object_clones":0,"num_object_copies":11043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12003,"num_write_kb":501888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":198,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.184115+0000","last_change":"2024-09-13T14:40:27.868402+0000","last_active":"2024-09-13T14:55:55.184115+0000","last_peered":"2024-09-13T14:55:55.184115+0000","last_clean":"2024-09-13T14:55:55.184115+0000","last_became_active":"2024-09-13T14:40:22.966301+0000","last_became_peered":"2024-09-13T14:40:22.966301+0000","last_unstale":"2024-09-13T14:55:55.184115+0000","last_undegraded":"2024-09-13T14:55:55.184115+0000","last_fullsized":"2024-09-13T14:55:55.184115+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:18.660186+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":75,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.181761+0000","last_change":"2024-09-13T14:48:32.200521+0000","last_active":"2024-09-13T14:55:55.181761+0000","last_peered":"2024-09-13T14:55:55.181761+0000","last_clean":"2024-09-13T14:55:55.181761+0000","last_became_active":"2024-09-13T14:48:32.200256+0000","last_became_peered":"2024-09-13T14:48:32.200256+0000","last_unstale":"2024-09-13T14:55:55.181761+0000","last_undegraded":"2024-09-13T14:55:55.181761+0000","last_fullsized":"2024-09-13T14:55:55.181761+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:38.899629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":272,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.177955+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:55:55.177955+0000","last_peered":"2024-09-13T14:55:55.177955+0000","last_clean":"2024-09-13T14:55:55.177955+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:55:55.177955+0000","last_undegraded":"2024-09-13T14:55:55.177955+0000","last_fullsized":"2024-09-13T14:55:55.177955+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":602,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:01.254863+0000","last_change":"2024-09-13T14:56:01.254863+0000","last_active":"2024-09-13T14:56:01.254863+0000","last_peered":"2024-09-13T14:56:01.254863+0000","last_clean":"2024-09-13T14:56:01.254863+0000","last_became_active":"2024-09-13T14:55:55.214258+0000","last_became_peered":"2024-09-13T14:55:55.214258+0000","last_unstale":"2024-09-13T14:56:01.254863+0000","last_undegraded":"2024-09-13T14:56:01.254863+0000","last_fullsized":"2024-09-13T14:56:01.254863+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.254778+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.254778+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:13:40.694950+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"291'11991","reported_seq":12020,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:07.005374+0000","last_change":"2024-09-13T14:53:13.572602+0000","last_active":"2024-09-13T14:56:07.005374+0000","last_peered":"2024-09-13T14:56:07.005374+0000","last_clean":"2024-09-13T14:56:07.005374+0000","last_became_active":"2024-09-13T14:53:13.572101+0000","last_became_peered":"2024-09-13T14:53:13.572101+0000","last_unstale":"2024-09-13T14:56:07.005374+0000","last_undegraded":"2024-09-13T14:56:07.005374+0000","last_fullsized":"2024-09-13T14:56:07.005374+0000","mapping_epoch":270,"log_start":"276'4587","ondisk_log_start":"276'4587","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7404,"log_dups_size":0,"ondisk_log_size":7404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:49.928419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246480896,"num_objects":3761,"num_object_clones":0,"num_object_copies":11283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11991,"num_write_kb":504064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":433,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.234592+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:55:58.234592+0000","last_peered":"2024-09-13T14:55:58.234592+0000","last_clean":"2024-09-13T14:55:58.234592+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:55:58.234592+0000","last_undegraded":"2024-09-13T14:55:58.234592+0000","last_fullsized":"2024-09-13T14:55:58.234592+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":275,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.177456+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T14:55:55.177456+0000","last_peered":"2024-09-13T14:55:55.177456+0000","last_clean":"2024-09-13T14:55:55.177456+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T14:55:55.177456+0000","last_undegraded":"2024-09-13T14:55:55.177456+0000","last_fullsized":"2024-09-13T14:55:55.177456+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"291'11990","reported_seq":12044,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.372706+0000","last_change":"2024-09-13T14:53:13.571521+0000","last_active":"2024-09-13T14:56:08.372706+0000","last_peered":"2024-09-13T14:56:08.372706+0000","last_clean":"2024-09-13T14:56:08.372706+0000","last_became_active":"2024-09-13T14:53:13.571234+0000","last_became_peered":"2024-09-13T14:53:13.571234+0000","last_unstale":"2024-09-13T14:56:08.372706+0000","last_undegraded":"2024-09-13T14:56:08.372706+0000","last_fullsized":"2024-09-13T14:56:08.372706+0000","mapping_epoch":270,"log_start":"276'4811","ondisk_log_start":"276'4811","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7179,"log_dups_size":0,"ondisk_log_size":7179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:15.825916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236584960,"num_objects":3610,"num_object_clones":0,"num_object_copies":10830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11990,"num_write_kb":499200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":201,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.182853+0000","last_change":"2024-09-13T14:48:32.205288+0000","last_active":"2024-09-13T14:55:55.182853+0000","last_peered":"2024-09-13T14:55:55.182853+0000","last_clean":"2024-09-13T14:55:55.182853+0000","last_became_active":"2024-09-13T14:48:32.204657+0000","last_became_peered":"2024-09-13T14:48:32.204657+0000","last_unstale":"2024-09-13T14:55:55.182853+0000","last_undegraded":"2024-09-13T14:55:55.182853+0000","last_fullsized":"2024-09-13T14:55:55.182853+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:52.513329+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":106,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.235341+0000","last_change":"2024-09-13T14:51:11.059644+0000","last_active":"2024-09-13T14:55:58.235341+0000","last_peered":"2024-09-13T14:55:58.235341+0000","last_clean":"2024-09-13T14:55:58.235341+0000","last_became_active":"2024-09-13T14:51:11.058874+0000","last_became_peered":"2024-09-13T14:51:11.058874+0000","last_unstale":"2024-09-13T14:55:58.235341+0000","last_undegraded":"2024-09-13T14:55:58.235341+0000","last_fullsized":"2024-09-13T14:55:58.235341+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:15:14.736476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":434,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.239241+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:55:58.239241+0000","last_peered":"2024-09-13T14:55:58.239241+0000","last_clean":"2024-09-13T14:55:58.239241+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:55:58.239241+0000","last_undegraded":"2024-09-13T14:55:58.239241+0000","last_fullsized":"2024-09-13T14:55:58.239241+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"291'11922","reported_seq":11951,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:06.810022+0000","last_change":"2024-09-13T14:53:13.575349+0000","last_active":"2024-09-13T14:56:06.810022+0000","last_peered":"2024-09-13T14:56:06.810022+0000","last_clean":"2024-09-13T14:56:06.810022+0000","last_became_active":"2024-09-13T14:53:13.575026+0000","last_became_peered":"2024-09-13T14:53:13.575026+0000","last_unstale":"2024-09-13T14:56:06.810022+0000","last_undegraded":"2024-09-13T14:56:06.810022+0000","last_fullsized":"2024-09-13T14:56:06.810022+0000","mapping_epoch":270,"log_start":"276'4545","ondisk_log_start":"276'4545","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7377,"log_dups_size":0,"ondisk_log_size":7377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:42:27.458209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249430016,"num_objects":3806,"num_object_clones":0,"num_object_copies":11418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11922,"num_write_kb":503296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":439,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.568960+0000","last_change":"2024-09-13T14:55:58.568852+0000","last_active":"2024-09-13T14:55:58.568960+0000","last_peered":"2024-09-13T14:55:58.568960+0000","last_clean":"2024-09-13T14:55:58.568960+0000","last_became_active":"2024-09-13T14:55:51.169987+0000","last_became_peered":"2024-09-13T14:55:51.169987+0000","last_unstale":"2024-09-13T14:55:58.568960+0000","last_undegraded":"2024-09-13T14:55:58.568960+0000","last_fullsized":"2024-09-13T14:55:58.568960+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.568750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.568750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:56.273429+0000","scrub_duration":0.0074886340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":817,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.182144+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:55:55.182144+0000","last_peered":"2024-09-13T14:55:55.182144+0000","last_clean":"2024-09-13T14:55:55.182144+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:55:55.182144+0000","last_undegraded":"2024-09-13T14:55:55.182144+0000","last_fullsized":"2024-09-13T14:55:55.182144+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":106,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.233932+0000","last_change":"2024-09-13T14:51:11.059953+0000","last_active":"2024-09-13T14:55:58.233932+0000","last_peered":"2024-09-13T14:55:58.233932+0000","last_clean":"2024-09-13T14:55:58.233932+0000","last_became_active":"2024-09-13T14:51:11.059396+0000","last_became_peered":"2024-09-13T14:51:11.059396+0000","last_unstale":"2024-09-13T14:55:58.233932+0000","last_undegraded":"2024-09-13T14:55:58.233932+0000","last_fullsized":"2024-09-13T14:55:58.233932+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:11:51.119142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":75,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.182726+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T14:55:55.182726+0000","last_peered":"2024-09-13T14:55:55.182726+0000","last_clean":"2024-09-13T14:55:55.182726+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T14:55:55.182726+0000","last_undegraded":"2024-09-13T14:55:55.182726+0000","last_fullsized":"2024-09-13T14:55:55.182726+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"291'11946","reported_seq":12000,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.389796+0000","last_change":"2024-09-13T14:53:13.569268+0000","last_active":"2024-09-13T14:56:08.389796+0000","last_peered":"2024-09-13T14:56:08.389796+0000","last_clean":"2024-09-13T14:56:08.389796+0000","last_became_active":"2024-09-13T14:53:13.568997+0000","last_became_peered":"2024-09-13T14:53:13.568997+0000","last_unstale":"2024-09-13T14:56:08.389796+0000","last_undegraded":"2024-09-13T14:56:08.389796+0000","last_fullsized":"2024-09-13T14:56:08.389796+0000","mapping_epoch":270,"log_start":"276'4707","ondisk_log_start":"276'4707","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7239,"log_dups_size":0,"ondisk_log_size":7239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:20:18.821269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239075328,"num_objects":3648,"num_object_clones":0,"num_object_copies":10944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11946,"num_write_kb":499008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":596,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.245841+0000","last_change":"2024-09-13T14:55:59.245841+0000","last_active":"2024-09-13T14:55:59.245841+0000","last_peered":"2024-09-13T14:55:59.245841+0000","last_clean":"2024-09-13T14:55:59.245841+0000","last_became_active":"2024-09-13T14:55:51.159487+0000","last_became_peered":"2024-09-13T14:55:51.159487+0000","last_unstale":"2024-09-13T14:55:59.245841+0000","last_undegraded":"2024-09-13T14:55:59.245841+0000","last_fullsized":"2024-09-13T14:55:59.245841+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.245213+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.245213+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:54.299577+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":502,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.179900+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:55:55.179900+0000","last_peered":"2024-09-13T14:55:55.179900+0000","last_clean":"2024-09-13T14:55:55.179900+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:55:55.179900+0000","last_undegraded":"2024-09-13T14:55:55.179900+0000","last_fullsized":"2024-09-13T14:55:55.179900+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":468,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.234680+0000","last_change":"2024-09-13T14:51:11.061677+0000","last_active":"2024-09-13T14:55:58.234680+0000","last_peered":"2024-09-13T14:55:58.234680+0000","last_clean":"2024-09-13T14:55:58.234680+0000","last_became_active":"2024-09-13T14:51:11.061205+0000","last_became_peered":"2024-09-13T14:51:11.061205+0000","last_unstale":"2024-09-13T14:55:58.234680+0000","last_undegraded":"2024-09-13T14:55:58.234680+0000","last_fullsized":"2024-09-13T14:55:58.234680+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:38.521784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":472,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.226832+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T14:55:58.226832+0000","last_peered":"2024-09-13T14:55:58.226832+0000","last_clean":"2024-09-13T14:55:58.226832+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T14:55:58.226832+0000","last_undegraded":"2024-09-13T14:55:58.226832+0000","last_fullsized":"2024-09-13T14:55:58.226832+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"291'12285","reported_seq":12338,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.714788+0000","last_change":"2024-09-13T14:53:13.571649+0000","last_active":"2024-09-13T14:56:08.714788+0000","last_peered":"2024-09-13T14:56:08.714788+0000","last_clean":"2024-09-13T14:56:08.714788+0000","last_became_active":"2024-09-13T14:53:13.571377+0000","last_became_peered":"2024-09-13T14:53:13.571377+0000","last_unstale":"2024-09-13T14:56:08.714788+0000","last_undegraded":"2024-09-13T14:56:08.714788+0000","last_fullsized":"2024-09-13T14:56:08.714788+0000","mapping_epoch":270,"log_start":"277'5370","ondisk_log_start":"277'5370","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":6915,"log_dups_size":0,"ondisk_log_size":6915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:34:54.743293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237961216,"num_objects":3631,"num_object_clones":0,"num_object_copies":10893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12285,"num_write_kb":509312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":68,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.574851+0000","last_change":"2024-09-13T14:55:59.574827+0000","last_active":"2024-09-13T14:55:59.574851+0000","last_peered":"2024-09-13T14:55:59.574851+0000","last_clean":"2024-09-13T14:55:59.574851+0000","last_became_active":"2024-09-13T14:55:51.162903+0000","last_became_peered":"2024-09-13T14:55:51.162903+0000","last_unstale":"2024-09-13T14:55:59.574851+0000","last_undegraded":"2024-09-13T14:55:59.574851+0000","last_fullsized":"2024-09-13T14:55:59.574851+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.574773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.574773+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:39.074072+0000","scrub_duration":0.001553708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":920,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.241222+0000","last_change":"2024-09-13T14:51:11.072066+0000","last_active":"2024-09-13T14:55:58.241222+0000","last_peered":"2024-09-13T14:55:58.241222+0000","last_clean":"2024-09-13T14:55:58.241222+0000","last_became_active":"2024-09-13T14:51:11.061307+0000","last_became_peered":"2024-09-13T14:51:11.061307+0000","last_unstale":"2024-09-13T14:55:58.241222+0000","last_undegraded":"2024-09-13T14:55:58.241222+0000","last_fullsized":"2024-09-13T14:55:58.241222+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:06.774106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":449,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.238527+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T14:55:58.238527+0000","last_peered":"2024-09-13T14:55:58.238527+0000","last_clean":"2024-09-13T14:55:58.238527+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T14:55:58.238527+0000","last_undegraded":"2024-09-13T14:55:58.238527+0000","last_fullsized":"2024-09-13T14:55:58.238527+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":467,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.239160+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:55:58.239160+0000","last_peered":"2024-09-13T14:55:58.239160+0000","last_clean":"2024-09-13T14:55:58.239160+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:55:58.239160+0000","last_undegraded":"2024-09-13T14:55:58.239160+0000","last_fullsized":"2024-09-13T14:55:58.239160+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"291'11737","reported_seq":11766,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:07.124649+0000","last_change":"2024-09-13T14:53:13.864463+0000","last_active":"2024-09-13T14:56:07.124649+0000","last_peered":"2024-09-13T14:56:07.124649+0000","last_clean":"2024-09-13T14:56:07.124649+0000","last_became_active":"2024-09-13T14:53:13.864138+0000","last_became_peered":"2024-09-13T14:53:13.864138+0000","last_unstale":"2024-09-13T14:56:07.124649+0000","last_undegraded":"2024-09-13T14:56:07.124649+0000","last_fullsized":"2024-09-13T14:56:07.124649+0000","mapping_epoch":270,"log_start":"276'3966","ondisk_log_start":"276'3966","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7771,"log_dups_size":0,"ondisk_log_size":7771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:40.441931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239140864,"num_objects":3649,"num_object_clones":0,"num_object_copies":10947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11737,"num_write_kb":492352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":211,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:01.658290+0000","last_change":"2024-09-13T14:56:01.658240+0000","last_active":"2024-09-13T14:56:01.658290+0000","last_peered":"2024-09-13T14:56:01.658290+0000","last_clean":"2024-09-13T14:56:01.658290+0000","last_became_active":"2024-09-13T14:55:51.160480+0000","last_became_peered":"2024-09-13T14:55:51.160480+0000","last_unstale":"2024-09-13T14:56:01.658290+0000","last_undegraded":"2024-09-13T14:56:01.658290+0000","last_fullsized":"2024-09-13T14:56:01.658290+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.658143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.658143+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:19.262955+0000","scrub_duration":0.0038903079999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":917,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.183383+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T14:55:55.183383+0000","last_peered":"2024-09-13T14:55:55.183383+0000","last_clean":"2024-09-13T14:55:55.183383+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T14:55:55.183383+0000","last_undegraded":"2024-09-13T14:55:55.183383+0000","last_fullsized":"2024-09-13T14:55:55.183383+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":276,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.177408+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T14:55:55.177408+0000","last_peered":"2024-09-13T14:55:55.177408+0000","last_clean":"2024-09-13T14:55:55.177408+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T14:55:55.177408+0000","last_undegraded":"2024-09-13T14:55:55.177408+0000","last_fullsized":"2024-09-13T14:55:55.177408+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":467,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.237116+0000","last_change":"2024-09-13T14:51:11.060994+0000","last_active":"2024-09-13T14:55:58.237116+0000","last_peered":"2024-09-13T14:55:58.237116+0000","last_clean":"2024-09-13T14:55:58.237116+0000","last_became_active":"2024-09-13T14:51:11.060666+0000","last_became_peered":"2024-09-13T14:51:11.060666+0000","last_unstale":"2024-09-13T14:55:58.237116+0000","last_undegraded":"2024-09-13T14:55:58.237116+0000","last_fullsized":"2024-09-13T14:55:58.237116+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:40:15.828766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"291'12084","reported_seq":12138,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.747805+0000","last_change":"2024-09-13T14:53:13.572932+0000","last_active":"2024-09-13T14:56:08.747805+0000","last_peered":"2024-09-13T14:56:08.747805+0000","last_clean":"2024-09-13T14:56:08.747805+0000","last_became_active":"2024-09-13T14:53:13.572464+0000","last_became_peered":"2024-09-13T14:53:13.572464+0000","last_unstale":"2024-09-13T14:56:08.747805+0000","last_undegraded":"2024-09-13T14:56:08.747805+0000","last_fullsized":"2024-09-13T14:56:08.747805+0000","mapping_epoch":270,"log_start":"276'4908","ondisk_log_start":"276'4908","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7176,"log_dups_size":0,"ondisk_log_size":7176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:47:06.440193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238944256,"num_objects":3646,"num_object_clones":0,"num_object_copies":10938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12084,"num_write_kb":503360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":115,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:01.916925+0000","last_change":"2024-09-13T14:56:01.916851+0000","last_active":"2024-09-13T14:56:01.916925+0000","last_peered":"2024-09-13T14:56:01.916925+0000","last_clean":"2024-09-13T14:56:01.916925+0000","last_became_active":"2024-09-13T14:55:51.195404+0000","last_became_peered":"2024-09-13T14:55:51.195404+0000","last_unstale":"2024-09-13T14:56:01.916925+0000","last_undegraded":"2024-09-13T14:56:01.916925+0000","last_fullsized":"2024-09-13T14:56:01.916925+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.916764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.916764+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:47:35.114766+0000","scrub_duration":0.0035298819999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":272,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.182372+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:55:55.182372+0000","last_peered":"2024-09-13T14:55:55.182372+0000","last_clean":"2024-09-13T14:55:55.182372+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:55:55.182372+0000","last_undegraded":"2024-09-13T14:55:55.182372+0000","last_fullsized":"2024-09-13T14:55:55.182372+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":952,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.237307+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:55:58.237307+0000","last_peered":"2024-09-13T14:55:58.237307+0000","last_clean":"2024-09-13T14:55:58.237307+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:55:58.237307+0000","last_undegraded":"2024-09-13T14:55:58.237307+0000","last_fullsized":"2024-09-13T14:55:58.237307+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":424,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.237985+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:55:58.237985+0000","last_peered":"2024-09-13T14:55:58.237985+0000","last_clean":"2024-09-13T14:55:58.237985+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:55:58.237985+0000","last_undegraded":"2024-09-13T14:55:58.237985+0000","last_fullsized":"2024-09-13T14:55:58.237985+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"291'12011","reported_seq":12065,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.696034+0000","last_change":"2024-09-13T14:53:13.575181+0000","last_active":"2024-09-13T14:56:08.696034+0000","last_peered":"2024-09-13T14:56:08.696034+0000","last_clean":"2024-09-13T14:56:08.696034+0000","last_became_active":"2024-09-13T14:53:13.574893+0000","last_became_peered":"2024-09-13T14:53:13.574893+0000","last_unstale":"2024-09-13T14:56:08.696034+0000","last_undegraded":"2024-09-13T14:56:08.696034+0000","last_fullsized":"2024-09-13T14:56:08.696034+0000","mapping_epoch":270,"log_start":"277'5179","ondisk_log_start":"277'5179","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":6832,"log_dups_size":0,"ondisk_log_size":6832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:38:28.331899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242024448,"num_objects":3693,"num_object_clones":0,"num_object_copies":11079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12011,"num_write_kb":502528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":105,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:04.997218+0000","last_change":"2024-09-13T14:56:04.997125+0000","last_active":"2024-09-13T14:56:04.997218+0000","last_peered":"2024-09-13T14:56:04.997218+0000","last_clean":"2024-09-13T14:56:04.997218+0000","last_became_active":"2024-09-13T14:55:51.164957+0000","last_became_peered":"2024-09-13T14:55:51.164957+0000","last_unstale":"2024-09-13T14:56:04.997218+0000","last_undegraded":"2024-09-13T14:56:04.997218+0000","last_fullsized":"2024-09-13T14:56:04.997218+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.997001+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.997001+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:58:48.818249+0000","scrub_duration":0.019401547000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":449,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.237215+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T14:55:58.237215+0000","last_peered":"2024-09-13T14:55:58.237215+0000","last_clean":"2024-09-13T14:55:58.237215+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T14:55:58.237215+0000","last_undegraded":"2024-09-13T14:55:58.237215+0000","last_fullsized":"2024-09-13T14:55:58.237215+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":504,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.181090+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:55:55.181090+0000","last_peered":"2024-09-13T14:55:55.181090+0000","last_clean":"2024-09-13T14:55:55.181090+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:55:55.181090+0000","last_undegraded":"2024-09-13T14:55:55.181090+0000","last_fullsized":"2024-09-13T14:55:55.181090+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":436,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.226176+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:55:58.226176+0000","last_peered":"2024-09-13T14:55:58.226176+0000","last_clean":"2024-09-13T14:55:58.226176+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:55:58.226176+0000","last_undegraded":"2024-09-13T14:55:58.226176+0000","last_fullsized":"2024-09-13T14:55:58.226176+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"291'12090","reported_seq":12144,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.763219+0000","last_change":"2024-09-13T14:53:13.864099+0000","last_active":"2024-09-13T14:56:08.763219+0000","last_peered":"2024-09-13T14:56:08.763219+0000","last_clean":"2024-09-13T14:56:08.763219+0000","last_became_active":"2024-09-13T14:53:13.863777+0000","last_became_peered":"2024-09-13T14:53:13.863777+0000","last_unstale":"2024-09-13T14:56:08.763219+0000","last_undegraded":"2024-09-13T14:56:08.763219+0000","last_fullsized":"2024-09-13T14:56:08.763219+0000","mapping_epoch":270,"log_start":"276'4864","ondisk_log_start":"276'4864","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7226,"log_dups_size":0,"ondisk_log_size":7226,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:33.743908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237109248,"num_objects":3618,"num_object_clones":0,"num_object_copies":10854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12090,"num_write_kb":502656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":242,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.308535+0000","last_change":"2024-09-13T14:55:58.308418+0000","last_active":"2024-09-13T14:55:58.308535+0000","last_peered":"2024-09-13T14:55:58.308535+0000","last_clean":"2024-09-13T14:55:58.308535+0000","last_became_active":"2024-09-13T14:55:51.160771+0000","last_became_peered":"2024-09-13T14:55:51.160771+0000","last_unstale":"2024-09-13T14:55:58.308535+0000","last_undegraded":"2024-09-13T14:55:58.308535+0000","last_fullsized":"2024-09-13T14:55:58.308535+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.308240+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.308240+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:05.665088+0000","scrub_duration":0.0063105979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":450,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.234751+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:55:58.234751+0000","last_peered":"2024-09-13T14:55:58.234751+0000","last_clean":"2024-09-13T14:55:58.234751+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:55:58.234751+0000","last_undegraded":"2024-09-13T14:55:58.234751+0000","last_fullsized":"2024-09-13T14:55:58.234751+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":806,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.237894+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T14:55:58.237894+0000","last_peered":"2024-09-13T14:55:58.237894+0000","last_clean":"2024-09-13T14:55:58.237894+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T14:55:58.237894+0000","last_undegraded":"2024-09-13T14:55:58.237894+0000","last_fullsized":"2024-09-13T14:55:58.237894+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":113,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.235455+0000","last_change":"2024-09-13T14:51:44.203186+0000","last_active":"2024-09-13T14:55:58.235455+0000","last_peered":"2024-09-13T14:55:58.235455+0000","last_clean":"2024-09-13T14:55:58.235455+0000","last_became_active":"2024-09-13T14:51:44.202851+0000","last_became_peered":"2024-09-13T14:51:44.202851+0000","last_unstale":"2024-09-13T14:55:58.235455+0000","last_undegraded":"2024-09-13T14:55:58.235455+0000","last_fullsized":"2024-09-13T14:55:58.235455+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:05:49.173340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"291'11755","reported_seq":11785,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:07.108418+0000","last_change":"2024-09-13T14:53:13.864419+0000","last_active":"2024-09-13T14:56:07.108418+0000","last_peered":"2024-09-13T14:56:07.108418+0000","last_clean":"2024-09-13T14:56:07.108418+0000","last_became_active":"2024-09-13T14:53:13.864057+0000","last_became_peered":"2024-09-13T14:53:13.864057+0000","last_unstale":"2024-09-13T14:56:07.108418+0000","last_undegraded":"2024-09-13T14:56:07.108418+0000","last_fullsized":"2024-09-13T14:56:07.108418+0000","mapping_epoch":270,"log_start":"276'4050","ondisk_log_start":"276'4050","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:12.550479+0000","last_clean_scrub_stamp":"2024-09-13T14:53:12.550479+0000","objects_scrubbed":0,"log_size":7705,"log_dups_size":0,"ondisk_log_size":7705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:24.873156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239075352,"num_objects":3649,"num_object_clones":0,"num_object_copies":10947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3649,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11755,"num_write_kb":492865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":403,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.914869+0000","last_change":"2024-09-13T14:55:59.914869+0000","last_active":"2024-09-13T14:55:59.914869+0000","last_peered":"2024-09-13T14:55:59.914869+0000","last_clean":"2024-09-13T14:55:59.914869+0000","last_became_active":"2024-09-13T14:55:51.162703+0000","last_became_peered":"2024-09-13T14:55:51.162703+0000","last_unstale":"2024-09-13T14:55:59.914869+0000","last_undegraded":"2024-09-13T14:55:59.914869+0000","last_fullsized":"2024-09-13T14:55:59.914869+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.914801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.914801+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:28:01.134521+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":458,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.226584+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:55:58.226584+0000","last_peered":"2024-09-13T14:55:58.226584+0000","last_clean":"2024-09-13T14:55:58.226584+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:55:58.226584+0000","last_undegraded":"2024-09-13T14:55:58.226584+0000","last_fullsized":"2024-09-13T14:55:58.226584+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1104,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.182748+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T14:55:55.182748+0000","last_peered":"2024-09-13T14:55:55.182748+0000","last_clean":"2024-09-13T14:55:55.182748+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T14:55:55.182748+0000","last_undegraded":"2024-09-13T14:55:55.182748+0000","last_fullsized":"2024-09-13T14:55:55.182748+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":258,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.176460+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T14:55:55.176460+0000","last_peered":"2024-09-13T14:55:55.176460+0000","last_clean":"2024-09-13T14:55:55.176460+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T14:55:55.176460+0000","last_undegraded":"2024-09-13T14:55:55.176460+0000","last_fullsized":"2024-09-13T14:55:55.176460+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":275,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.177459+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T14:55:55.177459+0000","last_peered":"2024-09-13T14:55:55.177459+0000","last_clean":"2024-09-13T14:55:55.177459+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T14:55:55.177459+0000","last_undegraded":"2024-09-13T14:55:55.177459+0000","last_fullsized":"2024-09-13T14:55:55.177459+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":465,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.225858+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T14:55:58.225858+0000","last_peered":"2024-09-13T14:55:58.225858+0000","last_clean":"2024-09-13T14:55:58.225858+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T14:55:58.225858+0000","last_undegraded":"2024-09-13T14:55:58.225858+0000","last_fullsized":"2024-09-13T14:55:58.225858+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":448,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.237756+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T14:55:58.237756+0000","last_peered":"2024-09-13T14:55:58.237756+0000","last_clean":"2024-09-13T14:55:58.237756+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T14:55:58.237756+0000","last_undegraded":"2024-09-13T14:55:58.237756+0000","last_fullsized":"2024-09-13T14:55:58.237756+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":271,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.180888+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:55:55.180888+0000","last_peered":"2024-09-13T14:55:55.180888+0000","last_clean":"2024-09-13T14:55:55.180888+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:55:55.180888+0000","last_undegraded":"2024-09-13T14:55:55.180888+0000","last_fullsized":"2024-09-13T14:55:55.180888+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":25,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:56:01.235655+0000","last_change":"2024-09-13T14:56:01.235655+0000","last_active":"2024-09-13T14:56:01.235655+0000","last_peered":"2024-09-13T14:56:01.235655+0000","last_clean":"2024-09-13T14:56:01.235655+0000","last_became_active":"2024-09-13T14:55:55.195703+0000","last_became_peered":"2024-09-13T14:55:55.195703+0000","last_unstale":"2024-09-13T14:56:01.235655+0000","last_undegraded":"2024-09-13T14:56:01.235655+0000","last_fullsized":"2024-09-13T14:56:01.235655+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.235497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.235497+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:30:50.632750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":255,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.181914+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:55:55.181914+0000","last_peered":"2024-09-13T14:55:55.181914+0000","last_clean":"2024-09-13T14:55:55.181914+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:55:55.181914+0000","last_undegraded":"2024-09-13T14:55:55.181914+0000","last_fullsized":"2024-09-13T14:55:55.181914+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":417,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.234249+0000","last_change":"2024-09-13T14:48:32.200953+0000","last_active":"2024-09-13T14:55:58.234249+0000","last_peered":"2024-09-13T14:55:58.234249+0000","last_clean":"2024-09-13T14:55:58.234249+0000","last_became_active":"2024-09-13T14:48:32.200634+0000","last_became_peered":"2024-09-13T14:48:32.200634+0000","last_unstale":"2024-09-13T14:55:58.234249+0000","last_undegraded":"2024-09-13T14:55:58.234249+0000","last_fullsized":"2024-09-13T14:55:58.234249+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:18.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":445,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.226637+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:55:58.226637+0000","last_peered":"2024-09-13T14:55:58.226637+0000","last_clean":"2024-09-13T14:55:58.226637+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:55:58.226637+0000","last_undegraded":"2024-09-13T14:55:58.226637+0000","last_fullsized":"2024-09-13T14:55:58.226637+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":466,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.235804+0000","last_change":"2024-09-13T14:48:32.201812+0000","last_active":"2024-09-13T14:55:58.235804+0000","last_peered":"2024-09-13T14:55:58.235804+0000","last_clean":"2024-09-13T14:55:58.235804+0000","last_became_active":"2024-09-13T14:48:32.201486+0000","last_became_peered":"2024-09-13T14:48:32.201486+0000","last_unstale":"2024-09-13T14:55:58.235804+0000","last_undegraded":"2024-09-13T14:55:58.235804+0000","last_fullsized":"2024-09-13T14:55:58.235804+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:23:29.693906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":75,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.182665+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T14:55:55.182665+0000","last_peered":"2024-09-13T14:55:55.182665+0000","last_clean":"2024-09-13T14:55:55.182665+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T14:55:55.182665+0000","last_undegraded":"2024-09-13T14:55:55.182665+0000","last_fullsized":"2024-09-13T14:55:55.182665+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":470,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.235682+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:55:58.235682+0000","last_peered":"2024-09-13T14:55:58.235682+0000","last_clean":"2024-09-13T14:55:58.235682+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:55:58.235682+0000","last_undegraded":"2024-09-13T14:55:58.235682+0000","last_fullsized":"2024-09-13T14:55:58.235682+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":469,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.226756+0000","last_change":"2024-09-13T14:51:44.198771+0000","last_active":"2024-09-13T14:55:58.226756+0000","last_peered":"2024-09-13T14:55:58.226756+0000","last_clean":"2024-09-13T14:55:58.226756+0000","last_became_active":"2024-09-13T14:51:44.198466+0000","last_became_peered":"2024-09-13T14:51:44.198466+0000","last_unstale":"2024-09-13T14:55:58.226756+0000","last_undegraded":"2024-09-13T14:55:58.226756+0000","last_fullsized":"2024-09-13T14:55:58.226756+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:41:16.099954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":433,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.237120+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:55:58.237120+0000","last_peered":"2024-09-13T14:55:58.237120+0000","last_clean":"2024-09-13T14:55:58.237120+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:55:58.237120+0000","last_undegraded":"2024-09-13T14:55:58.237120+0000","last_fullsized":"2024-09-13T14:55:58.237120+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":466,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.234397+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:55:58.234397+0000","last_peered":"2024-09-13T14:55:58.234397+0000","last_clean":"2024-09-13T14:55:58.234397+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:55:58.234397+0000","last_undegraded":"2024-09-13T14:55:58.234397+0000","last_fullsized":"2024-09-13T14:55:58.234397+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":274,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.178944+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:55:55.178944+0000","last_peered":"2024-09-13T14:55:55.178944+0000","last_clean":"2024-09-13T14:55:55.178944+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:55:55.178944+0000","last_undegraded":"2024-09-13T14:55:55.178944+0000","last_fullsized":"2024-09-13T14:55:55.178944+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":110,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.240808+0000","last_change":"2024-09-13T14:51:02.029301+0000","last_active":"2024-09-13T14:55:58.240808+0000","last_peered":"2024-09-13T14:55:58.240808+0000","last_clean":"2024-09-13T14:55:58.240808+0000","last_became_active":"2024-09-13T14:51:02.029032+0000","last_became_peered":"2024-09-13T14:51:02.029032+0000","last_unstale":"2024-09-13T14:55:58.240808+0000","last_undegraded":"2024-09-13T14:55:58.240808+0000","last_fullsized":"2024-09-13T14:55:58.240808+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:40.150320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":438,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.226086+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:55:58.226086+0000","last_peered":"2024-09-13T14:55:58.226086+0000","last_clean":"2024-09-13T14:55:58.226086+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:55:58.226086+0000","last_undegraded":"2024-09-13T14:55:58.226086+0000","last_fullsized":"2024-09-13T14:55:58.226086+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":272,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.177081+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:55:55.177081+0000","last_peered":"2024-09-13T14:55:55.177081+0000","last_clean":"2024-09-13T14:55:55.177081+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:55:55.177081+0000","last_undegraded":"2024-09-13T14:55:55.177081+0000","last_fullsized":"2024-09-13T14:55:55.177081+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":257,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.183921+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:55:55.183921+0000","last_peered":"2024-09-13T14:55:55.183921+0000","last_clean":"2024-09-13T14:55:55.183921+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:55:55.183921+0000","last_undegraded":"2024-09-13T14:55:55.183921+0000","last_fullsized":"2024-09-13T14:55:55.183921+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":272,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.178855+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:55:55.178855+0000","last_peered":"2024-09-13T14:55:55.178855+0000","last_clean":"2024-09-13T14:55:55.178855+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:55:55.178855+0000","last_undegraded":"2024-09-13T14:55:55.178855+0000","last_fullsized":"2024-09-13T14:55:55.178855+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":296,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.182297+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T14:55:55.182297+0000","last_peered":"2024-09-13T14:55:55.182297+0000","last_clean":"2024-09-13T14:55:55.182297+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T14:55:55.182297+0000","last_undegraded":"2024-09-13T14:55:55.182297+0000","last_fullsized":"2024-09-13T14:55:55.182297+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":246,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.184525+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:55:55.184525+0000","last_peered":"2024-09-13T14:55:55.184525+0000","last_clean":"2024-09-13T14:55:55.184525+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:55:55.184525+0000","last_undegraded":"2024-09-13T14:55:55.184525+0000","last_fullsized":"2024-09-13T14:55:55.184525+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":473,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.234300+0000","last_change":"2024-09-13T14:48:32.202404+0000","last_active":"2024-09-13T14:55:58.234300+0000","last_peered":"2024-09-13T14:55:58.234300+0000","last_clean":"2024-09-13T14:55:58.234300+0000","last_became_active":"2024-09-13T14:48:32.202064+0000","last_became_peered":"2024-09-13T14:48:32.202064+0000","last_unstale":"2024-09-13T14:55:58.234300+0000","last_undegraded":"2024-09-13T14:55:58.234300+0000","last_fullsized":"2024-09-13T14:55:58.234300+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:25.674819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":278,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.176350+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T14:55:55.176350+0000","last_peered":"2024-09-13T14:55:55.176350+0000","last_clean":"2024-09-13T14:55:55.176350+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T14:55:55.176350+0000","last_undegraded":"2024-09-13T14:55:55.176350+0000","last_fullsized":"2024-09-13T14:55:55.176350+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":489,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:58.226541+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T14:55:58.226541+0000","last_peered":"2024-09-13T14:55:58.226541+0000","last_clean":"2024-09-13T14:55:58.226541+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T14:55:58.226541+0000","last_undegraded":"2024-09-13T14:55:58.226541+0000","last_fullsized":"2024-09-13T14:55:58.226541+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3857907736,"num_objects":58868,"num_object_clones":0,"num_object_copies":176604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58868,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":191918,"num_write_kb":8025089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11603030016,"data_stored":11603017800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116457,"ondisk_log_size":116457,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":16172,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2100,"ondisk_log_size":2100,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397866,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1807920,"kb_used_data":1205544,"kb_used_omap":1,"kb_used_meta":602366,"kb_avail":91937232,"statfs":{"total":95995035648,"available":94143725568,"internally_reserved":0,"allocated":1234477056,"data_stored":1232486449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":616823128},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594095,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1540644,"kb_used_data":1230812,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92204508,"statfs":{"total":95995035648,"available":94417416192,"internally_reserved":0,"allocated":1260351488,"data_stored":1258372717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790324,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2818148,"kb_used_data":1929436,"kb_used_omap":1,"kb_used_meta":888702,"kb_avail":90927004,"statfs":{"total":95995035648,"available":93109252096,"internally_reserved":0,"allocated":1975742464,"data_stored":1973760158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910031192},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986553,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2124384,"kb_used_data":1223000,"kb_used_omap":1,"kb_used_meta":901374,"kb_avail":91620768,"statfs":{"total":95995035648,"available":93819666432,"internally_reserved":0,"allocated":1252352000,"data_stored":1250372953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923007320},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676599001,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2072276,"kb_used_data":1416524,"kb_used_omap":6,"kb_used_meta":655609,"kb_avail":91672876,"statfs":{"total":95995035648,"available":93873025024,"internally_reserved":0,"allocated":1450520576,"data_stored":1448553999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":671344608},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762528,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2160164,"kb_used_data":1411420,"kb_used_omap":8,"kb_used_meta":748727,"kb_avail":91584988,"statfs":{"total":95995035648,"available":93783027712,"internally_reserved":0,"allocated":1445294080,"data_stored":1443311769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":766696906},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926054,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2294616,"kb_used_data":1651088,"kb_used_omap":7,"kb_used_meta":643384,"kb_avail":91450536,"statfs":{"total":95995035648,"available":93645348864,"internally_reserved":0,"allocated":1690714112,"data_stored":1688743158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":658825929},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5,2,8,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122294,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2029100,"kb_used_data":1426276,"kb_used_omap":7,"kb_used_meta":602744,"kb_avail":91716052,"statfs":{"total":95995035648,"available":93917237248,"internally_reserved":0,"allocated":1460506624,"data_stored":1458520597,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":617210447},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1436418048,"data_stored":1436418048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1677787136,"data_stored":1677787136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1426132992,"data_stored":1426128920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1426128896,"data_stored":1426128896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1231290368,"data_stored":1231290368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1947406336,"data_stored":1947402264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1233649664,"data_stored":1233649664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1224216576,"data_stored":1224212504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:56:12.406 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 21 2024-09-13T14:56:12.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.442 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:12.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:12.544 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:12.557 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:12 smithi096 ceph-mon[102928]: from='client.55450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:12.561 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:12 smithi096 ceph-mon[98901]: from='client.55450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:12.565 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:12.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:12.667 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:12.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.696 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:12.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:12.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:12 smithi204 ceph-mon[87071]: from='client.55450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:12.797 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:12.843 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:12.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:12.945 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:12.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.988 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:12.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:13.089 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:56:13.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.125 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:13.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:13.226 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:13.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.266 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:13.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:13.367 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:13.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.399 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:13.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:13.500 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:13.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.540 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:13.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:13.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:13 smithi096 ceph-mon[98901]: pgmap v920: 109 pgs: 109 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2024-09-13T14:56:13.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:13 smithi096 ceph-mon[102928]: pgmap v920: 109 pgs: 109 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2024-09-13T14:56:13.641 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:13.683 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:13.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:13.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:13 smithi204 ceph-mon[87071]: pgmap v920: 109 pgs: 109 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2024-09-13T14:56:13.788 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:13.826 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:13.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:13.928 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:13.960 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:13.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:14.062 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:14.095 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:14.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:14.196 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:14.229 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:14.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:14.330 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:14.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.363 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:14.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:14.465 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:14.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.503 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:14.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:14.604 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:14.632 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:14.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:14.733 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:14.761 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:14.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:14.862 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:14.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.900 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:14.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:15.001 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:15.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.038 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:15.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:15.139 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:15.176 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:15.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:15.278 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:15.298 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:15 smithi096 ceph-mon[98901]: pgmap v921: 109 pgs: 109 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-09-13T14:56:15.309 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:15.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:15.410 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:15.444 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:15.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:15.546 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:15.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:15 smithi096 ceph-mon[102928]: pgmap v921: 109 pgs: 109 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-09-13T14:56:15.573 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:15.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:15.675 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:15.712 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:15.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:15.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:15 smithi204 ceph-mon[87071]: pgmap v921: 109 pgs: 109 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-09-13T14:56:15.814 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:15.850 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:15.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:15.952 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:15.984 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:15.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:16.086 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:16.121 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:16.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:16.223 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:16.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.269 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:16.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:16.370 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:16.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.408 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:16.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:16.509 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:16.537 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:16.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:16.639 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:16.668 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:16.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:16.770 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:16.801 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:16.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:16.903 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:16.934 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:16.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:17.036 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:17.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:16 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:17.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:16 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:17.049 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:16 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:56:17.049 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:16 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:56:17.049 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:16 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:17.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:16 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:17.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:16 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:17.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:16 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:56:17.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:16 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:56:17.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:16 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:17.076 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:17.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:17.121 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:56:17.178 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:17.202 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:17.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:17.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:16 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:17.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:16 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:17.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:16 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:56:17.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:16 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:56:17.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:16 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:17.304 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:17.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.353 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:17.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:17.455 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:17.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.498 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:17.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:17.599 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:17.622 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:17.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:17.724 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:17.746 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:17.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:17.848 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:17.888 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:17.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:17.990 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:18.012 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:18.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:18.048 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:17 smithi096 ceph-mon[98901]: pgmap v922: 109 pgs: 109 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-09-13T14:56:18.048 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:56:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:56:17] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.51.0" 2024-09-13T14:56:18.048 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:17 smithi096 ceph-mon[102928]: pgmap v922: 109 pgs: 109 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-09-13T14:56:18.114 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:18.120 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:18.138 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:18.139 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:56:18.139 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:56:18.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:18.175 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:56:18.175 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:56:18.207 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:56:18.207 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:56:18.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:17 smithi204 ceph-mon[87071]: pgmap v922: 109 pgs: 109 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-09-13T14:56:18.238 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:56:18.238 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:56:18.240 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:18.265 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:18.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:18.270 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:56:18.270 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:56:18.302 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:56:18.303 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:56:18.337 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:56:18.337 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:56:18.367 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:18.376 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:56:18.376 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:56:18.435 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:18.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:18.537 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:18.567 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:18.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:18.668 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:18.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.706 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:18.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:18.807 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:18.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.843 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:18.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:18.944 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:18.973 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:18.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:19.053 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:18 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2828203704' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:19.053 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:18 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2828203704' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:19.075 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:19.111 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:19.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:19.213 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:19.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:18 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2828203704' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:19.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.262 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:19.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:19.363 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:19.417 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:19.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:19.519 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:19.542 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:19.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:19.644 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:19.673 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:19.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:19.775 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:19.808 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:19.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:19.910 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:19.941 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:19.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:20.035 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:20.043 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:20.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:19 smithi096 ceph-mon[98901]: pgmap v923: 109 pgs: 109 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 964 op/s 2024-09-13T14:56:20.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:19 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2914148119' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:20.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:19 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:20.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:19 smithi096 ceph-mon[102928]: pgmap v923: 109 pgs: 109 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 964 op/s 2024-09-13T14:56:20.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:19 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2914148119' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:20.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:19 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:20.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.075 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:20.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:20.119 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:19 smithi204 ceph-mon[87071]: pgmap v923: 109 pgs: 109 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 964 op/s 2024-09-13T14:56:20.120 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:19 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2914148119' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:20.120 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:19 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:20.175 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:20.205 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:20.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:20.259 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:56:20.307 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:20.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.337 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:20.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:20.438 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:56:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:56:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:20.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.477 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:20.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:20.579 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:20.612 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:20.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:20.714 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:20.750 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:20.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:20.852 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:20.883 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:20.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:20.984 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:21.018 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:21.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:20 smithi096 ceph-mon[98901]: pgmap v924: 109 pgs: 109 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-13T14:56:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:20 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:56:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:20 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:56:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:20 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:20 smithi096 ceph-mon[98901]: osdmap e292: 8 total, 8 up, 8 in 2024-09-13T14:56:21.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:20 smithi096 ceph-mon[102928]: pgmap v924: 109 pgs: 109 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-13T14:56:21.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:20 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:56:21.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:20 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:56:21.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:20 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:21.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:20 smithi096 ceph-mon[102928]: osdmap e292: 8 total, 8 up, 8 in 2024-09-13T14:56:21.097 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:56:21.120 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:21.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.158 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:21.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:20 smithi204 ceph-mon[87071]: pgmap v924: 109 pgs: 109 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-13T14:56:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:20 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:56:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:20 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:56:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:20 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:20 smithi204 ceph-mon[87071]: osdmap e292: 8 total, 8 up, 8 in 2024-09-13T14:56:21.259 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:21.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.285 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:21.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:21.386 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:21.410 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:21.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:21.512 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:21.535 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:21.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:21.636 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:21.672 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:21.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:21.774 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:21.806 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:21.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:21.908 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:21.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.944 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:21.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:22.045 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:22.081 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:22.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:22.183 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:22.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.215 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:22.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:22.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:22.250 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:22.250 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:22.250 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:22.250 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:22.316 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:22.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.349 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:22.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:22.450 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:22.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.484 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:22.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:22.585 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:22.622 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:22.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:22.724 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:22.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.778 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:22.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:22.879 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:22.922 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:22.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:23.024 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:23.049 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:22 smithi096 ceph-mon[98901]: pgmap v926: 109 pgs: 109 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 954 op/s 2024-09-13T14:56:23.049 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:22 smithi096 ceph-mon[102928]: pgmap v926: 109 pgs: 109 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 954 op/s 2024-09-13T14:56:23.056 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:23.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:23.158 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:23.186 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:23.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:23.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:22 smithi204 ceph-mon[87071]: pgmap v926: 109 pgs: 109 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 954 op/s 2024-09-13T14:56:23.288 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:23.316 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:23.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:23.418 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:23.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.448 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:23.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:23.549 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:23.583 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:23.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:23.684 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:23.724 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:23.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:23.826 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:23.855 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:23.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:23.956 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:23.994 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:23.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:24.099 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:24.132 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:24.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:24.233 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:24.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.276 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:24.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:24.377 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:24.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.434 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:24.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:24.520 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:24.535 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:24.542 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:56:24.542 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:56:24.542 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:56:24.542 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:56:24.542 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:56:24.560 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:24.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:24.662 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:24.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.707 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:24.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:24.808 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:24.843 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:24.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:24.945 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:24.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.980 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:24.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:25.081 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:25.114 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:25.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:25.217 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:25.247 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:25.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:25.261 INFO:teuthology.orchestra.run.smithi096.stderr:set pool 1 pgp_num to 21 2024-09-13T14:56:25.350 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:25.353 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:56:25.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.429 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:25.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:25.530 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:25.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.567 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:25.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:25.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:25 smithi096 ceph-mon[98901]: pgmap v927: 109 pgs: 109 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 956 op/s 2024-09-13T14:56:25.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:25 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/252385451' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"} : dispatch 2024-09-13T14:56:25.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:25 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"} : dispatch 2024-09-13T14:56:25.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:25 smithi096 ceph-mon[102928]: pgmap v927: 109 pgs: 109 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 956 op/s 2024-09-13T14:56:25.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:25 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/252385451' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"} : dispatch 2024-09-13T14:56:25.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:25 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"} : dispatch 2024-09-13T14:56:25.668 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:25.715 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:25.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:25.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:25 smithi204 ceph-mon[87071]: pgmap v927: 109 pgs: 109 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 956 op/s 2024-09-13T14:56:25.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:25 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/252385451' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"} : dispatch 2024-09-13T14:56:25.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:25 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"} : dispatch 2024-09-13T14:56:25.805 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:56:25.816 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:25.883 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:25.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:25.985 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:26.024 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:26.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:26.125 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:26.187 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:26.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:26.289 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:26.333 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:26.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:26.396 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:26 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2024-09-13T14:56:26.397 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:26 smithi096 ceph-mon[98901]: osdmap e293: 8 total, 8 up, 8 in 2024-09-13T14:56:26.397 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:26 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2024-09-13T14:56:26.397 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:26 smithi096 ceph-mon[102928]: osdmap e293: 8 total, 8 up, 8 in 2024-09-13T14:56:26.436 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:26.468 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:26.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:26.570 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:26.605 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:26.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:26.707 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:26.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:26 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2024-09-13T14:56:26.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:26 smithi204 ceph-mon[87071]: osdmap e293: 8 total, 8 up, 8 in 2024-09-13T14:56:26.740 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:26.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:26.842 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:26.876 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:26.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:26.977 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:27.018 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:27.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:27.120 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:27.152 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:27.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:27.254 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:27.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.293 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:27.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:27.394 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:27.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.422 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:27.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:27.523 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:27.552 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:27.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:27.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:27 smithi096 ceph-mon[98901]: pgmap v929: 109 pgs: 109 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2024-09-13T14:56:27.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:27 smithi096 ceph-mon[102928]: pgmap v929: 109 pgs: 109 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2024-09-13T14:56:27.654 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:27.685 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:27.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:27.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:27 smithi204 ceph-mon[87071]: pgmap v929: 109 pgs: 109 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2024-09-13T14:56:27.787 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:27.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.826 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:27.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:27.927 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:27.950 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:27.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:28.052 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:28.088 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:28.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:28.190 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:28.216 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:28.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:28.318 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:28.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:56:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:56:27] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.51.0" 2024-09-13T14:56:28.349 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:28.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:28.451 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:28.485 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:28.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:28.587 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:28.624 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:28.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:28.726 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:28.759 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:28.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:28.862 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:28.896 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:28.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:28.998 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:29.033 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:29.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:29.135 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:29.168 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:29.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:29.270 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:29.383 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:29.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:29.485 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:29.514 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:29.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:29.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:29 smithi096 ceph-mon[98901]: pgmap v930: 109 pgs: 109 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 826 op/s 2024-09-13T14:56:29.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:29 smithi096 ceph-mon[102928]: pgmap v930: 109 pgs: 109 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 826 op/s 2024-09-13T14:56:29.616 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:29.652 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:29.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:29.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:29 smithi204 ceph-mon[87071]: pgmap v930: 109 pgs: 109 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 826 op/s 2024-09-13T14:56:29.754 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:29.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.787 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:29.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:29.888 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:29.926 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:29.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:30.028 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:30.062 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:30.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:30.164 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:30.189 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:30.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:30.291 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:30.324 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:30.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:30.426 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:30.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.448 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:30.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:56:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:56:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:30.549 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:30.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.587 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:30.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:30.688 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:30.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.724 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:30.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:30.825 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:30.859 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:30.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:30.961 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:30.985 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:30.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:31.087 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:31.116 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:31.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:31.147 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:56:31.147 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:56:31.147 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-13T14:56:31.148 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.270487 2024-09-13T14:56:31.148 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 2 0.27048667684154104 2024-09-13T14:56:31.218 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:31.258 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:31.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:31.361 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:31.386 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:31.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:31.488 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:31.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.521 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:31.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:31.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:31 smithi096 ceph-mon[102928]: pgmap v931: 109 pgs: 109 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 986 op/s 2024-09-13T14:56:31.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:31 smithi096 ceph-mon[98901]: pgmap v931: 109 pgs: 109 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 986 op/s 2024-09-13T14:56:31.622 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:31.670 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:31.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:31.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:31 smithi204 ceph-mon[87071]: pgmap v931: 109 pgs: 109 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 986 op/s 2024-09-13T14:56:31.772 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:31.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.802 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:31.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:31.893 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:31.903 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:31.907 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:31.931 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:31.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:32.033 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:32.080 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:32.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:32.182 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:32.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.218 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:32.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:32.319 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:32.352 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:32.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:32.455 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:32.494 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:32.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:32.598 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:32.626 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:32.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:32.728 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:32.760 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:32.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:32.862 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:32.887 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:32.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:32.960 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:56:32.961 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:56:32.989 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:33.002 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:56:33.002 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:56:33.032 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:33.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:33.036 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:56:33.036 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:56:33.084 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:56:33.085 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:56:33.117 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:56:33.117 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:56:33.134 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:33.154 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:56:33.154 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:56:33.161 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:33.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:33.192 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:56:33.192 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:56:33.222 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:56:33.222 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:56:33.263 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:33.298 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:33.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:33.400 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:33.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.435 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:33.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:33.505 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:33 smithi096 ceph-mon[98901]: pgmap v932: 109 pgs: 109 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 969 op/s 2024-09-13T14:56:33.505 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:33 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/987179882' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:33.506 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:33 smithi096 ceph-mon[102928]: pgmap v932: 109 pgs: 109 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 969 op/s 2024-09-13T14:56:33.506 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:33 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/987179882' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:33.536 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:33.565 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:33.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:33.667 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:33.699 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:33.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:33.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:33 smithi204 ceph-mon[87071]: pgmap v932: 109 pgs: 109 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 969 op/s 2024-09-13T14:56:33.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:33 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/987179882' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:33.801 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:33.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.829 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:33.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:33.930 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:33.968 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:33.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:34.070 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:34.110 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:34.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:34.212 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:34.238 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:34.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:34.340 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:34.374 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:34.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:34.476 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:34.513 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:34.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:34.614 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:34.641 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:34.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:34.743 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:34.780 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:34.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:34.885 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:34.909 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:34.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:34.970 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:35.010 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:35.037 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:35.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:35.139 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:35.170 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:35.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:35.272 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:35.304 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:35.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:35.406 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:35.436 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:35.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:35.457 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:35 smithi096 ceph-mon[102928]: pgmap v933: 109 pgs: 109 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 960 op/s 2024-09-13T14:56:35.459 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:35 smithi096 ceph-mon[98901]: pgmap v933: 109 pgs: 109 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 960 op/s 2024-09-13T14:56:35.538 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:35.564 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:35.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:35.666 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:35.704 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:35.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:35.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:35 smithi204 ceph-mon[87071]: pgmap v933: 109 pgs: 109 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 960 op/s 2024-09-13T14:56:35.806 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:35.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.846 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:35.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:35.947 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:35.980 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:35.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:36.082 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:36.115 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:36.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:36.217 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:36.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.256 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:36.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:36.347 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:56:36.358 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:36.445 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:36.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:36.547 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:36.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:36.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:36.577 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:36.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:36.678 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:36.717 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:36.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:36.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:36.820 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:36.847 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:36.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:36.949 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:36.984 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:36.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:37.086 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:37.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.119 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:37.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:37.221 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:37.246 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:37.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:37.348 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:37.383 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:37.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:37.485 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:37.515 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:37.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:37.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:37 smithi096 ceph-mon[98901]: pgmap v934: 109 pgs: 109 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-09-13T14:56:37.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:37 smithi096 ceph-mon[102928]: pgmap v934: 109 pgs: 109 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-09-13T14:56:37.617 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:37.652 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:37.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:37.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:37 smithi204 ceph-mon[87071]: pgmap v934: 109 pgs: 109 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-09-13T14:56:37.754 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:37.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.788 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:37.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:37.889 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:37.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.929 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:37.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:38.030 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:38.062 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:38.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:38.164 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:38.198 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:38.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:38.299 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:38.300 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:56:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:56:37] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.51.0" 2024-09-13T14:56:38.334 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:38.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:38.436 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:38.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.473 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:38.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:38.574 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:38.611 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:38.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:38.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:38.758 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:38.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:38.860 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:38.894 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:38.996 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:39.041 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:39.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:39.143 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:39.171 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:39.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:39.272 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:39.305 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:39.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:39.407 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:39.457 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:39.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:39.458 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:39 smithi096 ceph-mon[98901]: pgmap v935: 109 pgs: 109 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 949 op/s 2024-09-13T14:56:39.459 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:39 smithi096 ceph-mon[102928]: pgmap v935: 109 pgs: 109 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 949 op/s 2024-09-13T14:56:39.559 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:39.585 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:39.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:39.687 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:39.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:39 smithi204 ceph-mon[87071]: pgmap v935: 109 pgs: 109 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 949 op/s 2024-09-13T14:56:39.733 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:39.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:39.835 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:39.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.866 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:39.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:39.967 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:39.984 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:56:39.984 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:56:39.997 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:39.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:40.099 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:40.128 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:40.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:40.230 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:40.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.269 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:40.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:40.332 INFO:teuthology.orchestra.run.smithi096.stderr:set osd.2 primary-affinity to 0.270487 (8177262) 2024-09-13T14:56:40.370 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:40.394 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:40.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:40 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/778699672' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.27048667684154104} : dispatch 2024-09-13T14:56:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:40 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.27048667684154104} : dispatch 2024-09-13T14:56:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:56:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:56:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:40.496 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:40.526 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:40.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:40.627 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:40.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.659 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:40.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:40.760 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:40.791 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:40.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:40.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:40 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/778699672' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.27048667684154104} : dispatch 2024-09-13T14:56:40.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:40 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.27048667684154104} : dispatch 2024-09-13T14:56:40.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:40 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/778699672' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.27048667684154104} : dispatch 2024-09-13T14:56:40.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:40 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.27048667684154104} : dispatch 2024-09-13T14:56:40.892 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:40.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.922 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:40.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:41.023 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:41.051 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:41.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:41.097 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:56:41.153 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:41.201 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:41.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:41.304 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:41.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.346 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:41.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:41.359 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:41 smithi096 ceph-mon[98901]: pgmap v936: 109 pgs: 109 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:56:41.359 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:41 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.27048667684154104}]': finished 2024-09-13T14:56:41.360 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:41 smithi096 ceph-mon[98901]: osdmap e294: 8 total, 8 up, 8 in 2024-09-13T14:56:41.360 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:41 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3821657575' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:56:41.360 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:41 smithi096 ceph-mon[102928]: pgmap v936: 109 pgs: 109 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:56:41.360 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:41 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.27048667684154104}]': finished 2024-09-13T14:56:41.360 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:41 smithi096 ceph-mon[102928]: osdmap e294: 8 total, 8 up, 8 in 2024-09-13T14:56:41.360 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:41 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3821657575' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:56:41.360 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:56:41.447 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:41.492 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:41.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:41.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:41.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.620 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:41.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:41.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:41 smithi204 ceph-mon[87071]: pgmap v936: 109 pgs: 109 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:56:41.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:41 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.27048667684154104}]': finished 2024-09-13T14:56:41.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:41 smithi204 ceph-mon[87071]: osdmap e294: 8 total, 8 up, 8 in 2024-09-13T14:56:41.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:41 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3821657575' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:56:41.721 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:41.763 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:41.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:41.865 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:41.896 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:41.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:41.998 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:42.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.028 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:42.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:42.129 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:42.149 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:42.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.189 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:42.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:42.290 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:42.317 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:42.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:42.419 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:42.450 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:42.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:42.552 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:42.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:42 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:42.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:42 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3821657575' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:56:42.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:42 smithi096 ceph-mon[98901]: osdmap e295: 8 total, 8 up, 8 in 2024-09-13T14:56:42.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:42 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:42.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:42 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3821657575' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:56:42.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:42 smithi096 ceph-mon[102928]: osdmap e295: 8 total, 8 up, 8 in 2024-09-13T14:56:42.579 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:42.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:42.682 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:42.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:42 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:42.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:42 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3821657575' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:56:42.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:42 smithi204 ceph-mon[87071]: osdmap e295: 8 total, 8 up, 8 in 2024-09-13T14:56:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.721 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:42.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:42.822 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:42.848 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:42.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:42.950 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:42.981 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:42.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:43.082 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:43.108 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:43.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:43.210 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:43.237 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:43.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:43.339 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:43.369 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:43.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:43.471 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:43.502 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:43.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:43.603 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:43.639 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:43 smithi096 ceph-mon[98901]: pgmap v939: 109 pgs: 109 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2024-09-13T14:56:43.640 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:43 smithi096 ceph-mon[102928]: pgmap v939: 109 pgs: 109 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2024-09-13T14:56:43.646 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:43.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:43.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:43 smithi204 ceph-mon[87071]: pgmap v939: 109 pgs: 109 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2024-09-13T14:56:43.748 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:43.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.783 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:43.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:43.884 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:43.913 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:43.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:43.987 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:44.014 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:44.064 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:44.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:44.166 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:44.212 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:44.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:44.314 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:44.349 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:44.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:44.451 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:44.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.482 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:44.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:44.583 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:44.625 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:44.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:44.727 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:44.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.754 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:44.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:44.856 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:44.883 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:44.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:44.985 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:45.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.021 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:45.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:45.122 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:45.145 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:45.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:45.247 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:45.303 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:45.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:45.405 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:45.439 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:45.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:45.541 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:45.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:45 smithi096 ceph-mon[98901]: pgmap v940: 109 pgs: 109 active+clean; 1.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-13T14:56:45.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:45 smithi096 ceph-mon[102928]: pgmap v940: 109 pgs: 109 active+clean; 1.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-13T14:56:45.580 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:45.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:45.681 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:45.684 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:56:45.715 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:45.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:45.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:45 smithi204 ceph-mon[87071]: pgmap v940: 109 pgs: 109 active+clean; 1.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-13T14:56:45.817 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:45.861 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:45.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:45.963 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:46.016 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:46.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:46.118 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:46.154 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:46.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:46.256 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:46.292 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:46.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:46.394 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:46.427 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:46.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:46.529 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:46.568 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:46.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:46.670 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:46.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.703 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:46.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:46.779 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:56:46.780 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:56:46.805 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:46.811 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:56:46.811 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T14:56:46.817 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:56:46.817 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:56:46.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.833 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:46.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:46.853 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:56:46.853 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:56:46.897 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:56:46.897 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:56:46.934 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:46.936 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:56:46.937 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:56:46.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.965 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:46.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:46.970 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:56:46.971 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:56:47.006 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:56:47.006 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:56:47.040 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:56:47.041 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:56:47.067 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:47.128 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:47.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:47.230 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:47.272 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:47.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:47.374 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:47.419 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:47.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:47.521 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:47.559 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:47.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:47.661 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:47.692 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:47.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:47 smithi204 ceph-mon[87071]: pgmap v941: 109 pgs: 109 active+clean; 1.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 992 op/s 2024-09-13T14:56:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:47 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4031696484' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:47.794 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:47.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:47 smithi096 ceph-mon[98901]: pgmap v941: 109 pgs: 109 active+clean; 1.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 992 op/s 2024-09-13T14:56:47.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:47 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4031696484' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:47.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:47 smithi096 ceph-mon[102928]: pgmap v941: 109 pgs: 109 active+clean; 1.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 992 op/s 2024-09-13T14:56:47.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:47 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4031696484' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:47.832 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:47.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:47.934 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:47.972 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:47.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:48.074 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:48.107 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:48.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:48.115 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:56:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:56:47] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.51.0" 2024-09-13T14:56:48.208 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:48.244 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:48.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:48.247 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:56:48.247 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:56:48.247 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.247 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.45749.0:229639 8.f e295/270 8:f9724e2d:::benchmark_data_smithi096_227542_object104244:head v 295'14338, mlcod=295'14335)", 2024-09-13T14:56:48.247 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:47.339508+0000", 2024-09-13T14:56:48.247 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.015002255000000001, 2024-09-13T14:56:48.247 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.015030177, 2024-09-13T14:56:48.248 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:56:48.248 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:56:48.248 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:56:48.248 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:56:48.248 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.248 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:56:48.248 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.339508+0000", 2024-09-13T14:56:48.248 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:56:48.248 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.248 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.248 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:56:48.248 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.339508+0000", 2024-09-13T14:56:48.248 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:56:48.249 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.249 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.249 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:56:48.249 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.339508+0000", 2024-09-13T14:56:48.249 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5400000000000002e-07 2024-09-13T14:56:48.249 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.249 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.249 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:56:48.249 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.339512+0000", 2024-09-13T14:56:48.249 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.8190000000000002e-06 2024-09-13T14:56:48.249 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.249 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.249 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:56:48.250 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.339519+0000", 2024-09-13T14:56:48.250 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7129999999999999e-06 2024-09-13T14:56:48.250 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.250 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.250 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:56:48.250 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.339523+0000", 2024-09-13T14:56:48.250 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3889999999999998e-06 2024-09-13T14:56:48.250 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.250 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.250 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:56:48.250 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.340190+0000", 2024-09-13T14:56:48.250 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00066662700000000004 2024-09-13T14:56:48.251 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.251 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.251 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:56:48.251 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.340207+0000", 2024-09-13T14:56:48.251 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7459999999999999e-05 2024-09-13T14:56:48.251 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:56:48.251 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:56:48.251 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:56:48.251 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.251 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.251 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.45749.0:229658 8.e e295/270 8:7b6cfe8d:::benchmark_data_smithi096_227542_object104263:head v 295'14383, mlcod=295'14381)", 2024-09-13T14:56:48.251 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:47.339566+0000", 2024-09-13T14:56:48.252 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.01494357, 2024-09-13T14:56:48.252 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.015033816, 2024-09-13T14:56:48.252 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:56:48.252 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:56:48.252 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:56:48.252 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:56:48.252 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.252 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:56:48.252 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.339566+0000", 2024-09-13T14:56:48.252 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:56:48.252 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.252 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.252 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:56:48.253 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.339566+0000", 2024-09-13T14:56:48.253 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:56:48.253 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.253 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.253 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:56:48.253 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.339567+0000", 2024-09-13T14:56:48.253 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4799999999999999e-07 2024-09-13T14:56:48.253 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.253 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.253 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:56:48.253 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.339571+0000", 2024-09-13T14:56:48.253 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.121e-06 2024-09-13T14:56:48.253 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.254 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.254 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:56:48.254 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.339578+0000", 2024-09-13T14:56:48.254 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1690000000000003e-06 2024-09-13T14:56:48.254 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.254 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.254 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:56:48.254 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.339583+0000", 2024-09-13T14:56:48.254 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8670000000000002e-06 2024-09-13T14:56:48.254 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.254 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.254 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:56:48.254 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.339667+0000", 2024-09-13T14:56:48.254 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.4011999999999995e-05 2024-09-13T14:56:48.255 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.255 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.255 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:56:48.255 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.339685+0000", 2024-09-13T14:56:48.255 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8630999999999998e-05 2024-09-13T14:56:48.255 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:56:48.255 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:56:48.255 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:56:48.255 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.255 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.255 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.45749.0:229667 8.e e295/270 8:7fc13ec6:::benchmark_data_smithi096_227542_object104272:head v 295'14384, mlcod=295'14381)", 2024-09-13T14:56:48.255 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:47.351534+0000", 2024-09-13T14:56:48.255 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0029761789999999998, 2024-09-13T14:56:48.255 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0031167880000000001, 2024-09-13T14:56:48.256 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:56:48.256 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:56:48.256 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:56:48.256 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:56:48.256 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.256 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:56:48.256 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.351534+0000", 2024-09-13T14:56:48.256 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:56:48.256 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.256 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.256 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:56:48.256 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.351534+0000", 2024-09-13T14:56:48.256 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:56:48.257 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.257 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.257 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:56:48.257 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.351535+0000", 2024-09-13T14:56:48.257 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2529999999999999e-06 2024-09-13T14:56:48.257 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.257 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.257 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:56:48.257 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.351544+0000", 2024-09-13T14:56:48.257 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.4700000000000008e-06 2024-09-13T14:56:48.257 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.258 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.258 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:56:48.258 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.351566+0000", 2024-09-13T14:56:48.258 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.2022999999999999e-05 2024-09-13T14:56:48.258 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.258 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.258 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:56:48.258 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.351575+0000", 2024-09-13T14:56:48.258 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.4829999999999999e-06 2024-09-13T14:56:48.258 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.258 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.258 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:56:48.258 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.351787+0000", 2024-09-13T14:56:48.258 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000211719 2024-09-13T14:56:48.259 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.259 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.259 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:56:48.259 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.351889+0000", 2024-09-13T14:56:48.259 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010206599999999999 2024-09-13T14:56:48.259 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:56:48.259 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:56:48.259 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:56:48.259 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.259 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.259 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.45749.0:229660 8.a e295/270 8:5c52cc01:::benchmark_data_smithi096_227542_object104265:head v 295'14259, mlcod=295'14257)", 2024-09-13T14:56:48.259 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:47.352590+0000", 2024-09-13T14:56:48.260 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0019202170000000001, 2024-09-13T14:56:48.260 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.002112573, 2024-09-13T14:56:48.260 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:56:48.260 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:56:48.260 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:56:48.260 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:56:48.260 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.260 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:56:48.260 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.352590+0000", 2024-09-13T14:56:48.260 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:56:48.260 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.260 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.260 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:56:48.261 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.352590+0000", 2024-09-13T14:56:48.261 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:56:48.261 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.261 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.261 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:56:48.261 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.352590+0000", 2024-09-13T14:56:48.261 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9699999999999995e-07 2024-09-13T14:56:48.261 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.261 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.261 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:56:48.261 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.352597+0000", 2024-09-13T14:56:48.261 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.0190000000000001e-06 2024-09-13T14:56:48.261 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.262 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.262 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:56:48.262 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.352612+0000", 2024-09-13T14:56:48.262 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4338000000000001e-05 2024-09-13T14:56:48.262 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.262 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.262 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:56:48.262 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.352620+0000", 2024-09-13T14:56:48.262 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9459999999999998e-06 2024-09-13T14:56:48.262 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.262 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.262 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:56:48.262 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.352782+0000", 2024-09-13T14:56:48.263 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000162638 2024-09-13T14:56:48.263 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.263 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.263 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:56:48.263 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.352803+0000", 2024-09-13T14:56:48.263 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0707e-05 2024-09-13T14:56:48.263 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:56:48.263 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:56:48.263 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:56:48.263 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.263 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.263 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.45749.0:229668 8.a e295/270)", 2024-09-13T14:56:48.263 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:47.352640+0000", 2024-09-13T14:56:48.264 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.001869559, 2024-09-13T14:56:48.264 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0021104230000000002, 2024-09-13T14:56:48.264 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:56:48.264 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:56:48.264 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "queued for pg", 2024-09-13T14:56:48.264 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:56:48.264 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.264 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:56:48.264 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.352640+0000", 2024-09-13T14:56:48.264 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:56:48.264 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.264 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.264 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:56:48.264 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.352640+0000", 2024-09-13T14:56:48.265 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:56:48.265 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.265 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.265 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:56:48.265 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.352641+0000", 2024-09-13T14:56:48.265 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.2599999999999998e-07 2024-09-13T14:56:48.265 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.265 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.265 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:56:48.265 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.352644+0000", 2024-09-13T14:56:48.265 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.5609999999999999e-06 2024-09-13T14:56:48.265 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.265 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.265 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:56:48.266 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.352653+0000", 2024-09-13T14:56:48.266 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2210000000000003e-06 2024-09-13T14:56:48.266 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:56:48.266 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:56:48.266 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:56:48.266 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:47.352658+0000", 2024-09-13T14:56:48.266 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.481e-06 2024-09-13T14:56:48.266 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:56:48.266 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:56:48.266 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:56:48.266 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:56:48.266 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2024-09-13T14:56:48.266 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 5 2024-09-13T14:56:48.267 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:56:48.346 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:48.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.378 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:48.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:48.479 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:48.503 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:48.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:48.605 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:48.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.644 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:48.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:48.745 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:48.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.773 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:48.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:48.874 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:48.911 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:48.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:48.952 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:56:49.013 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:49.075 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:49.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.112 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:49.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:49.127 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:49.213 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:49.281 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:49.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:49.383 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:49.410 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:49.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:49.513 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:49.517 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:49 smithi096 ceph-mon[98901]: pgmap v942: 109 pgs: 109 active+clean; 1.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2024-09-13T14:56:49.517 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:49 smithi096 ceph-mon[102928]: pgmap v942: 109 pgs: 109 active+clean; 1.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2024-09-13T14:56:49.571 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:49.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:49.673 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:49.707 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:49.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:49.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:49 smithi204 ceph-mon[87071]: pgmap v942: 109 pgs: 109 active+clean; 1.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2024-09-13T14:56:49.809 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:49.838 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:49.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:49.940 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:49.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.975 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:49.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:50.076 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:50.112 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:50.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:50.156 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:56:50.214 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:50.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.255 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:50.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:50.357 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:50.420 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:50.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:56:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:56:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:50.522 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:50.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.559 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:50.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:50.660 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:50.694 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:50.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:50.796 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:50.835 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:50.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:50.937 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:50.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.965 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:50.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:51.066 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:51.101 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:51.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:51.203 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:51.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.250 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:51.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:51.351 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:51.381 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:51.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:51.483 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:51.519 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:51.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:51.526 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:51 smithi096 ceph-mon[98901]: pgmap v943: 109 pgs: 109 active+clean; 1.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2024-09-13T14:56:51.526 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:51.527 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:51.527 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:51 smithi096 ceph-mon[102928]: pgmap v943: 109 pgs: 109 active+clean; 1.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2024-09-13T14:56:51.527 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:51.527 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:51.621 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:51.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.661 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:51.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:51 smithi204 ceph-mon[87071]: pgmap v943: 109 pgs: 109 active+clean; 1.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2024-09-13T14:56:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:56:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:51.762 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:51.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.796 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:51.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:51.897 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:51.927 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:51.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:52.029 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:52.061 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:52.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:52.141 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:52.164 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:52.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.215 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:52.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:52.316 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:52.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.349 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:52.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:52.450 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:52.483 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:52.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:52.586 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:52.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.625 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:52.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:52.726 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:52.757 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:52.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:52.859 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:52.902 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:52.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:53.004 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:53.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.061 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:53.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:53.162 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:53.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.202 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:53.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:53.303 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:53.335 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:53.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:53.437 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:53.465 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:53.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:53.551 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:53 smithi096 ceph-mon[98901]: pgmap v944: 109 pgs: 109 active+clean; 1.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2024-09-13T14:56:53.551 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:53 smithi096 ceph-mon[102928]: pgmap v944: 109 pgs: 109 active+clean; 1.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2024-09-13T14:56:53.567 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:53.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.599 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:53.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:53.700 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:53.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:53 smithi204 ceph-mon[87071]: pgmap v944: 109 pgs: 109 active+clean; 1.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2024-09-13T14:56:53.745 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:53.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:53.847 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:53.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.881 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:53.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:53.982 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:54.007 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:54.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:54.109 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:54.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.157 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:54.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:54.258 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:54.294 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:54.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:54.393 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:56:54.396 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:54.401 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T14:56:54.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.442 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:54.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:54.460 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:54.543 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:54.578 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:54.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:54.680 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:54.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:54 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/90169758' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:56:54.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:54 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:56:54.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:54 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/262088202' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:54.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.727 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:54.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:54.779 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:54 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/90169758' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:56:54.779 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:54 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:56:54.779 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:54 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/262088202' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:54.779 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:54 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/90169758' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:56:54.779 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:54 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:56:54.780 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:54 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/262088202' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:54.828 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:54.865 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:54.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:54.967 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:54.999 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:54.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:55.101 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:55.142 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:55.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:55.244 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:55.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.294 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:55.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:55.395 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:55.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.428 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:55.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:55.529 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:55.554 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:55 smithi096 ceph-mon[102928]: pgmap v945: 109 pgs: 109 active+clean; 989 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-13T14:56:55.554 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:55 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:55.554 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:55 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:56:55.554 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:55 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/262088202' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:55.554 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:55 smithi096 ceph-mon[102928]: osdmap e296: 8 total, 8 up, 8 in 2024-09-13T14:56:55.554 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:55 smithi096 ceph-mon[98901]: pgmap v945: 109 pgs: 109 active+clean; 989 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-13T14:56:55.554 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:55 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:55.555 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:55 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:56:55.555 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:55 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/262088202' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:55.555 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:55 smithi096 ceph-mon[98901]: osdmap e296: 8 total, 8 up, 8 in 2024-09-13T14:56:55.560 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:55.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:55.662 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:55.699 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:55.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:55 smithi204 ceph-mon[87071]: pgmap v945: 109 pgs: 109 active+clean; 989 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-13T14:56:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:55 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:55 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:56:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:55 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/262088202' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:55 smithi204 ceph-mon[87071]: osdmap e296: 8 total, 8 up, 8 in 2024-09-13T14:56:55.800 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:55.832 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:55.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:55.934 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:55.966 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:55.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:56.068 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:56.102 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:56.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:56.204 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:56.225 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:56.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:56.326 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:56.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.361 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:56.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:56.462 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:56.499 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:56.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:56.601 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:56.634 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:56.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:56.736 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:56.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.778 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:56.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:56.879 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:56.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.918 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:56.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:57.019 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:57.040 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:57.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:57.142 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:57.165 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:57.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:57.267 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:57.270 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:56:57.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.303 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:57.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:57.404 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:57.439 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:57.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:57.541 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:57.570 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:57.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:57.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:57 smithi096 ceph-mon[98901]: pgmap v947: 109 pgs: 109 active+clean; 989 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 941 op/s 2024-09-13T14:56:57.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:57 smithi096 ceph-mon[102928]: pgmap v947: 109 pgs: 109 active+clean; 989 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 941 op/s 2024-09-13T14:56:57.672 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:57.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.694 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:57.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:57.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:57 smithi204 ceph-mon[87071]: pgmap v947: 109 pgs: 109 active+clean; 989 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 941 op/s 2024-09-13T14:56:57.794 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:57.828 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:57.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:57.930 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:57.961 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:57.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:58.064 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:58.099 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:58.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:58.201 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:58.233 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:58.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:58.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:56:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:56:57] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.51.0" 2024-09-13T14:56:58.335 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:58.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.368 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:58.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:58.469 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:58.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.498 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:58.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:58.599 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:58.639 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:58.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:58.741 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:58.768 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:58.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:58.870 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:58.903 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:58.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:59.005 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:59.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.031 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:59.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:59.132 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:59.170 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:59.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:59.272 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:59.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.298 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:59.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:59.398 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:59.431 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:59.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:59.533 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:59.570 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:59.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:59.672 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:59.705 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:59.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:59.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:56:59 smithi204 ceph-mon[87071]: pgmap v948: 109 pgs: 109 active+clean; 879 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 939 op/s 2024-09-13T14:56:59.807 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:59.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:56:59 smithi096 ceph-mon[98901]: pgmap v948: 109 pgs: 109 active+clean; 879 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 939 op/s 2024-09-13T14:56:59.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:56:59 smithi096 ceph-mon[102928]: pgmap v948: 109 pgs: 109 active+clean; 879 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 939 op/s 2024-09-13T14:56:59.839 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:56:59.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:59.941 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:59.978 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:56:59.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:00.080 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:00.116 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:00.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:00.218 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:00.259 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:00.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:00.361 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:00.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.399 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:00.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:57:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:57:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:00.500 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:00.537 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T14:57:00.548 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:00.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:00.651 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:00.677 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:00.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:00.779 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:00.815 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:00.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:00.917 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:00.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.956 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:00.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:01.057 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:01.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.090 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:01.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:01.191 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:01.218 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:01.241 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:01.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:01.342 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:01.372 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:01.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:01.474 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:01.500 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:01.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:01.587 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:01 smithi096 ceph-mon[98901]: pgmap v949: 109 pgs: 109 active+clean; 701 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 927 op/s 2024-09-13T14:57:01.588 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:01 smithi096 ceph-mon[102928]: pgmap v949: 109 pgs: 109 active+clean; 701 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 927 op/s 2024-09-13T14:57:01.602 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:01.635 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:01.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:01.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:01 smithi204 ceph-mon[87071]: pgmap v949: 109 pgs: 109 active+clean; 701 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 927 op/s 2024-09-13T14:57:01.737 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:01.782 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:01.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:01.884 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:01.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.911 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:01.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:02.012 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:02.045 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:02.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:02.062 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:57:02.063 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:57:02.096 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:57:02.097 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:57:02.128 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:57:02.128 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:57:02.147 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:02.160 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:57:02.160 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:57:02.181 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:02.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:02.191 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:57:02.191 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:57:02.229 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:57:02.229 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:57:02.259 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:57:02.259 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:57:02.283 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:02.289 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:57:02.289 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:57:02.332 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:02.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:02.437 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:02.478 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:02.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:02.579 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:02.612 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:02.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:02.714 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:02.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:02 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3516923136' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:02.724 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:02 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3516923136' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:02.724 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:02 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3516923136' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:02.749 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:02.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:02.851 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:02.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.883 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:02.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:02.984 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:03.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.010 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:03.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:03.112 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:03.140 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:03.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:03.242 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:03.276 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:03.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:03.380 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:03.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.412 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:03.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:03.494 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:03.513 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:03.561 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:03.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:03.666 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:03.701 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:03.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:03.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:03 smithi204 ceph-mon[87071]: pgmap v950: 109 pgs: 109 active+clean; 665 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 930 op/s 2024-09-13T14:57:03.754 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:03 smithi096 ceph-mon[98901]: pgmap v950: 109 pgs: 109 active+clean; 665 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 930 op/s 2024-09-13T14:57:03.754 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:03 smithi096 ceph-mon[102928]: pgmap v950: 109 pgs: 109 active+clean; 665 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 930 op/s 2024-09-13T14:57:03.803 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:03.817 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:57:03.818 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:57:03.839 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:03.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:03.941 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:03.968 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:03.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:04.071 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:04.099 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:04.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:04.202 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:04.238 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:04.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:04.340 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:04.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.382 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:04.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:04.483 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:04.512 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:04.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:04.614 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:04.644 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:04.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:04.746 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:04.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.779 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:04.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:04.880 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:04.913 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:04.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:05.015 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:05.049 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:05.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:05.151 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:05.178 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:05.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:05.280 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:05.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.315 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:05.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:05.416 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:05.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.454 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:05.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:05.555 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:05.583 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:05.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:05.685 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:05.711 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:05.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:05 smithi204 ceph-mon[87071]: pgmap v951: 109 pgs: 109 active+clean; 402 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2024-09-13T14:57:05.813 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:05.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:05 smithi096 ceph-mon[98901]: pgmap v951: 109 pgs: 109 active+clean; 402 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2024-09-13T14:57:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:05 smithi096 ceph-mon[102928]: pgmap v951: 109 pgs: 109 active+clean; 402 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2024-09-13T14:57:05.851 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:05.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:05.954 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:05.977 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:05.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:06.079 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:06.110 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:06.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:06.212 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:06.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.241 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:06.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:06.342 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:06.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.378 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:06.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:06.479 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:06.508 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:06.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:06.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:06.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:06.609 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:06.639 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:06.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:06.741 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:06.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.787 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:06.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:06.889 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:06.921 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:06.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:07.022 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:07.061 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:07.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:07.163 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:07.187 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:07.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:07.288 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:07.314 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:07.416 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:07.450 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:07.552 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:07 smithi096 ceph-mon[98901]: pgmap v952: 109 pgs: 109 active+clean; 402 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-13T14:57:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:07 smithi096 ceph-mon[102928]: pgmap v952: 109 pgs: 109 active+clean; 402 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-13T14:57:07.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.582 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:07.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:07.684 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:07.713 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:07.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:07 smithi204 ceph-mon[87071]: pgmap v952: 109 pgs: 109 active+clean; 402 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-13T14:57:07.816 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:07.851 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:07.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:07.881 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:57:07.889 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:57:07.953 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:07.976 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:07.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:08.078 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:08.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.141 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:08.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:08.242 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:08.266 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:08.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:08.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:57:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:57:07] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.51.0" 2024-09-13T14:57:08.368 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:08.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.421 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:08.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:08.522 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:08.556 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:08.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:08.658 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:08.691 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:08.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:08.793 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:08.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.814 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:08.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:08.915 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:08.953 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:08.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:09.054 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:09.092 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:09.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:09.194 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:09.226 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:09.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:09.328 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:09.364 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:09.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:09.466 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:09.514 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:09.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:09.616 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:09.649 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:09.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:09.714 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:09 smithi096 ceph-mon[98901]: pgmap v953: 109 pgs: 109 active+clean; 288 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 934 op/s 2024-09-13T14:57:09.714 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:09 smithi096 ceph-mon[102928]: pgmap v953: 109 pgs: 109 active+clean; 288 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 934 op/s 2024-09-13T14:57:09.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:09 smithi204 ceph-mon[87071]: pgmap v953: 109 pgs: 109 active+clean; 288 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 934 op/s 2024-09-13T14:57:09.751 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:09.791 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:09.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:09.871 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Removed 125392 objects 2024-09-13T14:57:09.871 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Clean up completed and total clean up time :136.233 2024-09-13T14:57:09.894 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:09.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.917 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:09.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:10.018 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:10.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.041 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:10.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:10.142 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:10.163 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:10.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:10.265 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:10.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.294 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:10.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:10.395 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:10.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.417 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:10.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:57:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:57:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:10.519 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:10.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.541 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:10.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:10.626 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:10.630 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:10.642 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:10.666 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:10.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:10.768 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:10.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.791 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:10.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:10.892 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:10.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.914 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:10.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:11.015 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:11.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.038 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:11.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:11.139 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:11.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.162 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:11.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:11.263 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:11.284 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:11.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:11.386 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:11.407 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:11.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:11.469 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:57:11.469 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T14:57:11.509 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:11.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.531 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:11.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:11.632 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:11.654 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:11.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:11 smithi204 ceph-mon[87071]: pgmap v954: 109 pgs: 109 active+clean; 107 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:57:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:11 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2708439805' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:11 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:11 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1581999867' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:57:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:11 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:57:11.757 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:11.778 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:11.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:11.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:11 smithi096 ceph-mon[102928]: pgmap v954: 109 pgs: 109 active+clean; 107 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:57:11.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:11 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2708439805' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:11.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:11 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:11.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:11 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1581999867' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:57:11.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:11 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:57:11.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:11 smithi096 ceph-mon[98901]: pgmap v954: 109 pgs: 109 active+clean; 107 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:57:11.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:11 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2708439805' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:11.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:11 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:11.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:11 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1581999867' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:57:11.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:11 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:57:11.880 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:11.902 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:11.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:12.004 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:12.027 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:12.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:12.129 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:12.151 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:12.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:12.253 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:12.276 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:12.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:12.378 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:12.403 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:12.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:12.505 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:12.529 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:12.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:12.631 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:12.655 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:12.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:12.717 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:12 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:12.717 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:12 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:57:12.717 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:12 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:57:12.718 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:12 smithi096 ceph-mon[98901]: osdmap e297: 8 total, 8 up, 8 in 2024-09-13T14:57:12.718 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:12 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:12.718 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:12 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:57:12.718 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:12 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:57:12.718 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:12 smithi096 ceph-mon[102928]: osdmap e297: 8 total, 8 up, 8 in 2024-09-13T14:57:12.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:12 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:12.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:12 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:57:12.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:12 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:57:12.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:12 smithi204 ceph-mon[87071]: osdmap e297: 8 total, 8 up, 8 in 2024-09-13T14:57:12.757 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:12.781 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:12.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:12.787 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T14:57:12.883 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:12.908 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:12.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:13.010 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:13.036 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:13.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:13.138 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:13.162 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:13.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:13.264 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:13.289 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:13.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:13.391 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:13.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.415 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:13.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:13.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:13.543 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:13.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:13.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:13 smithi096 ceph-mon[98901]: pgmap v956: 109 pgs: 109 active+clean; 87 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 924 op/s 2024-09-13T14:57:13.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:13 smithi096 ceph-mon[102928]: pgmap v956: 109 pgs: 109 active+clean; 87 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 924 op/s 2024-09-13T14:57:13.644 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:13.670 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:13.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:13.771 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:13.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.797 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:13.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:13.898 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:13.920 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:13.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:13.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:13 smithi204 ceph-mon[87071]: pgmap v956: 109 pgs: 109 active+clean; 87 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 924 op/s 2024-09-13T14:57:14.021 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:14.047 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:14.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:14.149 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:14.175 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:14.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:14.277 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:14.302 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:14.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:14.392 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:57:14.404 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:14.465 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:14.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:14.567 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:14.593 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:14.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:14.695 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:14.720 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:14.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:14.822 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:14.848 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:14.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:14.950 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:14.976 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:14.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:15.078 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:15.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.100 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:15.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:15.185 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:57:15.185 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:57:15.185 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:57:15.185 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:57:15.190 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:57:15.190 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:57:15.201 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:15.206 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:57:15.206 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:57:15.222 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:57:15.222 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:57:15.225 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:15.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:15.237 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:57:15.238 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:57:15.254 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:57:15.254 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:57:15.269 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:57:15.269 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:57:15.317 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-09-13T14:57:15.317 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-09-13T14:57:15.327 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:15.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.389 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:15.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:15.444 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:57:15.444 INFO:teuthology.orchestra.run.smithi096.stdout: "size": 20, 2024-09-13T14:57:15.444 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 600, 2024-09-13T14:57:15.444 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:57:15.444 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.445 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181921 6.14 6:291668c5:::benchmark_data_smithi096_167087_object64693:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:57:15.445 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:16.682370+0000", 2024-09-13T14:57:15.445 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 598.75788206000004, 2024-09-13T14:57:15.445 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.92264975199999999, 2024-09-13T14:57:15.445 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.445 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.445 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.445 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.445 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.445 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.445 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181921 2024-09-13T14:57:15.445 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.446 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.446 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.446 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.446 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682370+0000", 2024-09-13T14:57:15.446 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.446 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.446 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.446 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.446 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682370+0000", 2024-09-13T14:57:15.446 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.447 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.447 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.447 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.447 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682371+0000", 2024-09-13T14:57:15.447 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4750000000000001e-06 2024-09-13T14:57:15.447 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.447 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.447 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.447 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682377+0000", 2024-09-13T14:57:15.447 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7749999999999998e-06 2024-09-13T14:57:15.447 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.448 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.448 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.448 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682393+0000", 2024-09-13T14:57:15.448 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5710999999999999e-05 2024-09-13T14:57:15.448 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.448 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.449 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.449 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682401+0000", 2024-09-13T14:57:15.449 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9140000000000008e-06 2024-09-13T14:57:15.449 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.449 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.449 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.449 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682470+0000", 2024-09-13T14:57:15.449 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9227000000000007e-05 2024-09-13T14:57:15.449 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.450 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.450 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.450 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.682479+0000", 2024-09-13T14:57:15.450 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9220000000000003e-06 2024-09-13T14:57:15.450 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.450 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.450 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.450 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.601256+0000", 2024-09-13T14:57:15.450 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.91877762500000004 2024-09-13T14:57:15.450 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.451 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.451 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.451 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.601372+0000", 2024-09-13T14:57:15.451 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011595 2024-09-13T14:57:15.451 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.451 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.452 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:57:15.452 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.601494+0000", 2024-09-13T14:57:15.452 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012179400000000001 2024-09-13T14:57:15.452 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.452 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.452 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.452 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.604870+0000", 2024-09-13T14:57:15.452 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0033754520000000001 2024-09-13T14:57:15.452 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.453 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.453 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.453 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.604936+0000", 2024-09-13T14:57:15.453 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5977999999999997e-05 2024-09-13T14:57:15.453 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.454 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.454 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.454 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.604946+0000", 2024-09-13T14:57:15.454 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0729e-05 2024-09-13T14:57:15.454 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.454 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.454 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.455 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.604983+0000", 2024-09-13T14:57:15.455 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6902e-05 2024-09-13T14:57:15.455 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.455 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.455 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.455 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.605020+0000", 2024-09-13T14:57:15.455 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6297999999999997e-05 2024-09-13T14:57:15.456 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.456 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.456 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.456 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.456 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.456 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181940 6.14 6:2de03b2d:::benchmark_data_smithi096_167087_object64712:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:57:15.456 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:16.696647+0000", 2024-09-13T14:57:15.457 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 598.74360526400005, 2024-09-13T14:57:15.457 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.91128050299999996, 2024-09-13T14:57:15.457 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.457 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.457 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.457 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.457 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.458 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.458 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181940 2024-09-13T14:57:15.458 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.458 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.458 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.458 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.458 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696647+0000", 2024-09-13T14:57:15.459 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.459 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.459 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.459 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.459 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696647+0000", 2024-09-13T14:57:15.459 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.459 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.459 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.460 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.460 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696648+0000", 2024-09-13T14:57:15.460 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.579e-06 2024-09-13T14:57:15.460 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.460 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.460 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.460 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696654+0000", 2024-09-13T14:57:15.461 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1349999999999997e-06 2024-09-13T14:57:15.461 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.461 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.461 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.461 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696667+0000", 2024-09-13T14:57:15.461 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2727e-05 2024-09-13T14:57:15.461 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.462 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.462 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.462 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696675+0000", 2024-09-13T14:57:15.462 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1899999999999995e-06 2024-09-13T14:57:15.462 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.463 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.463 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.463 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696911+0000", 2024-09-13T14:57:15.463 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00023602599999999999 2024-09-13T14:57:15.463 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.463 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.463 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.463 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.696920+0000", 2024-09-13T14:57:15.464 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8300000000000002e-06 2024-09-13T14:57:15.464 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.464 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.464 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.464 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.603441+0000", 2024-09-13T14:57:15.464 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.90652066200000003 2024-09-13T14:57:15.464 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.465 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.465 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.465 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.603565+0000", 2024-09-13T14:57:15.465 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012396799999999999 2024-09-13T14:57:15.465 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.465 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.465 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:57:15.466 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.603668+0000", 2024-09-13T14:57:15.466 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000102938 2024-09-13T14:57:15.466 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.466 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.468 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.468 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.606432+0000", 2024-09-13T14:57:15.468 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027647000000000001 2024-09-13T14:57:15.468 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.468 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.468 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.468 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.606482+0000", 2024-09-13T14:57:15.469 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9243999999999998e-05 2024-09-13T14:57:15.469 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.469 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.469 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.469 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.607831+0000", 2024-09-13T14:57:15.469 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0013495219999999999 2024-09-13T14:57:15.469 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.470 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.470 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.470 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.607881+0000", 2024-09-13T14:57:15.470 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9648999999999998e-05 2024-09-13T14:57:15.470 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.470 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.473 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.473 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.607927+0000", 2024-09-13T14:57:15.473 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6332999999999998e-05 2024-09-13T14:57:15.473 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.473 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.473 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.473 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.474 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.474 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182060 6.14 6:2db5f18f:::benchmark_data_smithi096_167087_object64832:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:57:15.474 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:16.790348+0000", 2024-09-13T14:57:15.474 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 598.64990437599999, 2024-09-13T14:57:15.474 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.81781258199999995, 2024-09-13T14:57:15.474 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.474 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.475 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.475 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.475 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.475 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.475 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182060 2024-09-13T14:57:15.475 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.475 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.476 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.476 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.476 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790348+0000", 2024-09-13T14:57:15.476 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.476 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.476 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.476 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.477 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790348+0000", 2024-09-13T14:57:15.477 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.477 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.477 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.477 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.477 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790349+0000", 2024-09-13T14:57:15.477 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6309999999999999e-06 2024-09-13T14:57:15.478 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.478 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.478 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.478 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790360+0000", 2024-09-13T14:57:15.478 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1270999999999999e-05 2024-09-13T14:57:15.478 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.478 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.479 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.479 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790374+0000", 2024-09-13T14:57:15.479 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3995000000000001e-05 2024-09-13T14:57:15.479 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.479 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.479 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.479 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790383+0000", 2024-09-13T14:57:15.480 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.7520000000000002e-06 2024-09-13T14:57:15.480 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.480 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.480 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.480 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790459+0000", 2024-09-13T14:57:15.480 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.5407000000000005e-05 2024-09-13T14:57:15.481 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.481 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.481 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.481 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.790467+0000", 2024-09-13T14:57:15.481 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.3089999999999996e-06 2024-09-13T14:57:15.481 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.481 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.482 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.482 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.605048+0000", 2024-09-13T14:57:15.482 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.81458113899999995 2024-09-13T14:57:15.482 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.482 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.482 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.482 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.605154+0000", 2024-09-13T14:57:15.482 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010594600000000001 2024-09-13T14:57:15.483 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.483 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.483 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:57:15.483 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.605265+0000", 2024-09-13T14:57:15.483 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011089700000000001 2024-09-13T14:57:15.483 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.483 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.483 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.483 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608038+0000", 2024-09-13T14:57:15.484 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027728029999999999 2024-09-13T14:57:15.484 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.484 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.484 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.484 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608082+0000", 2024-09-13T14:57:15.484 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4183e-05 2024-09-13T14:57:15.484 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.484 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.484 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.484 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608091+0000", 2024-09-13T14:57:15.484 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9770000000000006e-06 2024-09-13T14:57:15.485 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.485 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.485 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.485 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608118+0000", 2024-09-13T14:57:15.485 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7341000000000001e-05 2024-09-13T14:57:15.485 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.485 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.485 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.485 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608160+0000", 2024-09-13T14:57:15.485 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1931000000000002e-05 2024-09-13T14:57:15.485 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.485 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.486 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.486 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.486 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.486 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182076 6.14 6:2fdfb44e:::benchmark_data_smithi096_167087_object64848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:57:15.486 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:16.803957+0000", 2024-09-13T14:57:15.486 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 598.63629532699997, 2024-09-13T14:57:15.486 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.80631315599999998, 2024-09-13T14:57:15.486 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.486 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.486 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.486 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.486 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.487 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.487 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182076 2024-09-13T14:57:15.487 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.487 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.487 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.487 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.487 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803957+0000", 2024-09-13T14:57:15.487 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.487 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.487 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.487 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.487 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803957+0000", 2024-09-13T14:57:15.488 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.488 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.488 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.488 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.488 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803958+0000", 2024-09-13T14:57:15.488 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8989999999999999e-06 2024-09-13T14:57:15.488 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.488 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.488 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.488 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803964+0000", 2024-09-13T14:57:15.488 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6319999999999996e-06 2024-09-13T14:57:15.489 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.489 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.489 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.489 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803979+0000", 2024-09-13T14:57:15.489 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5045000000000001e-05 2024-09-13T14:57:15.489 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.489 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.489 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.489 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.803987+0000", 2024-09-13T14:57:15.489 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3799999999999996e-06 2024-09-13T14:57:15.489 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.489 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.490 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.490 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.804041+0000", 2024-09-13T14:57:15.490 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4199000000000001e-05 2024-09-13T14:57:15.490 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.490 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.490 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.490 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.804048+0000", 2024-09-13T14:57:15.490 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3050000000000004e-06 2024-09-13T14:57:15.490 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.490 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.490 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.490 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.606528+0000", 2024-09-13T14:57:15.491 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.80247967200000003 2024-09-13T14:57:15.491 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.491 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.491 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.491 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.606642+0000", 2024-09-13T14:57:15.491 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011442699999999999 2024-09-13T14:57:15.491 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.491 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.491 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:57:15.491 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.606726+0000", 2024-09-13T14:57:15.491 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.4073000000000006e-05 2024-09-13T14:57:15.491 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.492 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.492 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.492 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608231+0000", 2024-09-13T14:57:15.492 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0015046090000000001 2024-09-13T14:57:15.492 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.492 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.492 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.492 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608248+0000", 2024-09-13T14:57:15.492 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7244e-05 2024-09-13T14:57:15.492 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.492 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.493 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.493 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.609959+0000", 2024-09-13T14:57:15.493 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001710563 2024-09-13T14:57:15.493 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.493 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.493 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.493 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.610178+0000", 2024-09-13T14:57:15.493 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00021929600000000001 2024-09-13T14:57:15.493 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.493 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.494 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.494 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.610270+0000", 2024-09-13T14:57:15.494 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.1811999999999994e-05 2024-09-13T14:57:15.494 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.494 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.494 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.494 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.495 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.495 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182107 6.14 6:2cba648f:::benchmark_data_smithi096_167087_object64879:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:57:15.495 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:16.833170+0000", 2024-09-13T14:57:15.495 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 598.60708146299999, 2024-09-13T14:57:15.495 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.78124621000000005, 2024-09-13T14:57:15.495 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.495 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.496 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.496 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.496 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.496 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.496 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182107 2024-09-13T14:57:15.496 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.496 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.496 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.496 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.497 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833170+0000", 2024-09-13T14:57:15.497 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.497 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.497 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.497 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.497 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833170+0000", 2024-09-13T14:57:15.497 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.497 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.497 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.497 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.497 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833172+0000", 2024-09-13T14:57:15.498 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6449999999999999e-06 2024-09-13T14:57:15.498 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.498 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.498 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.498 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833178+0000", 2024-09-13T14:57:15.498 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1770000000000003e-06 2024-09-13T14:57:15.498 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.498 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.498 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.498 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833192+0000", 2024-09-13T14:57:15.499 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4082e-05 2024-09-13T14:57:15.499 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.499 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.499 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.499 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833200+0000", 2024-09-13T14:57:15.499 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2520000000000002e-06 2024-09-13T14:57:15.499 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.499 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.500 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.500 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833256+0000", 2024-09-13T14:57:15.500 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6427e-05 2024-09-13T14:57:15.500 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.500 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.500 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.500 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:16.833264+0000", 2024-09-13T14:57:15.500 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1629999999999996e-06 2024-09-13T14:57:15.500 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.501 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.501 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.501 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608283+0000", 2024-09-13T14:57:15.501 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.77501878800000001 2024-09-13T14:57:15.501 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.501 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.501 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.501 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608389+0000", 2024-09-13T14:57:15.501 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010569300000000001 2024-09-13T14:57:15.501 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.502 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.502 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:57:15.502 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.608505+0000", 2024-09-13T14:57:15.502 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000116434 2024-09-13T14:57:15.502 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.502 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.502 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.502 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612122+0000", 2024-09-13T14:57:15.502 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0036171860000000001 2024-09-13T14:57:15.502 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.502 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.503 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.503 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.614303+0000", 2024-09-13T14:57:15.503 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0021807839999999998 2024-09-13T14:57:15.503 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.503 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.503 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.503 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.614323+0000", 2024-09-13T14:57:15.503 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9644e-05 2024-09-13T14:57:15.503 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.504 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.504 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.504 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.614369+0000", 2024-09-13T14:57:15.504 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6048999999999999e-05 2024-09-13T14:57:15.504 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.504 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.504 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.504 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.614417+0000", 2024-09-13T14:57:15.504 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7886000000000002e-05 2024-09-13T14:57:15.505 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.505 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.505 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.505 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.505 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.505 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181897 6.18 6:1a0fd876:::benchmark_data_smithi096_167087_object64669:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:57:15.505 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583616+0000", 2024-09-13T14:57:15.506 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 597.85663616299996, 2024-09-13T14:57:15.506 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.110115685, 2024-09-13T14:57:15.506 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.506 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.506 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.506 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.506 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.506 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.507 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181897 2024-09-13T14:57:15.507 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.507 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.507 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.507 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.507 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583616+0000", 2024-09-13T14:57:15.507 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.508 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.508 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.508 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.508 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583616+0000", 2024-09-13T14:57:15.508 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.509 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.509 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.509 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.509 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583617+0000", 2024-09-13T14:57:15.509 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.482e-06 2024-09-13T14:57:15.509 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.509 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.509 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.510 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583623+0000", 2024-09-13T14:57:15.510 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1190000000000002e-06 2024-09-13T14:57:15.510 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.510 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.510 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.510 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583637+0000", 2024-09-13T14:57:15.510 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3718e-05 2024-09-13T14:57:15.511 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.511 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.511 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.511 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583646+0000", 2024-09-13T14:57:15.511 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.602e-06 2024-09-13T14:57:15.511 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.511 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.511 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.512 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583943+0000", 2024-09-13T14:57:15.512 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00029740399999999997 2024-09-13T14:57:15.512 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.512 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.512 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.512 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583952+0000", 2024-09-13T14:57:15.512 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.4169999999999998e-06 2024-09-13T14:57:15.513 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.513 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.513 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.513 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.686789+0000", 2024-09-13T14:57:15.513 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1028368799999999 2024-09-13T14:57:15.513 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.513 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.513 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.514 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.686879+0000", 2024-09-13T14:57:15.514 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9676e-05 2024-09-13T14:57:15.514 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.514 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.514 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:57:15.514 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.687143+0000", 2024-09-13T14:57:15.514 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00026446800000000002 2024-09-13T14:57:15.515 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.515 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.515 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.515 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.692990+0000", 2024-09-13T14:57:15.515 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0058462310000000003 2024-09-13T14:57:15.515 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.515 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.516 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.516 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693001+0000", 2024-09-13T14:57:15.516 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1552999999999999e-05 2024-09-13T14:57:15.516 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.516 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.516 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.516 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693028+0000", 2024-09-13T14:57:15.517 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7104999999999999e-05 2024-09-13T14:57:15.517 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.517 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.517 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.517 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693064+0000", 2024-09-13T14:57:15.517 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.5503000000000002e-05 2024-09-13T14:57:15.517 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.517 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.518 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.518 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693489+0000", 2024-09-13T14:57:15.518 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00042502099999999998 2024-09-13T14:57:15.518 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.518 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.518 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.518 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693603+0000", 2024-09-13T14:57:15.519 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011386 2024-09-13T14:57:15.519 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.519 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.519 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.519 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693731+0000", 2024-09-13T14:57:15.519 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000128646 2024-09-13T14:57:15.519 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.519 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.520 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.520 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.520 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.520 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181961 6.18 6:1b4cf7bc:::benchmark_data_smithi096_167087_object64733:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:57:15.520 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583723+0000", 2024-09-13T14:57:15.520 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 597.85652894600003, 2024-09-13T14:57:15.520 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1144489200000001, 2024-09-13T14:57:15.521 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.521 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.521 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.521 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.521 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.521 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.521 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181961 2024-09-13T14:57:15.522 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.522 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.522 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.522 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.522 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583723+0000", 2024-09-13T14:57:15.522 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.522 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.523 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.523 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.523 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583723+0000", 2024-09-13T14:57:15.523 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.523 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.523 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.523 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.524 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583724+0000", 2024-09-13T14:57:15.524 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9100000000000004e-07 2024-09-13T14:57:15.524 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.524 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.524 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.524 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583727+0000", 2024-09-13T14:57:15.524 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.191e-06 2024-09-13T14:57:15.525 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.525 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.525 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.525 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583736+0000", 2024-09-13T14:57:15.525 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2480000000000002e-06 2024-09-13T14:57:15.525 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.525 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.526 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.526 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583742+0000", 2024-09-13T14:57:15.526 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3840000000000002e-06 2024-09-13T14:57:15.526 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.526 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.526 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.526 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583997+0000", 2024-09-13T14:57:15.526 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00025479700000000002 2024-09-13T14:57:15.527 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.527 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.527 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.527 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584006+0000", 2024-09-13T14:57:15.527 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9900000000000003e-06 2024-09-13T14:57:15.527 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.527 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.528 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.528 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.689501+0000", 2024-09-13T14:57:15.528 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.105495082 2024-09-13T14:57:15.528 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.528 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.528 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.528 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.689618+0000", 2024-09-13T14:57:15.529 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000116469 2024-09-13T14:57:15.529 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.529 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.529 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:57:15.529 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.689865+0000", 2024-09-13T14:57:15.529 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00024748899999999999 2024-09-13T14:57:15.529 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.530 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.530 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.530 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693075+0000", 2024-09-13T14:57:15.530 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0032099450000000001 2024-09-13T14:57:15.530 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.530 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.530 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.531 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693314+0000", 2024-09-13T14:57:15.531 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00023886600000000001 2024-09-13T14:57:15.531 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.531 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.531 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.531 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693417+0000", 2024-09-13T14:57:15.531 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000103482 2024-09-13T14:57:15.532 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.532 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.532 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.532 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693894+0000", 2024-09-13T14:57:15.532 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00047631099999999999 2024-09-13T14:57:15.532 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.532 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.533 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.533 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.697947+0000", 2024-09-13T14:57:15.533 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0040532220000000004 2024-09-13T14:57:15.533 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.533 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.533 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.533 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698010+0000", 2024-09-13T14:57:15.534 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3177999999999997e-05 2024-09-13T14:57:15.534 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.534 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.534 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.534 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698172+0000", 2024-09-13T14:57:15.534 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00016167500000000001 2024-09-13T14:57:15.534 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.535 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.535 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.535 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.535 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.535 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181983 6.18 6:1d5af90f:::benchmark_data_smithi096_167087_object64755:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:57:15.535 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583752+0000", 2024-09-13T14:57:15.535 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 597.85649996400002, 2024-09-13T14:57:15.536 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.114628411, 2024-09-13T14:57:15.536 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.536 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.536 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.536 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.536 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.536 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.537 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181983 2024-09-13T14:57:15.537 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.537 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.537 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.537 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.537 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583752+0000", 2024-09-13T14:57:15.537 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.538 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.538 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.538 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.538 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583752+0000", 2024-09-13T14:57:15.538 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.538 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.538 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.539 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.539 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583752+0000", 2024-09-13T14:57:15.539 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9099999999999999e-07 2024-09-13T14:57:15.539 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.539 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.539 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.539 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583754+0000", 2024-09-13T14:57:15.539 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7969999999999999e-06 2024-09-13T14:57:15.539 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.539 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.539 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.540 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583760+0000", 2024-09-13T14:57:15.540 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1600000000000003e-06 2024-09-13T14:57:15.540 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.540 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.540 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.540 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583765+0000", 2024-09-13T14:57:15.540 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0540000000000002e-06 2024-09-13T14:57:15.540 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.540 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.540 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.540 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584049+0000", 2024-09-13T14:57:15.541 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00028396299999999998 2024-09-13T14:57:15.541 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.541 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.541 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.541 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584057+0000", 2024-09-13T14:57:15.541 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.6920000000000002e-06 2024-09-13T14:57:15.541 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.541 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.541 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.541 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.691347+0000", 2024-09-13T14:57:15.541 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.107290417 2024-09-13T14:57:15.542 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.542 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.542 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.542 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.691416+0000", 2024-09-13T14:57:15.542 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9029000000000006e-05 2024-09-13T14:57:15.542 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.542 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.542 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:57:15.542 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.691630+0000", 2024-09-13T14:57:15.542 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00021387500000000001 2024-09-13T14:57:15.542 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.542 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.543 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.543 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693330+0000", 2024-09-13T14:57:15.543 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0016996730000000001 2024-09-13T14:57:15.543 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.543 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.543 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.543 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693794+0000", 2024-09-13T14:57:15.543 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00046411 2024-09-13T14:57:15.543 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.543 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.543 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.544 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693850+0000", 2024-09-13T14:57:15.544 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5733999999999999e-05 2024-09-13T14:57:15.544 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.544 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.544 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.544 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.693924+0000", 2024-09-13T14:57:15.544 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3755000000000001e-05 2024-09-13T14:57:15.544 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.544 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.544 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.544 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698216+0000", 2024-09-13T14:57:15.544 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0042927490000000002 2024-09-13T14:57:15.545 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.545 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.545 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.545 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698273+0000", 2024-09-13T14:57:15.545 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6558000000000003e-05 2024-09-13T14:57:15.545 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.545 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.545 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.545 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698380+0000", 2024-09-13T14:57:15.545 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000107454 2024-09-13T14:57:15.545 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.545 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.546 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.546 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.546 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.546 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:181992 6.18 6:1dd34154:::benchmark_data_smithi096_167087_object64764:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:57:15.546 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583771+0000", 2024-09-13T14:57:15.546 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 597.85648044599998, 2024-09-13T14:57:15.546 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1206707570000001, 2024-09-13T14:57:15.546 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.546 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.546 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.546 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.547 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.547 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.547 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 181992 2024-09-13T14:57:15.547 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.547 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.547 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.547 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.547 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583771+0000", 2024-09-13T14:57:15.547 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.547 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.547 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.547 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.548 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583771+0000", 2024-09-13T14:57:15.548 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.548 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.548 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.548 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.548 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583772+0000", 2024-09-13T14:57:15.548 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6199999999999999e-07 2024-09-13T14:57:15.548 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.548 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.549 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.549 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583774+0000", 2024-09-13T14:57:15.549 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7179999999999999e-06 2024-09-13T14:57:15.549 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.549 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.549 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.549 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583779+0000", 2024-09-13T14:57:15.550 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9340000000000003e-06 2024-09-13T14:57:15.550 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.550 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.550 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.550 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583784+0000", 2024-09-13T14:57:15.550 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6569999999999996e-06 2024-09-13T14:57:15.550 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.551 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.551 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.551 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584091+0000", 2024-09-13T14:57:15.551 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00030664100000000002 2024-09-13T14:57:15.551 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.551 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.551 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.552 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584097+0000", 2024-09-13T14:57:15.552 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6640000000000001e-06 2024-09-13T14:57:15.552 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.552 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.552 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.552 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.694002+0000", 2024-09-13T14:57:15.552 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1099042059999999 2024-09-13T14:57:15.552 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.553 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.553 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.553 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.694103+0000", 2024-09-13T14:57:15.553 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000101208 2024-09-13T14:57:15.553 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.553 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.553 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:57:15.554 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.694412+0000", 2024-09-13T14:57:15.554 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00030881000000000002 2024-09-13T14:57:15.554 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.554 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.554 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.554 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698418+0000", 2024-09-13T14:57:15.554 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0040063989999999999 2024-09-13T14:57:15.555 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.555 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.555 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.555 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.703944+0000", 2024-09-13T14:57:15.555 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0055258440000000002 2024-09-13T14:57:15.555 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.555 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.555 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.556 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.703970+0000", 2024-09-13T14:57:15.556 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6594999999999999e-05 2024-09-13T14:57:15.556 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.556 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.556 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.556 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704017+0000", 2024-09-13T14:57:15.556 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6727999999999997e-05 2024-09-13T14:57:15.557 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.557 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.557 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.557 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704259+0000", 2024-09-13T14:57:15.557 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000241745 2024-09-13T14:57:15.557 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.557 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.557 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.557 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704318+0000", 2024-09-13T14:57:15.557 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9117000000000003e-05 2024-09-13T14:57:15.558 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.558 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.558 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.558 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704442+0000", 2024-09-13T14:57:15.558 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000124129 2024-09-13T14:57:15.558 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.558 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.558 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.558 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.558 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.559 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182029 6.18 6:1b014b20:::benchmark_data_smithi096_167087_object64801:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:57:15.559 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583790+0000", 2024-09-13T14:57:15.559 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 597.85646228799999, 2024-09-13T14:57:15.559 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1236003029999999, 2024-09-13T14:57:15.559 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.559 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.559 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.559 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.559 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.559 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.560 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182029 2024-09-13T14:57:15.560 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.560 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.560 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.560 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.560 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583790+0000", 2024-09-13T14:57:15.560 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.560 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.560 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.560 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.561 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583790+0000", 2024-09-13T14:57:15.561 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.561 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.561 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.561 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.561 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583790+0000", 2024-09-13T14:57:15.561 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.41e-07 2024-09-13T14:57:15.561 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.561 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.561 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.562 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583792+0000", 2024-09-13T14:57:15.562 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.734e-06 2024-09-13T14:57:15.562 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.562 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.562 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.562 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583798+0000", 2024-09-13T14:57:15.562 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8749999999999997e-06 2024-09-13T14:57:15.562 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.562 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.562 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.562 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583802+0000", 2024-09-13T14:57:15.563 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.685e-06 2024-09-13T14:57:15.563 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.563 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.563 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.563 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584124+0000", 2024-09-13T14:57:15.563 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00032207999999999998 2024-09-13T14:57:15.563 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.563 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.563 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.564 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584131+0000", 2024-09-13T14:57:15.564 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2439999999999998e-06 2024-09-13T14:57:15.564 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.564 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.564 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.564 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.695817+0000", 2024-09-13T14:57:15.564 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1116864829999999 2024-09-13T14:57:15.564 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.564 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.564 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.564 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.695938+0000", 2024-09-13T14:57:15.565 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000121177 2024-09-13T14:57:15.565 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.565 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.565 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:57:15.565 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.696188+0000", 2024-09-13T14:57:15.565 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00024953900000000001 2024-09-13T14:57:15.565 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.565 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.565 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.565 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698457+0000", 2024-09-13T14:57:15.566 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0022691069999999998 2024-09-13T14:57:15.566 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.566 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.566 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.566 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.703978+0000", 2024-09-13T14:57:15.566 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0055207859999999997 2024-09-13T14:57:15.566 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.566 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.566 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.567 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704061+0000", 2024-09-13T14:57:15.567 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.3146999999999995e-05 2024-09-13T14:57:15.567 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.567 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.567 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.567 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704145+0000", 2024-09-13T14:57:15.567 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.373e-05 2024-09-13T14:57:15.567 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.568 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.568 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.568 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707052+0000", 2024-09-13T14:57:15.568 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0029077640000000002 2024-09-13T14:57:15.568 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.568 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.568 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.569 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707257+0000", 2024-09-13T14:57:15.569 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00020468299999999999 2024-09-13T14:57:15.569 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.569 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.569 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.569 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707390+0000", 2024-09-13T14:57:15.569 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013292800000000001 2024-09-13T14:57:15.570 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.570 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.570 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.570 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.570 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.570 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182109 6.18 6:1e340a78:::benchmark_data_smithi096_167087_object64881:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:57:15.570 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583808+0000", 2024-09-13T14:57:15.570 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 597.85644425600003, 2024-09-13T14:57:15.571 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1239120069999999, 2024-09-13T14:57:15.571 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.571 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.571 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.571 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.571 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.571 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.572 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182109 2024-09-13T14:57:15.572 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.572 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.572 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.572 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.572 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583808+0000", 2024-09-13T14:57:15.572 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.572 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.573 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.573 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.573 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583808+0000", 2024-09-13T14:57:15.573 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.573 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.573 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.573 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.574 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583808+0000", 2024-09-13T14:57:15.574 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7099999999999997e-07 2024-09-13T14:57:15.574 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.574 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.574 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.574 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583810+0000", 2024-09-13T14:57:15.574 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.629e-06 2024-09-13T14:57:15.574 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.575 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.575 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.575 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583816+0000", 2024-09-13T14:57:15.575 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8930000000000002e-06 2024-09-13T14:57:15.575 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.575 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.575 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.575 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583820+0000", 2024-09-13T14:57:15.576 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7879999999999997e-06 2024-09-13T14:57:15.576 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.576 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.576 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.576 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584160+0000", 2024-09-13T14:57:15.576 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00034013899999999999 2024-09-13T14:57:15.576 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.577 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.577 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.577 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584168+0000", 2024-09-13T14:57:15.577 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.063e-06 2024-09-13T14:57:15.577 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.577 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.577 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.577 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698486+0000", 2024-09-13T14:57:15.578 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1143184820000001 2024-09-13T14:57:15.578 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.578 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.578 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.578 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698605+0000", 2024-09-13T14:57:15.578 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000119276 2024-09-13T14:57:15.578 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.579 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.579 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:57:15.579 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.698944+0000", 2024-09-13T14:57:15.579 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0003389 2024-09-13T14:57:15.579 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.579 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.579 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.579 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.703983+0000", 2024-09-13T14:57:15.580 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0050390319999999997 2024-09-13T14:57:15.580 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.580 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.580 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.580 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704099+0000", 2024-09-13T14:57:15.580 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011535300000000001 2024-09-13T14:57:15.580 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.581 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.581 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.581 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704202+0000", 2024-09-13T14:57:15.581 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010301700000000001 2024-09-13T14:57:15.581 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.581 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.581 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.582 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707435+0000", 2024-09-13T14:57:15.582 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0032336909999999999 2024-09-13T14:57:15.582 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.582 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.582 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.582 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707572+0000", 2024-09-13T14:57:15.582 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000136923 2024-09-13T14:57:15.583 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.583 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.583 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.583 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707624+0000", 2024-09-13T14:57:15.583 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.1490000000000003e-05 2024-09-13T14:57:15.583 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.583 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.584 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.584 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707720+0000", 2024-09-13T14:57:15.584 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.5959999999999996e-05 2024-09-13T14:57:15.584 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.584 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.584 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.584 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.585 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.585 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182126 6.18 6:1d91b7c9:::benchmark_data_smithi096_167087_object64898:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:57:15.585 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583826+0000", 2024-09-13T14:57:15.585 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 597.85642602300004, 2024-09-13T14:57:15.585 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1240857499999999, 2024-09-13T14:57:15.585 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.585 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.586 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.586 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.586 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.586 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.586 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182126 2024-09-13T14:57:15.586 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.586 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.587 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.587 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.587 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583826+0000", 2024-09-13T14:57:15.587 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.587 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.587 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.587 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.587 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583826+0000", 2024-09-13T14:57:15.588 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.588 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.588 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.588 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.588 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583826+0000", 2024-09-13T14:57:15.588 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4900000000000001e-07 2024-09-13T14:57:15.588 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.589 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.589 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.589 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583828+0000", 2024-09-13T14:57:15.589 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6589999999999999e-06 2024-09-13T14:57:15.589 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.589 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.589 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.590 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583834+0000", 2024-09-13T14:57:15.590 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7799999999999997e-06 2024-09-13T14:57:15.590 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.590 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.590 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.590 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583839+0000", 2024-09-13T14:57:15.590 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.887e-06 2024-09-13T14:57:15.591 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.591 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.591 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.591 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584196+0000", 2024-09-13T14:57:15.591 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00035786699999999999 2024-09-13T14:57:15.591 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.591 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.592 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.592 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584204+0000", 2024-09-13T14:57:15.592 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2289999999999997e-06 2024-09-13T14:57:15.592 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.592 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.592 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.592 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.701272+0000", 2024-09-13T14:57:15.593 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1170683269999999 2024-09-13T14:57:15.593 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.593 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.593 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.593 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.701391+0000", 2024-09-13T14:57:15.593 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011928 2024-09-13T14:57:15.593 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.594 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.594 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:57:15.594 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.701768+0000", 2024-09-13T14:57:15.594 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00037707099999999998 2024-09-13T14:57:15.594 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.594 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.595 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.595 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707000+0000", 2024-09-13T14:57:15.595 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0052320379999999996 2024-09-13T14:57:15.595 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.595 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.595 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.595 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707013+0000", 2024-09-13T14:57:15.596 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2853999999999999e-05 2024-09-13T14:57:15.596 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.596 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.596 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.596 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707476+0000", 2024-09-13T14:57:15.596 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000462595 2024-09-13T14:57:15.596 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.597 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.597 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.597 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707518+0000", 2024-09-13T14:57:15.597 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2154000000000003e-05 2024-09-13T14:57:15.597 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.597 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.597 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.598 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707760+0000", 2024-09-13T14:57:15.598 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00024246099999999999 2024-09-13T14:57:15.598 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.598 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.598 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.598 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707802+0000", 2024-09-13T14:57:15.598 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1146999999999998e-05 2024-09-13T14:57:15.599 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.599 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.599 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.599 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707912+0000", 2024-09-13T14:57:15.599 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000110052 2024-09-13T14:57:15.599 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.599 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.599 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.599 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.599 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.600 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182128 6.18 6:1c1d9d13:::benchmark_data_smithi096_167087_object64900:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:57:15.600 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583844+0000", 2024-09-13T14:57:15.600 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 597.85640800900001, 2024-09-13T14:57:15.600 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1297894319999999, 2024-09-13T14:57:15.600 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.600 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.600 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.600 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.600 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.600 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.600 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182128 2024-09-13T14:57:15.601 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.601 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.601 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.601 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.601 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583844+0000", 2024-09-13T14:57:15.601 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.601 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.601 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.601 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.601 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583844+0000", 2024-09-13T14:57:15.601 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.602 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.602 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.602 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.602 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583844+0000", 2024-09-13T14:57:15.602 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6300000000000001e-07 2024-09-13T14:57:15.602 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.602 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.602 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.602 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583850+0000", 2024-09-13T14:57:15.602 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8209999999999998e-06 2024-09-13T14:57:15.602 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.602 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.603 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.603 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583856+0000", 2024-09-13T14:57:15.603 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8309999999999997e-06 2024-09-13T14:57:15.603 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.603 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.603 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.603 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583861+0000", 2024-09-13T14:57:15.603 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9969999999999998e-06 2024-09-13T14:57:15.603 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.603 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.603 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.603 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584231+0000", 2024-09-13T14:57:15.604 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00037027000000000002 2024-09-13T14:57:15.604 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.604 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.604 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.604 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584238+0000", 2024-09-13T14:57:15.604 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.562e-06 2024-09-13T14:57:15.604 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.604 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.604 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.604 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704485+0000", 2024-09-13T14:57:15.604 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.12024762 2024-09-13T14:57:15.605 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.605 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.605 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.605 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704619+0000", 2024-09-13T14:57:15.605 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013364200000000001 2024-09-13T14:57:15.605 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.605 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.605 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:57:15.605 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.704986+0000", 2024-09-13T14:57:15.605 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00036733799999999999 2024-09-13T14:57:15.606 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.606 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.606 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.606 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713182+0000", 2024-09-13T14:57:15.606 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.008195239 2024-09-13T14:57:15.606 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.606 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.606 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.607 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713224+0000", 2024-09-13T14:57:15.607 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2001e-05 2024-09-13T14:57:15.607 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.607 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.607 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.607 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713287+0000", 2024-09-13T14:57:15.607 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3285000000000001e-05 2024-09-13T14:57:15.608 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.608 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.608 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.608 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713391+0000", 2024-09-13T14:57:15.608 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000104104 2024-09-13T14:57:15.609 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:15.611 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.611 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.611 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.611 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713434+0000", 2024-09-13T14:57:15.612 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3541000000000002e-05 2024-09-13T14:57:15.612 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.612 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.612 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.612 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713495+0000", 2024-09-13T14:57:15.612 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0489000000000003e-05 2024-09-13T14:57:15.612 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.613 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.613 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.613 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713633+0000", 2024-09-13T14:57:15.613 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000138329 2024-09-13T14:57:15.613 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.613 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.613 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.614 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.614 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.614 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182157 6.18 6:1a63e408:::benchmark_data_smithi096_167087_object64929:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:57:15.614 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583866+0000", 2024-09-13T14:57:15.614 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 597.856385467, 2024-09-13T14:57:15.614 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1353496999999999, 2024-09-13T14:57:15.614 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.614 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.615 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.615 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.615 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.615 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.615 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182157 2024-09-13T14:57:15.615 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.615 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.615 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.615 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.615 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583866+0000", 2024-09-13T14:57:15.616 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.616 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.616 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.616 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.616 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583866+0000", 2024-09-13T14:57:15.616 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.616 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.616 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.616 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.616 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583867+0000", 2024-09-13T14:57:15.617 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.41e-07 2024-09-13T14:57:15.617 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.617 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.617 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.617 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583868+0000", 2024-09-13T14:57:15.617 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5990000000000001e-06 2024-09-13T14:57:15.617 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.617 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.617 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.617 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583874+0000", 2024-09-13T14:57:15.617 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7579999999999998e-06 2024-09-13T14:57:15.618 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.618 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.618 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.618 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583890+0000", 2024-09-13T14:57:15.618 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6322999999999999e-05 2024-09-13T14:57:15.618 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.618 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.618 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.618 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584264+0000", 2024-09-13T14:57:15.618 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00037345899999999998 2024-09-13T14:57:15.618 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.618 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.619 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.619 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584271+0000", 2024-09-13T14:57:15.619 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6989999999999997e-06 2024-09-13T14:57:15.619 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.619 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.619 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.619 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.707962+0000", 2024-09-13T14:57:15.619 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1236913660000001 2024-09-13T14:57:15.619 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.619 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.619 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.620 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.708079+0000", 2024-09-13T14:57:15.620 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000116842 2024-09-13T14:57:15.620 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.620 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.620 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:57:15.620 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.708445+0000", 2024-09-13T14:57:15.620 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00036596699999999997 2024-09-13T14:57:15.620 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.620 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.620 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.620 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713237+0000", 2024-09-13T14:57:15.621 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0047922049999999999 2024-09-13T14:57:15.621 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.621 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.621 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.621 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713680+0000", 2024-09-13T14:57:15.621 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000443163 2024-09-13T14:57:15.621 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.621 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.621 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.621 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713732+0000", 2024-09-13T14:57:15.622 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2092000000000002e-05 2024-09-13T14:57:15.622 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.622 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.622 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.622 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713822+0000", 2024-09-13T14:57:15.622 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9532999999999998e-05 2024-09-13T14:57:15.622 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.622 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.622 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.622 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.718875+0000", 2024-09-13T14:57:15.623 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0050529379999999999 2024-09-13T14:57:15.623 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.623 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.623 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.623 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719086+0000", 2024-09-13T14:57:15.623 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00021114800000000001 2024-09-13T14:57:15.623 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.623 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.624 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.624 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719216+0000", 2024-09-13T14:57:15.624 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013026699999999999 2024-09-13T14:57:15.624 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.624 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.624 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.624 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.624 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.624 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182208 6.18 6:1bb7076d:::benchmark_data_smithi096_167087_object64980:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:57:15.624 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.583897+0000", 2024-09-13T14:57:15.625 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 597.85635471600006, 2024-09-13T14:57:15.625 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.135635011, 2024-09-13T14:57:15.625 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.625 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.625 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.625 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.625 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.625 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.625 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182208 2024-09-13T14:57:15.625 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.625 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.626 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.626 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.626 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583897+0000", 2024-09-13T14:57:15.626 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.626 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.626 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.626 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.626 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583897+0000", 2024-09-13T14:57:15.626 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.626 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.626 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.627 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.627 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583898+0000", 2024-09-13T14:57:15.627 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6300000000000001e-07 2024-09-13T14:57:15.627 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.627 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.627 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.627 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583899+0000", 2024-09-13T14:57:15.627 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.885e-06 2024-09-13T14:57:15.627 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.627 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.627 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.628 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583905+0000", 2024-09-13T14:57:15.628 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.003e-06 2024-09-13T14:57:15.628 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.628 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.628 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.628 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.583910+0000", 2024-09-13T14:57:15.628 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8919999999999999e-06 2024-09-13T14:57:15.628 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.628 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.628 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.629 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584608+0000", 2024-09-13T14:57:15.629 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00069783300000000003 2024-09-13T14:57:15.629 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.629 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.629 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.629 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.584616+0000", 2024-09-13T14:57:15.629 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.2209999999999997e-06 2024-09-13T14:57:15.629 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.629 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.630 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.630 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.710538+0000", 2024-09-13T14:57:15.630 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.125921197 2024-09-13T14:57:15.630 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.630 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.630 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.630 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.710659+0000", 2024-09-13T14:57:15.630 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000121261 2024-09-13T14:57:15.631 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.631 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.631 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:57:15.631 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.710998+0000", 2024-09-13T14:57:15.631 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00033875300000000001 2024-09-13T14:57:15.631 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.631 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.632 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.632 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713773+0000", 2024-09-13T14:57:15.632 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027755750000000002 2024-09-13T14:57:15.632 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.632 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.632 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.632 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.718788+0000", 2024-09-13T14:57:15.633 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0050151279999999998 2024-09-13T14:57:15.633 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.633 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.633 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.633 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.718829+0000", 2024-09-13T14:57:15.633 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0686000000000003e-05 2024-09-13T14:57:15.633 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.633 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.634 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.634 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719261+0000", 2024-09-13T14:57:15.634 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00043173400000000002 2024-09-13T14:57:15.634 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.634 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.634 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.634 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719336+0000", 2024-09-13T14:57:15.635 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.5703000000000003e-05 2024-09-13T14:57:15.635 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.635 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.635 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.635 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719425+0000", 2024-09-13T14:57:15.635 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.852e-05 2024-09-13T14:57:15.635 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.636 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.636 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.636 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719532+0000", 2024-09-13T14:57:15.636 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010725700000000001 2024-09-13T14:57:15.636 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.636 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.636 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.637 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.637 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.637 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182214 6.18 6:197a4439:::benchmark_data_smithi096_167087_object64986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:57:15.637 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.612524+0000", 2024-09-13T14:57:15.637 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 597.82772738999995, 2024-09-13T14:57:15.637 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.11521797, 2024-09-13T14:57:15.637 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.638 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.638 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.638 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.638 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.638 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.638 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182214 2024-09-13T14:57:15.638 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.639 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.639 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.639 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.639 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612524+0000", 2024-09-13T14:57:15.639 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.639 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.639 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.640 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.640 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612524+0000", 2024-09-13T14:57:15.640 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.640 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.640 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.640 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.640 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612526+0000", 2024-09-13T14:57:15.641 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.347e-06 2024-09-13T14:57:15.641 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.641 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.641 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.641 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612531+0000", 2024-09-13T14:57:15.641 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9799999999999998e-06 2024-09-13T14:57:15.641 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.642 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.642 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.642 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612542+0000", 2024-09-13T14:57:15.642 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0831e-05 2024-09-13T14:57:15.642 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.642 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.642 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.642 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612554+0000", 2024-09-13T14:57:15.643 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2283000000000001e-05 2024-09-13T14:57:15.643 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.643 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.643 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.643 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612613+0000", 2024-09-13T14:57:15.643 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8967e-05 2024-09-13T14:57:15.643 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.644 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.644 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.644 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.612622+0000", 2024-09-13T14:57:15.644 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0399999999999998e-06 2024-09-13T14:57:15.644 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.644 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.644 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.645 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713854+0000", 2024-09-13T14:57:15.645 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.101231611 2024-09-13T14:57:15.645 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.645 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.645 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.645 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.713963+0000", 2024-09-13T14:57:15.645 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010902 2024-09-13T14:57:15.646 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.646 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.646 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:57:15.646 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.714339+0000", 2024-09-13T14:57:15.646 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00037666499999999999 2024-09-13T14:57:15.646 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.646 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.647 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.647 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.718841+0000", 2024-09-13T14:57:15.647 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0045017240000000004 2024-09-13T14:57:15.647 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.647 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.647 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.648 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719298+0000", 2024-09-13T14:57:15.648 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00045695499999999999 2024-09-13T14:57:15.648 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.648 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.648 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.648 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.724722+0000", 2024-09-13T14:57:15.648 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0054238799999999998 2024-09-13T14:57:15.649 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.649 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.649 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.649 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.724759+0000", 2024-09-13T14:57:15.649 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6752999999999999e-05 2024-09-13T14:57:15.649 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.649 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.649 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.650 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727512+0000", 2024-09-13T14:57:15.650 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027530179999999999 2024-09-13T14:57:15.650 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.650 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.650 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.650 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727622+0000", 2024-09-13T14:57:15.650 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011055100000000001 2024-09-13T14:57:15.651 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.651 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.651 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.651 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727742+0000", 2024-09-13T14:57:15.651 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000120345 2024-09-13T14:57:15.651 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.651 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.652 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.652 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.652 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.652 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182224 6.18 6:1a82ea4f:::benchmark_data_smithi096_167087_object64996:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:57:15.652 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.621148+0000", 2024-09-13T14:57:15.652 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 597.81910389699999, 2024-09-13T14:57:15.652 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.115259811, 2024-09-13T14:57:15.653 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.653 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.653 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.653 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.653 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.653 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.653 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182224 2024-09-13T14:57:15.654 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.654 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.654 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.654 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.654 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621148+0000", 2024-09-13T14:57:15.654 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.654 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.655 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.655 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.655 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621148+0000", 2024-09-13T14:57:15.655 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.655 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.655 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.655 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.655 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621149+0000", 2024-09-13T14:57:15.655 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4729999999999999e-06 2024-09-13T14:57:15.655 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.656 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.656 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.656 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621154+0000", 2024-09-13T14:57:15.656 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0010000000000001e-06 2024-09-13T14:57:15.656 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.656 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.656 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.656 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621167+0000", 2024-09-13T14:57:15.656 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2558000000000001e-05 2024-09-13T14:57:15.656 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.656 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.657 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.657 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621174+0000", 2024-09-13T14:57:15.657 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.4390000000000003e-06 2024-09-13T14:57:15.657 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.657 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.657 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.657 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621688+0000", 2024-09-13T14:57:15.657 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00051370099999999998 2024-09-13T14:57:15.657 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.657 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.657 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.658 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.621697+0000", 2024-09-13T14:57:15.658 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0529999999999996e-06 2024-09-13T14:57:15.658 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.658 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.658 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.658 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.716358+0000", 2024-09-13T14:57:15.658 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0946611879999999 2024-09-13T14:57:15.658 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.658 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.658 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.658 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.716475+0000", 2024-09-13T14:57:15.658 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000116692 2024-09-13T14:57:15.659 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.659 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.659 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:57:15.659 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.716866+0000", 2024-09-13T14:57:15.659 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00039121499999999999 2024-09-13T14:57:15.659 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.659 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.659 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.659 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.724682+0000", 2024-09-13T14:57:15.659 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.007815443 2024-09-13T14:57:15.659 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.659 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.660 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.660 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727435+0000", 2024-09-13T14:57:15.660 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027537030000000001 2024-09-13T14:57:15.660 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.660 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.660 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.660 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727474+0000", 2024-09-13T14:57:15.660 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9038000000000001e-05 2024-09-13T14:57:15.660 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.660 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.660 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.661 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727783+0000", 2024-09-13T14:57:15.661 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00030899999999999998 2024-09-13T14:57:15.661 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.661 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.661 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.661 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736174+0000", 2024-09-13T14:57:15.661 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0083904540000000003 2024-09-13T14:57:15.661 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.662 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.662 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.662 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736271+0000", 2024-09-13T14:57:15.662 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7455999999999997e-05 2024-09-13T14:57:15.662 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.662 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.662 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.663 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736408+0000", 2024-09-13T14:57:15.663 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013639699999999999 2024-09-13T14:57:15.663 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.663 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.663 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.663 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.663 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.663 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182244 6.18 6:1bc8bd61:::benchmark_data_smithi096_167087_object65016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:57:15.663 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.649532+0000", 2024-09-13T14:57:15.663 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 597.79071987700002, 2024-09-13T14:57:15.664 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0882464169999999, 2024-09-13T14:57:15.664 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.664 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.664 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.664 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.664 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.664 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.664 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182244 2024-09-13T14:57:15.664 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.664 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.664 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.665 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.665 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649532+0000", 2024-09-13T14:57:15.665 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.665 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.665 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.665 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.665 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649532+0000", 2024-09-13T14:57:15.665 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.666 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.666 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.666 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.666 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649534+0000", 2024-09-13T14:57:15.666 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.643e-06 2024-09-13T14:57:15.666 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.666 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.666 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.667 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649539+0000", 2024-09-13T14:57:15.667 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.835e-06 2024-09-13T14:57:15.667 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.667 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.667 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.667 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649556+0000", 2024-09-13T14:57:15.667 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6881e-05 2024-09-13T14:57:15.668 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.668 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.668 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.668 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649564+0000", 2024-09-13T14:57:15.668 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.8609999999999998e-06 2024-09-13T14:57:15.668 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.668 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.668 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.668 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649620+0000", 2024-09-13T14:57:15.668 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5701000000000001e-05 2024-09-13T14:57:15.669 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.669 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.669 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.669 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.649629+0000", 2024-09-13T14:57:15.669 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.867e-06 2024-09-13T14:57:15.669 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.669 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.669 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.669 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719571+0000", 2024-09-13T14:57:15.669 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0699419960000001 2024-09-13T14:57:15.669 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.669 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.670 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.670 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.719712+0000", 2024-09-13T14:57:15.670 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000140896 2024-09-13T14:57:15.670 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.670 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.670 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:57:15.670 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.720080+0000", 2024-09-13T14:57:15.670 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00036818400000000002 2024-09-13T14:57:15.670 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.670 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.671 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.671 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727822+0000", 2024-09-13T14:57:15.671 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0077419489999999997 2024-09-13T14:57:15.671 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.671 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.671 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.671 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736451+0000", 2024-09-13T14:57:15.671 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0086296340000000006 2024-09-13T14:57:15.671 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.671 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.671 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.672 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736465+0000", 2024-09-13T14:57:15.672 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3386e-05 2024-09-13T14:57:15.672 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.672 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.672 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.672 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737385+0000", 2024-09-13T14:57:15.672 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00092018600000000003 2024-09-13T14:57:15.672 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.672 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.672 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.672 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737579+0000", 2024-09-13T14:57:15.672 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00019378700000000001 2024-09-13T14:57:15.673 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.673 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.673 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.673 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737639+0000", 2024-09-13T14:57:15.673 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0445999999999997e-05 2024-09-13T14:57:15.673 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.673 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.673 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.673 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737778+0000", 2024-09-13T14:57:15.673 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013916500000000001 2024-09-13T14:57:15.674 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.674 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.674 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.674 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.674 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.674 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182259 6.18 6:188e1d88:::benchmark_data_smithi096_167087_object65031:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:57:15.674 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.675788+0000", 2024-09-13T14:57:15.674 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 597.76446410899996, 2024-09-13T14:57:15.674 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0622151870000001, 2024-09-13T14:57:15.674 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.674 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.674 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.675 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.675 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.675 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.675 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182259 2024-09-13T14:57:15.675 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.675 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.675 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.675 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.675 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675788+0000", 2024-09-13T14:57:15.675 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.675 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.676 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.676 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.676 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675788+0000", 2024-09-13T14:57:15.676 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.676 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.676 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.676 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.676 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675789+0000", 2024-09-13T14:57:15.676 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3960000000000001e-06 2024-09-13T14:57:15.676 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.677 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.677 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.677 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675795+0000", 2024-09-13T14:57:15.677 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7359999999999998e-06 2024-09-13T14:57:15.677 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.677 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.677 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.677 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675806+0000", 2024-09-13T14:57:15.677 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0953e-05 2024-09-13T14:57:15.677 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.677 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.677 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.678 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675813+0000", 2024-09-13T14:57:15.678 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7660000000000001e-06 2024-09-13T14:57:15.678 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.678 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.678 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.678 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675869+0000", 2024-09-13T14:57:15.678 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6610000000000002e-05 2024-09-13T14:57:15.678 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.678 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.678 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.678 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.675878+0000", 2024-09-13T14:57:15.679 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6039999999999993e-06 2024-09-13T14:57:15.679 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.679 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.679 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.679 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.722283+0000", 2024-09-13T14:57:15.679 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.046405502 2024-09-13T14:57:15.679 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.679 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.679 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.679 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.722393+0000", 2024-09-13T14:57:15.679 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000109314 2024-09-13T14:57:15.679 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.680 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.680 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:57:15.680 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.722717+0000", 2024-09-13T14:57:15.680 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00032454599999999999 2024-09-13T14:57:15.680 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.680 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.680 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.680 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727862+0000", 2024-09-13T14:57:15.680 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.005144511 2024-09-13T14:57:15.680 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.681 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.681 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.681 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.736481+0000", 2024-09-13T14:57:15.681 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0086192339999999999 2024-09-13T14:57:15.681 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.681 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.681 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.681 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737339+0000", 2024-09-13T14:57:15.681 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00085788100000000001 2024-09-13T14:57:15.681 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.681 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.681 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.682 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737433+0000", 2024-09-13T14:57:15.682 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3827999999999997e-05 2024-09-13T14:57:15.682 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.682 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.682 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.682 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737825+0000", 2024-09-13T14:57:15.682 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00039279099999999997 2024-09-13T14:57:15.682 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.682 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.682 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.682 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737878+0000", 2024-09-13T14:57:15.682 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2431e-05 2024-09-13T14:57:15.683 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.683 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.683 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.683 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.738003+0000", 2024-09-13T14:57:15.683 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000125084 2024-09-13T14:57:15.683 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.683 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.683 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.683 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.683 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.683 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45167.0:182332 6.18 6:1cc35df1:::benchmark_data_smithi096_167087_object65104:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:57:15.683 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:47:17.849479+0000", 2024-09-13T14:57:15.684 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 597.59077249999996, 2024-09-13T14:57:15.684 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.90088822300000004, 2024-09-13T14:57:15.684 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:15.684 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:15.684 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:15.684 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:15.684 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45167", 2024-09-13T14:57:15.684 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/679622833", 2024-09-13T14:57:15.684 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 182332 2024-09-13T14:57:15.685 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.685 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:15.685 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.685 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:15.685 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849479+0000", 2024-09-13T14:57:15.685 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.685 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.685 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.685 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:15.685 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849479+0000", 2024-09-13T14:57:15.686 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:15.686 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.686 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.686 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:15.686 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849481+0000", 2024-09-13T14:57:15.686 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1820000000000001e-06 2024-09-13T14:57:15.686 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.687 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.687 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:15.687 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849485+0000", 2024-09-13T14:57:15.687 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4660000000000001e-06 2024-09-13T14:57:15.687 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.687 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.687 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:15.688 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849495+0000", 2024-09-13T14:57:15.688 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7189999999999996e-06 2024-09-13T14:57:15.688 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.688 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.688 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:15.688 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849500+0000", 2024-09-13T14:57:15.688 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2650000000000004e-06 2024-09-13T14:57:15.688 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.688 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.688 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.688 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849548+0000", 2024-09-13T14:57:15.689 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7787999999999997e-05 2024-09-13T14:57:15.689 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.689 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.689 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:57:15.689 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:17.849564+0000", 2024-09-13T14:57:15.689 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6016e-05 2024-09-13T14:57:15.689 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.689 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.689 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:15.689 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727890+0000", 2024-09-13T14:57:15.689 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.87832629399999995 2024-09-13T14:57:15.690 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.690 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.690 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:15.690 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.727992+0000", 2024-09-13T14:57:15.690 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000101783 2024-09-13T14:57:15.690 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.690 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.690 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:57:15.690 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.728376+0000", 2024-09-13T14:57:15.690 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00038430400000000003 2024-09-13T14:57:15.690 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.691 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.691 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.691 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737484+0000", 2024-09-13T14:57:15.691 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.009108105 2024-09-13T14:57:15.691 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.691 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.691 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.691 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.737529+0000", 2024-09-13T14:57:15.691 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4348999999999998e-05 2024-09-13T14:57:15.691 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.691 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.691 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.692 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.738046+0000", 2024-09-13T14:57:15.692 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00051743500000000001 2024-09-13T14:57:15.692 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.692 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.692 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:15.692 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.750093+0000", 2024-09-13T14:57:15.692 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.012047333 2024-09-13T14:57:15.692 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.692 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.692 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:15.692 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.750184+0000", 2024-09-13T14:57:15.693 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0740000000000002e-05 2024-09-13T14:57:15.693 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.693 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.693 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:15.693 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.750249+0000", 2024-09-13T14:57:15.693 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4417000000000003e-05 2024-09-13T14:57:15.693 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:15.694 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:15.694 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:15.694 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:47:18.750368+0000", 2024-09-13T14:57:15.694 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000119027 2024-09-13T14:57:15.694 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.694 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.694 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.695 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:15.695 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:15.695 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:57:15.716 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:15.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:15.819 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:15.821 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:15 smithi096 ceph-mon[98901]: pgmap v957: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2024-09-13T14:57:15.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:15 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2993501884' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:15.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:15 smithi096 ceph-mon[102928]: pgmap v957: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2024-09-13T14:57:15.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:15 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2993501884' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:15.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.842 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:15.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:15.943 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:15.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.966 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:15.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:15.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:15 smithi204 ceph-mon[87071]: pgmap v957: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2024-09-13T14:57:15.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:15 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2993501884' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:16.066 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:16.088 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:16.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:16.190 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:16.211 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:16.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:16.313 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:16.334 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:16.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:16.436 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:16.463 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:16.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:16.565 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:16.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.587 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:16.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:16.668 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:57:16.688 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:16.714 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:16.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:16.816 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:16.838 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:16.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:16.940 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:16.962 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:16.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:17.064 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:17.087 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:17.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:17.164 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:17.189 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:17.211 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:17.211 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:17.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:17.313 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:17.337 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:17.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:17.438 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:17.460 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:17.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:17.561 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:17.583 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:17.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:17.688 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:17.710 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:17.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:17.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:17 smithi204 ceph-mon[87071]: pgmap v958: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2024-09-13T14:57:17.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:17 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:57:17.813 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:17.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:17 smithi096 ceph-mon[98901]: pgmap v958: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2024-09-13T14:57:17.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:17 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:57:17.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:17 smithi096 ceph-mon[102928]: pgmap v958: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2024-09-13T14:57:17.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:17 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:57:17.834 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:17.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:17.936 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:17.957 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:17.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:18.059 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:18.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.082 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:18.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:18.139 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:18.183 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:18.205 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:18.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:18.265 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:57:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:57:17] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.51.0" 2024-09-13T14:57:18.306 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:18.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.329 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:18.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:18.430 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:18.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.453 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:18.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:18.554 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:18.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.576 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:18.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:18.677 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:18.699 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:18.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:18.801 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:18.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.822 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:18.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:18.924 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:18.945 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:18.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:19.047 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:19.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.069 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:19.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:19.170 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:19.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.192 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:19.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:19.195 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:57:19.293 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:19.314 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:19.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:19.416 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:19.438 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:19.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:19.540 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:19.562 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:19.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:19.664 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:19.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.687 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:19.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:19.789 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:19.810 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:19.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:19.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:19 smithi096 ceph-mon[98901]: pgmap v959: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 453 op/s 2024-09-13T14:57:19.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:19 smithi096 ceph-mon[102928]: pgmap v959: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 453 op/s 2024-09-13T14:57:19.912 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:19.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.938 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:19.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:19.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:19 smithi204 ceph-mon[87071]: pgmap v959: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 453 op/s 2024-09-13T14:57:20.040 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:20.049 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:20.061 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:20.164 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:20.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.186 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:20.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:20.287 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:20.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.313 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:20.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:20.414 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:20.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.437 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:20.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:57:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:57:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:20.495 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T14:57:20.538 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:20.560 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:20.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:20.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:20 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1824471983' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:57:20.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:20 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:57:20.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:20 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1824471983' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:57:20.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:20 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:57:20.662 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:20.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.685 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:20.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:20.724 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:20 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1824471983' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:57:20.724 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:20 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:57:20.787 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:20.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.811 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:20.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:20.912 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:20.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.935 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:20.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:21.037 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:21.058 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:21.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:21.160 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:21.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.186 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:21.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:21.287 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:21.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.310 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:21.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:21.411 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:21.432 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:21.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:21.534 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:21.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.555 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:21.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:21.656 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:21.678 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:21.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:21.705 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T14:57:21.748 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:21 smithi204 ceph-mon[87071]: pgmap v960: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 163 op/s 2024-09-13T14:57:21.748 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:21 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:57:21.748 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:21 smithi204 ceph-mon[87071]: osdmap e298: 8 total, 8 up, 8 in 2024-09-13T14:57:21.748 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:21.748 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:21.748 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:21.766 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:21 smithi096 ceph-mon[98901]: pgmap v960: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 163 op/s 2024-09-13T14:57:21.767 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:21 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:57:21.767 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:21 smithi096 ceph-mon[98901]: osdmap e298: 8 total, 8 up, 8 in 2024-09-13T14:57:21.767 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:21.767 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:21.767 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:21.767 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:21 smithi096 ceph-mon[102928]: pgmap v960: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 163 op/s 2024-09-13T14:57:21.767 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:21 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:57:21.767 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:21 smithi096 ceph-mon[102928]: osdmap e298: 8 total, 8 up, 8 in 2024-09-13T14:57:21.767 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:21.767 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:21.768 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:21.780 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:21.801 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:21.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:21.903 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:21.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.925 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:21.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:22.026 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:22.048 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:22.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:22.150 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:22.171 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:22.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:22.272 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:22.295 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:22.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:22.397 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:22.418 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:22.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:22.520 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:22.548 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:22.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:22.650 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:22.671 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:22.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:22.745 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'unique_pool_3' removed 2024-09-13T14:57:22.746 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:22 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1467842428' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:57:22.746 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:22 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:57:22.746 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:22 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1467842428' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:57:22.746 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:22 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:57:22.773 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:22.795 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:22.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:22.896 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:22.942 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:22.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:22.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:22 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1467842428' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:57:22.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:22 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:57:23.044 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:23.066 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:23.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:23.167 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:23.189 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:23.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:23.291 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:23.313 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:23.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:23.415 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:23.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.437 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:23.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:23.538 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:23.560 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:23.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:23.662 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:23.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.684 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:23.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:23.785 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:23.807 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:23.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:23.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:23 smithi096 ceph-mon[98901]: pgmap v962: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 132 op/s 2024-09-13T14:57:23.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:23 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:57:23.825 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:23 smithi096 ceph-mon[98901]: osdmap e299: 8 total, 8 up, 8 in 2024-09-13T14:57:23.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:23 smithi096 ceph-mon[102928]: pgmap v962: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 132 op/s 2024-09-13T14:57:23.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:23 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:57:23.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:23 smithi096 ceph-mon[102928]: osdmap e299: 8 total, 8 up, 8 in 2024-09-13T14:57:23.908 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:23.910 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:57:23.911 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:57:23.911 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2024-09-13T14:57:23.912 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_4 16 2024-09-13T14:57:23.932 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:23.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:23.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:23 smithi204 ceph-mon[87071]: pgmap v962: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 132 op/s 2024-09-13T14:57:23.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:23 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:57:23.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:23 smithi204 ceph-mon[87071]: osdmap e299: 8 total, 8 up, 8 in 2024-09-13T14:57:24.035 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:24.056 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:24.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:24.158 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:24.179 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:24.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:24.281 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:24.303 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:24.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:24.405 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:24.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.429 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:24.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:24.454 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:24.530 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:24.552 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:24.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:24.654 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:24.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.678 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:24.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:24.779 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:24.801 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:24.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:24.902 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:24.927 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:24.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:25.029 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:25.050 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:25.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:25.151 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:25.172 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:25.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:25.274 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:25.300 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:25.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:25.402 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:25.423 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:25.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:25.525 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:25.546 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:25.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:25.648 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:25.671 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:25.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:25.750 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:25 smithi096 ceph-mon[98901]: pgmap v964: 93 pgs: 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:25.750 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:25 smithi096 ceph-mon[102928]: pgmap v964: 93 pgs: 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:25.773 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:25.799 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:25.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:25.901 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:25.923 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:25.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:25.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:25 smithi204 ceph-mon[87071]: pgmap v964: 93 pgs: 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:26.024 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:26.046 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:26.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:26.148 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:26.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.175 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:26.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:26.276 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:26.302 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:26.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:26.404 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:26.430 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:26.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:26.532 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:26.554 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:26.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:26.655 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:26.681 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:26.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:26.783 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:26.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.806 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:26.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:26.907 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:26.933 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:26.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:26.961 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:27.034 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:27.057 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:27.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:27.158 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:27.182 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:27.284 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:27.310 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:27.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:27.412 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:27.438 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:27.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:27.504 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:57:27.539 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:27.566 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:27.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:27.668 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:27.693 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:27.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:27.730 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:27 smithi096 ceph-mon[98901]: pgmap v965: 93 pgs: 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:27.730 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:27 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4037351217' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:27.730 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:27 smithi096 ceph-mon[102928]: pgmap v965: 93 pgs: 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:27.730 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:27 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4037351217' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:27.794 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:27.820 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:27.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:27.922 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:27.945 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:27.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:27.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:27 smithi204 ceph-mon[87071]: pgmap v965: 93 pgs: 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:27.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:27 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4037351217' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:28.042 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:57:28.043 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:57:28.043 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:57:28.043 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:57:28.043 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:57:28.043 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:57:28.043 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:57:28.043 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:57:28.044 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:57:28.044 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:57:28.044 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:57:28.044 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:57:28.044 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:57:28.045 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:57:28.045 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:57:28.045 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:57:28.045 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:57:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:57:27] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.51.0" 2024-09-13T14:57:28.047 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:28.068 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:28.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:28.170 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:28.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.192 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:28.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:28.293 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:28.314 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:28.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:28.416 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:28.443 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:28.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:28.545 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:28.565 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:28.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:28.667 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:28.688 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:28.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:28.790 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:28.816 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:28.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:28.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:28 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:57:28.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:28 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:57:28.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:28 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4037351217' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:28.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:28 smithi096 ceph-mon[98901]: osdmap e300: 8 total, 8 up, 8 in 2024-09-13T14:57:28.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:28 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3733103892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:28.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:28 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:57:28.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:28 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:57:28.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:28 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4037351217' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:28.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:28 smithi096 ceph-mon[102928]: osdmap e300: 8 total, 8 up, 8 in 2024-09-13T14:57:28.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:28 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3733103892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:28.918 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:28.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.944 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:28.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:28.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:28 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:57:28.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:28 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:57:28.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:28 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4037351217' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:28.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:28 smithi204 ceph-mon[87071]: osdmap e300: 8 total, 8 up, 8 in 2024-09-13T14:57:28.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:28 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3733103892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:29.046 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:29.068 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:29.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:29.170 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:29.196 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:29.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:29.297 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:29.323 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:29.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:29.425 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:29.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.447 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:29.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:29.548 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:29.569 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:29.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:29.602 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:29 smithi096 ceph-mon[98901]: pgmap v967: 93 pgs: 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:29.603 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:29 smithi096 ceph-mon[102928]: pgmap v967: 93 pgs: 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:29.670 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:29.696 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:29.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:29.798 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:29.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.823 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:29.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:29.924 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:29.949 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:29.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:29.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:29 smithi204 ceph-mon[87071]: pgmap v967: 93 pgs: 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:30.050 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:30.074 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:30.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:30.176 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:30.202 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:30.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:30.303 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:30.330 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:30.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:30.431 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:30.453 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:30.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:57:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:57:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:30.555 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:30.566 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:30.573 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:57:30.574 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:57:30.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.581 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:30.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:30.682 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:30.708 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:30.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:30.810 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:30.835 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:30.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:30.937 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:30.960 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:30.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:31.062 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:31.085 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:31.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:31.187 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:31.210 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:31.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:31.312 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:31.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.336 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:31.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:31.393 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:57:31.437 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:31.460 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:31.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:31.527 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T14:57:31.562 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:31.585 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:31.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:31.686 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:31.709 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:31.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:31.804 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:31 smithi204 ceph-mon[87071]: pgmap v968: 93 pgs: 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:31.805 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:31 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1538210626' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:57:31.805 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:31 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:57:31.812 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:31.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:31 smithi096 ceph-mon[102928]: pgmap v968: 93 pgs: 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:31.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:31 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1538210626' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:57:31.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:31 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:57:31.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:31 smithi096 ceph-mon[98901]: pgmap v968: 93 pgs: 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:31.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:31 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1538210626' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:57:31.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:31 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:57:31.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.834 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:31.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:31.935 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:31.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.958 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:31.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:32.059 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:32.081 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:32.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:32.148 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:57:32.183 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:32.212 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:32.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:32.314 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:32.337 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:32.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:32.439 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:32.464 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:32.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:32.565 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:32.587 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:32.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:32.689 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:32.711 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:32.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:32.812 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:32.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:32 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:57:32.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:32 smithi096 ceph-mon[98901]: osdmap e301: 8 total, 8 up, 8 in 2024-09-13T14:57:32.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:32 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:57:32.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:32 smithi096 ceph-mon[102928]: osdmap e301: 8 total, 8 up, 8 in 2024-09-13T14:57:32.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.835 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:32.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:32.927 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T14:57:32.935 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:32.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:32 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:57:32.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:32 smithi204 ceph-mon[87071]: osdmap e301: 8 total, 8 up, 8 in 2024-09-13T14:57:32.991 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:32.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:33.093 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:33.119 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:33.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:33.221 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:33.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.247 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:33.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:33.348 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:33.371 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:33.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:33.473 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:33.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.495 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:33.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:33.538 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'unique_pool_4' created 2024-09-13T14:57:33.597 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:33.620 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:33.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:33.722 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:33.744 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:33.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:33.783 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:33 smithi096 ceph-mon[98901]: pgmap v970: 93 pgs: 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:33.783 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:33 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2153943362' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T14:57:33.783 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:33 smithi096 ceph-mon[102928]: pgmap v970: 93 pgs: 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:33.783 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:33 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2153943362' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T14:57:33.846 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:33.868 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:33.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:33.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:33 smithi204 ceph-mon[87071]: pgmap v970: 93 pgs: 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:33.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:33 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2153943362' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T14:57:33.969 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:33.991 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:33.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:34.093 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:34.114 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:34.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:34.216 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:34.238 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:34.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:34.341 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:34.362 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:34.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:34.464 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:34.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.490 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:34.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:34.560 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:57:34.584 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:34.587 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:34.592 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:34.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.626 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:34.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:34.727 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:34.748 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:34.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:34.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:34 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2153943362' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-13T14:57:34.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:34 smithi096 ceph-mon[98901]: osdmap e302: 8 total, 8 up, 8 in 2024-09-13T14:57:34.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:34 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2153943362' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-13T14:57:34.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:34 smithi096 ceph-mon[102928]: osdmap e302: 8 total, 8 up, 8 in 2024-09-13T14:57:34.850 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:34.872 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:34.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:34.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:34 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2153943362' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-13T14:57:34.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:34 smithi204 ceph-mon[87071]: osdmap e302: 8 total, 8 up, 8 in 2024-09-13T14:57:34.973 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:34.994 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:34.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:35.047 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2024-09-13T14:57:35.096 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:35.125 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:35.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:35.227 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:35.251 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:35.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:35.353 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:35.377 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:35.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:35.479 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:35.501 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:35.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:35.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:35 smithi096 ceph-mon[98901]: pgmap v972: 109 pgs: 7 creating+peering, 9 unknown, 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:35.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:35 smithi096 ceph-mon[98901]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:57:35.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:35 smithi096 ceph-mon[98901]: osdmap e303: 8 total, 8 up, 8 in 2024-09-13T14:57:35.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:35 smithi096 ceph-mon[102928]: pgmap v972: 109 pgs: 7 creating+peering, 9 unknown, 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:35.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:35 smithi096 ceph-mon[102928]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:57:35.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:35 smithi096 ceph-mon[102928]: osdmap e303: 8 total, 8 up, 8 in 2024-09-13T14:57:35.603 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:35.625 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:35.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:35.726 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:35.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.749 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:35.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:35.850 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:35.876 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:35.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:35 smithi204 ceph-mon[87071]: pgmap v972: 109 pgs: 7 creating+peering, 9 unknown, 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:35 smithi204 ceph-mon[87071]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:57:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:35 smithi204 ceph-mon[87071]: osdmap e303: 8 total, 8 up, 8 in 2024-09-13T14:57:35.979 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:36.000 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:36.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:36.102 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:36.128 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:36.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:36.230 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:36.251 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:36.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:36.353 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:36.378 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:36.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:36.480 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:36.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.507 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:36.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:36.608 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:36.630 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:36.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:36.732 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:36.753 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:36.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:36.797 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:36 smithi096 ceph-mon[98901]: osdmap e304: 8 total, 8 up, 8 in 2024-09-13T14:57:36.797 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:36.797 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:36.798 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:36 smithi096 ceph-mon[102928]: osdmap e304: 8 total, 8 up, 8 in 2024-09-13T14:57:36.798 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:36.798 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:36.855 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:36.876 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:36.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:36.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:36 smithi204 ceph-mon[87071]: osdmap e304: 8 total, 8 up, 8 in 2024-09-13T14:57:36.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:36.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:36.978 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:37.004 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:37.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:37.106 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:37.131 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:37.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:37.233 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:37.258 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:37.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:37.360 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:37.381 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:37.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:37.483 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:37.509 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:37.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:37.554 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:37.591 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:37.611 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:37.636 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:37.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:37.737 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:37.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.760 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:37.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:37.861 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:37.886 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:37.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:37.942 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:37 smithi096 ceph-mon[98901]: pgmap v975: 109 pgs: 7 creating+peering, 9 unknown, 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:37.942 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:37 smithi096 ceph-mon[102928]: pgmap v975: 109 pgs: 7 creating+peering, 9 unknown, 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:37.988 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:38.013 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:38.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:38.115 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:38.140 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:38.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:37 smithi204 ceph-mon[87071]: pgmap v975: 109 pgs: 7 creating+peering, 9 unknown, 93 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:38.242 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:38.264 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:38.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:38.275 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:57:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:57:37] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.51.0" 2024-09-13T14:57:38.366 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:38.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.389 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:38.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:38.490 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:38.512 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:38.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:38.614 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:38.634 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:38.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:38.736 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:38.762 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:38.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:38.864 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:38.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.887 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:38.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:38.988 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:39.013 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:39.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:39.115 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:39.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.142 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:39.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:38 smithi204 ceph-mon[87071]: pgmap v976: 109 pgs: 7 creating+peering, 2 unknown, 100 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:39.242 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:39.264 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:39.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:39.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:38 smithi096 ceph-mon[98901]: pgmap v976: 109 pgs: 7 creating+peering, 2 unknown, 100 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:39.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:38 smithi096 ceph-mon[102928]: pgmap v976: 109 pgs: 7 creating+peering, 2 unknown, 100 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:39.366 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:39.388 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:39.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:39.490 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:39.516 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:39.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:39.618 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:39.643 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:39.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:39.745 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:39.770 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:39.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:39.872 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:39.898 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:39.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:40.000 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:40.025 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:39 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3295766979' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:40.026 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:39 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:40.026 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:39 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3295766979' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:40.026 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:39 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:40.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.027 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:40.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:40.128 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:40.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.151 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:40.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:40.236 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:57:40.252 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:40.277 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:40.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:40.379 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:40.401 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:40.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:39 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3295766979' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:39 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:57:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:57:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:40.503 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:40.510 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:40.529 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:40.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:40.632 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:40.657 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:40.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:40.759 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:40.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.782 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:40.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:40.883 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:40.909 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:40.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:41.011 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:41.033 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:41.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:41.135 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:41.161 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:41.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:41.263 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:41.310 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:40 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:41.310 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:40 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:57:41.310 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:40 smithi096 ceph-mon[98901]: osdmap e305: 8 total, 8 up, 8 in 2024-09-13T14:57:41.310 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:40 smithi096 ceph-mon[98901]: pgmap v978: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:41.311 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:40 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:41.311 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:40 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:57:41.311 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:40 smithi096 ceph-mon[102928]: osdmap e305: 8 total, 8 up, 8 in 2024-09-13T14:57:41.311 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:40 smithi096 ceph-mon[102928]: pgmap v978: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:41.333 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:41.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:41.435 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:41.461 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:41.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:41.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:40 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:41.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:40 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:57:41.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:40 smithi204 ceph-mon[87071]: osdmap e305: 8 total, 8 up, 8 in 2024-09-13T14:57:41.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:40 smithi204 ceph-mon[87071]: pgmap v978: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:41.564 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:41.586 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:41.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:41.688 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:41.714 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:41.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:41.815 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:41.837 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:41.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:41.916 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:57:41.917 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:57:41.937 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:57:41.937 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:57:41.940 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:41.955 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:57:41.956 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:57:41.962 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:41.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:41.971 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:57:41.972 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:57:41.988 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:57:41.988 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:57:42.006 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:57:42.006 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:57:42.022 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:57:42.022 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:57:42.038 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:57:42.038 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:57:42.063 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:42.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:41 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/427198221' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:42.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:41 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/427198221' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:42.084 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:42.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:42.186 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:42.207 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:42.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:42.309 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:42.336 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:42.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:42.438 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:42.464 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:42.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:42.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:41 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/427198221' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:42.565 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:42.587 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:42.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:42.689 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:42.709 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:42.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:42.812 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:42.833 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:42.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:42.935 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:42.959 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:42.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:43.061 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:43.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:42 smithi096 ceph-mon[98901]: pgmap v979: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:43.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:42 smithi096 ceph-mon[102928]: pgmap v979: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:43.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.085 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:43.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:43.186 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:43.207 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:43.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:43.309 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:43.334 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:43.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:43.436 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:43.459 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:43.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:43.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:42 smithi204 ceph-mon[87071]: pgmap v979: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:43.560 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:43.581 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:43.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:43.683 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:43.709 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:43.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:43.810 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:43.836 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:43.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:43.937 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:43.958 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:43.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:44.060 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:44.085 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:44.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:44.187 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:44.209 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:44.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:44.311 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:44.332 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:44.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:44.434 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:44.460 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:44.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:44.516 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:44 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:44.517 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:44 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:44.517 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:44 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:57:44.517 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:44 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:57:44.517 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:44 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:44.517 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:44 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:44.517 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:44 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:44.517 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:44 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:57:44.517 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:44 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:57:44.517 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:44 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:44.562 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:44.583 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:44.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:44.685 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:44.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.707 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:44.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:44.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:44 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:44.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:44 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:44.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:44 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:57:44.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:44 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:57:44.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:44 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:44.808 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:44.830 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:44.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:44.932 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:44.954 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:44.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:45.055 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:45.076 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:45.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:45.178 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:45.199 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:45.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:45.206 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:57:45.301 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:45.327 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:45.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:45.403 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:57:45.403 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:57:45.404 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:57:45.404 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:57:45.404 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:57:45.429 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:45.451 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:45.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:45.553 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:45.578 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:45.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:45.680 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:45.705 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:45.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:45.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:45 smithi204 ceph-mon[87071]: pgmap v980: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:45.807 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:45.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:45 smithi096 ceph-mon[98901]: pgmap v980: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:45.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:45 smithi096 ceph-mon[102928]: pgmap v980: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:45.828 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:45.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:45.930 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:45.951 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:45.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:46.053 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:46.074 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:46.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:46.123 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:57:46.177 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:46.203 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:46.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:46.283 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:57:46.305 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:46.330 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:46.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:46.431 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:46.461 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:46.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:46.563 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:46.584 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:46.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:46.686 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:46.712 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:46.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:46.814 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:46.840 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:46.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:46.912 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:46.913 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:46.918 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:46.942 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:46.967 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:46.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:47.069 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:47.090 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:47.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:47.192 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:47.213 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:47.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:47.307 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:47 smithi096 ceph-mon[98901]: pgmap v981: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:47.307 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:47 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:47.308 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:47 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3643378426' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:57:47.308 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:47 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:57:47.308 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:47 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/975657456' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:57:47.308 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:47 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:57:47.308 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:47 smithi096 ceph-mon[102928]: pgmap v981: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:47.308 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:47 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:47.308 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:47 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3643378426' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:57:47.308 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:47 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:57:47.308 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:47 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/975657456' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:57:47.308 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:47 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:57:47.315 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:47.342 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:47.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:47.444 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:47.468 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T14:57:47.468 INFO:teuthology.orchestra.run.smithi096.stderr:enabled application 'rados' on pool 'unique_pool_4' 2024-09-13T14:57:47.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:47 smithi204 ceph-mon[87071]: pgmap v981: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:47.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:47 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:47.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:47 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3643378426' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:57:47.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:47 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:57:47.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:47 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/975657456' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:57:47.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:47 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:57:47.469 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:47.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:47.571 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:47.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.598 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:47.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:47.699 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:47.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.724 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:47.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:47.826 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:47.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.850 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:47.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:47.951 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:47.973 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:47.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:48.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:57:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:57:47] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.51.0" 2024-09-13T14:57:48.075 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:48.096 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:48.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:48.198 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:48.222 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T14:57:48.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.225 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:48.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:48.326 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:48.347 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:48.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:48.449 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:48.476 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:48.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:48.577 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:48.603 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:48.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:48.705 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:48.727 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:48.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:48.812 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:48 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:57:48.812 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:48 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:57:48.812 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:48 smithi096 ceph-mon[102928]: osdmap e306: 8 total, 8 up, 8 in 2024-09-13T14:57:48.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:48 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:57:48.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:48 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:57:48.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:48 smithi096 ceph-mon[98901]: osdmap e306: 8 total, 8 up, 8 in 2024-09-13T14:57:48.830 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:48.855 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:48.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:48.957 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:48.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:48 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:57:48.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:48 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:57:48.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:48 smithi204 ceph-mon[87071]: osdmap e306: 8 total, 8 up, 8 in 2024-09-13T14:57:48.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.980 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:48.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:49.081 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:49.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.102 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:49.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:49.203 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:49.229 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:49.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:49.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:49.352 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:49.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:49.454 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:49.480 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:49.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:49.582 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:49.603 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:49.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:49.705 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:49.731 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:49.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:49.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:49 smithi096 ceph-mon[98901]: pgmap v983: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:49.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:49 smithi096 ceph-mon[98901]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:57:49.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:49 smithi096 ceph-mon[102928]: pgmap v983: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:49.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:49 smithi096 ceph-mon[102928]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:57:49.832 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:49.857 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:49.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:49.959 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:49.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:49 smithi204 ceph-mon[87071]: pgmap v983: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:49.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:49 smithi204 ceph-mon[87071]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:57:49.983 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:49.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:50.085 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.107 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:50.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:50.208 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:50.230 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:50.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:50.332 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:50.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.354 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:50.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:50.455 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:57:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:57:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:50.478 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:50.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:50.531 DEBUG:teuthology.orchestra.run.smithi096:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2024-09-13T14:57:50.534 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:57:50.534 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:57:50.579 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.600 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:50.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:50.606 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:hints = 1 2024-09-13T14:57:50.606 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:57:50.606 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Object prefix: benchmark_data_smithi096_256909 2024-09-13T14:57:50.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:57:50.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:57:50.703 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:50.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.728 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:50.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:50.830 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:50.863 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:50.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:50.965 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:51.001 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:51.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:51.104 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:51.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.141 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:51.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:51.242 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:51.278 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:51.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:51.380 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:51.418 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:51.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:51.521 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:51.555 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:51.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:51.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 1 16 3672 3656 228.487 228.5 0.00241637 0.00436258 2024-09-13T14:57:51.656 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:51.693 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:51.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:51.795 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:51.828 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:51.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:51.930 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:51.968 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:51.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:51 smithi204 ceph-mon[87071]: pgmap v984: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:51.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:51.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:51.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:52.071 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:52.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:51 smithi096 ceph-mon[102928]: pgmap v984: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:52.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:52.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:51 smithi096 ceph-mon[98901]: pgmap v984: 109 pgs: 109 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:57:52.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:57:52.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:52.103 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:52.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:52.206 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:52.240 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:52.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:52.341 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:52.375 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:52.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:52.417 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:57:52.417 INFO:teuthology.orchestra.run.smithi096.stdout: "size": 20, 2024-09-13T14:57:52.417 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 600, 2024-09-13T14:57:52.417 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:57:52.417 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.417 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:177156 8.0 8:053af0ee:::benchmark_data_smithi096_227542_object51761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:57:52.417 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:55:52.950914+0000", 2024-09-13T14:57:52.417 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 119.460685263, 2024-09-13T14:57:52.418 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.074504418000000003, 2024-09-13T14:57:52.418 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.418 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.418 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.418 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.418 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:57:52.418 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:57:52.418 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 177156 2024-09-13T14:57:52.418 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.418 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.418 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.418 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.418 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:55:52.950914+0000", 2024-09-13T14:57:52.419 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.419 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.419 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.419 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.419 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:55:52.950914+0000", 2024-09-13T14:57:52.419 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.419 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.419 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.419 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.419 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:55:52.950916+0000", 2024-09-13T14:57:52.419 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1100000000000001e-06 2024-09-13T14:57:52.419 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.419 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.419 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.420 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:55:52.950923+0000", 2024-09-13T14:57:52.420 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0650000000000004e-06 2024-09-13T14:57:52.420 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.420 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.420 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.420 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:55:52.950941+0000", 2024-09-13T14:57:52.420 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8791999999999999e-05 2024-09-13T14:57:52.420 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.420 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.420 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.420 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:55:52.950950+0000", 2024-09-13T14:57:52.420 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.755e-06 2024-09-13T14:57:52.420 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.420 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.421 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.421 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:55:52.994922+0000", 2024-09-13T14:57:52.421 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.043971458999999997 2024-09-13T14:57:52.421 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.421 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.421 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.421 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:55:52.995077+0000", 2024-09-13T14:57:52.421 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00015533800000000001 2024-09-13T14:57:52.421 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.421 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.421 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:57:52.421 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:55:52.995195+0000", 2024-09-13T14:57:52.421 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011850599999999999 2024-09-13T14:57:52.422 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.422 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.422 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.422 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:55:53.009370+0000", 2024-09-13T14:57:52.422 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.014174954 2024-09-13T14:57:52.422 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.422 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.422 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.422 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:55:53.025114+0000", 2024-09-13T14:57:52.422 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.015743331999999999 2024-09-13T14:57:52.422 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.422 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.422 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.423 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:55:53.025326+0000", 2024-09-13T14:57:52.423 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00021210200000000001 2024-09-13T14:57:52.423 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.423 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.423 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.423 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:55:53.025369+0000", 2024-09-13T14:57:52.423 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3195999999999998e-05 2024-09-13T14:57:52.423 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.423 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.423 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.423 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:55:53.025419+0000", 2024-09-13T14:57:52.423 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9808999999999997e-05 2024-09-13T14:57:52.423 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.423 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.424 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.424 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.424 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.424 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:186446 8.0 8:09efebc4:::benchmark_data_smithi096_227542_object61051:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e291)", 2024-09-13T14:57:52.424 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:02.442026+0000", 2024-09-13T14:57:52.424 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 109.96957357300001, 2024-09-13T14:57:52.424 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.069419496999999997, 2024-09-13T14:57:52.424 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.424 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.424 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.424 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.424 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:57:52.424 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:57:52.424 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 186446 2024-09-13T14:57:52.425 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.425 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.425 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.425 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.425 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.442026+0000", 2024-09-13T14:57:52.425 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.425 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.425 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.425 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.425 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.442026+0000", 2024-09-13T14:57:52.425 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.425 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.425 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.425 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.426 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.442027+0000", 2024-09-13T14:57:52.426 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5600000000000004e-07 2024-09-13T14:57:52.426 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.426 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.426 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.426 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.442031+0000", 2024-09-13T14:57:52.426 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2119999999999997e-06 2024-09-13T14:57:52.426 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.426 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.426 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.426 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.442042+0000", 2024-09-13T14:57:52.426 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0587999999999999e-05 2024-09-13T14:57:52.426 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.426 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.427 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.427 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.442050+0000", 2024-09-13T14:57:52.427 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.2190000000000004e-06 2024-09-13T14:57:52.427 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.427 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.427 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.427 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.451974+0000", 2024-09-13T14:57:52.427 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0099243609999999996 2024-09-13T14:57:52.427 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.427 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.427 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.427 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.452068+0000", 2024-09-13T14:57:52.427 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.4197999999999995e-05 2024-09-13T14:57:52.428 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.428 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.428 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:57:52.428 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.452165+0000", 2024-09-13T14:57:52.428 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.6339e-05 2024-09-13T14:57:52.428 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.428 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.428 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.428 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.456735+0000", 2024-09-13T14:57:52.428 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0045705449999999996 2024-09-13T14:57:52.428 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.428 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.428 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.429 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.511276+0000", 2024-09-13T14:57:52.429 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.054541043999999997 2024-09-13T14:57:52.429 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.429 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.429 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.429 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.511377+0000", 2024-09-13T14:57:52.429 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000100468 2024-09-13T14:57:52.429 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.429 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.429 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.429 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.511415+0000", 2024-09-13T14:57:52.429 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7679000000000003e-05 2024-09-13T14:57:52.429 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.429 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.430 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.430 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.511446+0000", 2024-09-13T14:57:52.430 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.0988000000000002e-05 2024-09-13T14:57:52.430 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.430 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.430 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.430 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.430 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.430 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:186673 8.f 8:f1ed3d7e:::benchmark_data_smithi096_227542_object61278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e291)", 2024-09-13T14:57:52.430 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:02.706326+0000", 2024-09-13T14:57:52.430 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 109.705273484, 2024-09-13T14:57:52.430 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.068650526000000003, 2024-09-13T14:57:52.430 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.431 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.431 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.431 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.431 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:57:52.431 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:57:52.431 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 186673 2024-09-13T14:57:52.431 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.431 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.431 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.431 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.431 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.706326+0000", 2024-09-13T14:57:52.431 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.431 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.432 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.432 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.432 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.706326+0000", 2024-09-13T14:57:52.432 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.432 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.432 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.432 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.432 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.706326+0000", 2024-09-13T14:57:52.432 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.77e-07 2024-09-13T14:57:52.432 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.432 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.432 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.432 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.706328+0000", 2024-09-13T14:57:52.432 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7740000000000001e-06 2024-09-13T14:57:52.433 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.433 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.433 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.433 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.706334+0000", 2024-09-13T14:57:52.433 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5149999999999997e-06 2024-09-13T14:57:52.433 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.433 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.433 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.433 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.706339+0000", 2024-09-13T14:57:52.433 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.169e-06 2024-09-13T14:57:52.433 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.433 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.433 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.433 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.728354+0000", 2024-09-13T14:57:52.434 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.022015495 2024-09-13T14:57:52.434 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.434 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.434 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.434 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.728462+0000", 2024-09-13T14:57:52.434 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000107812 2024-09-13T14:57:52.434 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.434 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.434 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:57:52.434 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.728573+0000", 2024-09-13T14:57:52.434 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000110747 2024-09-13T14:57:52.434 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.434 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.435 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.435 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.736325+0000", 2024-09-13T14:57:52.435 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.007752391 2024-09-13T14:57:52.435 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.435 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.435 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.435 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.748229+0000", 2024-09-13T14:57:52.435 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011904108 2024-09-13T14:57:52.435 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.435 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.435 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.435 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.774816+0000", 2024-09-13T14:57:52.435 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.026586939 2024-09-13T14:57:52.436 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.436 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.436 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.436 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.774929+0000", 2024-09-13T14:57:52.436 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000112403 2024-09-13T14:57:52.436 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.436 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.436 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.436 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:02.774977+0000", 2024-09-13T14:57:52.436 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7796000000000001e-05 2024-09-13T14:57:52.436 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.436 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.436 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.437 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.437 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.437 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:187838 8.0 8:0e8c442b:::benchmark_data_smithi096_227542_object62443:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e291)", 2024-09-13T14:57:52.437 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:03.889540+0000", 2024-09-13T14:57:52.437 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 108.52205963599999, 2024-09-13T14:57:52.437 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.076660289000000006, 2024-09-13T14:57:52.437 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.437 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.437 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.437 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.437 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:57:52.437 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:57:52.437 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 187838 2024-09-13T14:57:52.437 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.438 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.438 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.438 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.438 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:03.889540+0000", 2024-09-13T14:57:52.438 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.438 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.438 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.438 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.438 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:03.889540+0000", 2024-09-13T14:57:52.438 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.438 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.438 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.438 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.438 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:03.889541+0000", 2024-09-13T14:57:52.439 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.333e-06 2024-09-13T14:57:52.439 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.439 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.439 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.439 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:03.889559+0000", 2024-09-13T14:57:52.439 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7849000000000001e-05 2024-09-13T14:57:52.439 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.439 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.439 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.439 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:03.932984+0000", 2024-09-13T14:57:52.439 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.043424732000000001 2024-09-13T14:57:52.439 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.439 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.440 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.440 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:03.932994+0000", 2024-09-13T14:57:52.440 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0419e-05 2024-09-13T14:57:52.440 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.440 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.440 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.440 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:03.937218+0000", 2024-09-13T14:57:52.440 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0042236790000000001 2024-09-13T14:57:52.440 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.440 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.440 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.440 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:03.937508+0000", 2024-09-13T14:57:52.440 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00028957100000000002 2024-09-13T14:57:52.440 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.441 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.441 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:57:52.441 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:03.937864+0000", 2024-09-13T14:57:52.441 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00035663200000000001 2024-09-13T14:57:52.441 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.441 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.441 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.441 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:03.943434+0000", 2024-09-13T14:57:52.441 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0055702360000000001 2024-09-13T14:57:52.441 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.441 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.441 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.441 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:03.953379+0000", 2024-09-13T14:57:52.441 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0099444400000000006 2024-09-13T14:57:52.441 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.442 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.442 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.442 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:03.966117+0000", 2024-09-13T14:57:52.442 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.012738485000000001 2024-09-13T14:57:52.442 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.442 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.442 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.442 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:03.966163+0000", 2024-09-13T14:57:52.442 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5260999999999999e-05 2024-09-13T14:57:52.442 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.442 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.442 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.442 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:03.966200+0000", 2024-09-13T14:57:52.443 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7651999999999998e-05 2024-09-13T14:57:52.443 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.443 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.443 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.443 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.443 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.443 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:204760 8.f 8:f07496dd:::benchmark_data_smithi096_227542_object79365:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e292)", 2024-09-13T14:57:52.443 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:21.503556+0000", 2024-09-13T14:57:52.443 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 90.908044082000004, 2024-09-13T14:57:52.443 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.080922571999999998, 2024-09-13T14:57:52.443 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.443 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.443 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.443 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.444 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:57:52.444 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:57:52.444 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 204760 2024-09-13T14:57:52.444 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.444 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.444 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.444 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.444 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:21.503556+0000", 2024-09-13T14:57:52.444 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.444 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.444 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.444 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.444 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:21.503556+0000", 2024-09-13T14:57:52.444 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.445 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.445 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.445 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.445 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:21.503556+0000", 2024-09-13T14:57:52.445 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.4799999999999997e-07 2024-09-13T14:57:52.445 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.445 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.445 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.446 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:21.503560+0000", 2024-09-13T14:57:52.446 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7890000000000001e-06 2024-09-13T14:57:52.446 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.446 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.446 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.446 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:21.503570+0000", 2024-09-13T14:57:52.446 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.764e-06 2024-09-13T14:57:52.446 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.446 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.446 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.446 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:21.503576+0000", 2024-09-13T14:57:52.447 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8969999999999996e-06 2024-09-13T14:57:52.447 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.447 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.447 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.447 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:21.551599+0000", 2024-09-13T14:57:52.447 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.048022918999999997 2024-09-13T14:57:52.447 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.447 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.447 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.447 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:21.551708+0000", 2024-09-13T14:57:52.447 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000108866 2024-09-13T14:57:52.447 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.447 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.447 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:57:52.448 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:21.551910+0000", 2024-09-13T14:57:52.448 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00020217499999999999 2024-09-13T14:57:52.448 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.448 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.448 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.448 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:21.560044+0000", 2024-09-13T14:57:52.448 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0081346560000000005 2024-09-13T14:57:52.448 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.448 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.448 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.448 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:21.575564+0000", 2024-09-13T14:57:52.448 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.015519431 2024-09-13T14:57:52.448 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.449 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.449 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.449 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:21.583879+0000", 2024-09-13T14:57:52.449 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0083148939999999998 2024-09-13T14:57:52.449 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.449 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.449 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.449 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:21.584066+0000", 2024-09-13T14:57:52.449 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000187357 2024-09-13T14:57:52.449 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.449 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.449 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.449 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:21.584478+0000", 2024-09-13T14:57:52.449 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00041197599999999999 2024-09-13T14:57:52.450 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.450 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.450 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.450 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.450 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.450 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:211538 8.f 8:fa909132:::benchmark_data_smithi096_227542_object86143:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T14:57:52.450 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:28.351285+0000", 2024-09-13T14:57:52.450 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 84.060315005999996, 2024-09-13T14:57:52.450 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.103903042, 2024-09-13T14:57:52.450 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.450 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.450 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.450 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.450 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:57:52.451 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:57:52.451 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 211538 2024-09-13T14:57:52.451 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.451 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.451 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.451 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.451 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.351285+0000", 2024-09-13T14:57:52.451 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.451 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.451 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.451 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.451 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.351285+0000", 2024-09-13T14:57:52.451 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.451 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.452 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.452 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.452 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.351285+0000", 2024-09-13T14:57:52.452 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9999999999999996e-07 2024-09-13T14:57:52.452 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.452 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.452 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.452 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.351289+0000", 2024-09-13T14:57:52.452 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7129999999999999e-06 2024-09-13T14:57:52.452 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.452 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.452 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.452 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.351300+0000", 2024-09-13T14:57:52.452 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0967e-05 2024-09-13T14:57:52.453 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.453 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.453 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.453 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.351307+0000", 2024-09-13T14:57:52.453 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3919999999999997e-06 2024-09-13T14:57:52.453 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.453 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.453 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.453 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.351363+0000", 2024-09-13T14:57:52.453 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5404e-05 2024-09-13T14:57:52.453 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.453 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.453 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.454 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.351444+0000", 2024-09-13T14:57:52.454 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1403000000000006e-05 2024-09-13T14:57:52.454 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.454 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.454 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:57:52.454 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.351526+0000", 2024-09-13T14:57:52.454 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.2157000000000006e-05 2024-09-13T14:57:52.454 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.454 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.454 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.454 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.355013+0000", 2024-09-13T14:57:52.454 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0034869720000000001 2024-09-13T14:57:52.454 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.455 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.455 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.455 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.355289+0000", 2024-09-13T14:57:52.455 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00027542099999999998 2024-09-13T14:57:52.455 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.455 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.455 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.455 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.455085+0000", 2024-09-13T14:57:52.455 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.099796244000000006 2024-09-13T14:57:52.455 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.455 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.455 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.455 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.455141+0000", 2024-09-13T14:57:52.455 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5572000000000003e-05 2024-09-13T14:57:52.456 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.456 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.456 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.456 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.455188+0000", 2024-09-13T14:57:52.456 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6996999999999997e-05 2024-09-13T14:57:52.456 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.456 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.456 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.456 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.457 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.457 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:211541 8.f 8:fe401fa6:::benchmark_data_smithi096_227542_object86146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T14:57:52.457 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:28.355866+0000", 2024-09-13T14:57:52.457 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 84.055734051000002, 2024-09-13T14:57:52.457 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.099864126999999997, 2024-09-13T14:57:52.457 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.457 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.457 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.457 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.457 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:57:52.457 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:57:52.457 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 211541 2024-09-13T14:57:52.457 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.458 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.458 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.458 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.458 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.355866+0000", 2024-09-13T14:57:52.458 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.458 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.458 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.458 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.458 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.355866+0000", 2024-09-13T14:57:52.458 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.458 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.458 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.458 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.458 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.355867+0000", 2024-09-13T14:57:52.458 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.6899999999999996e-07 2024-09-13T14:57:52.459 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.459 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.459 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.459 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.355870+0000", 2024-09-13T14:57:52.459 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9920000000000001e-06 2024-09-13T14:57:52.459 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.459 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.459 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.459 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.355882+0000", 2024-09-13T14:57:52.459 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1576999999999999e-05 2024-09-13T14:57:52.459 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.459 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.459 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.460 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.355889+0000", 2024-09-13T14:57:52.460 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6919999999999996e-06 2024-09-13T14:57:52.460 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.460 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.460 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.460 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.355941+0000", 2024-09-13T14:57:52.460 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2695999999999998e-05 2024-09-13T14:57:52.460 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.460 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.461 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.461 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.356054+0000", 2024-09-13T14:57:52.461 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000112576 2024-09-13T14:57:52.461 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.461 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.461 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:57:52.461 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.356161+0000", 2024-09-13T14:57:52.461 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010703 2024-09-13T14:57:52.461 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.461 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.461 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.461 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.359687+0000", 2024-09-13T14:57:52.461 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0035256979999999999 2024-09-13T14:57:52.461 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.462 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.462 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.462 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.359749+0000", 2024-09-13T14:57:52.462 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1972999999999998e-05 2024-09-13T14:57:52.462 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.462 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.462 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.462 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.455640+0000", 2024-09-13T14:57:52.462 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.095891265000000003 2024-09-13T14:57:52.462 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.462 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.462 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.462 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.455689+0000", 2024-09-13T14:57:52.463 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9456000000000002e-05 2024-09-13T14:57:52.463 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.463 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.463 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.463 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:28.455730+0000", 2024-09-13T14:57:52.463 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0203000000000001e-05 2024-09-13T14:57:52.463 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.463 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.463 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.463 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.463 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.463 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:213512 8.b 8:ddf33072:::benchmark_data_smithi096_227542_object88117:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T14:57:52.463 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:30.378739+0000", 2024-09-13T14:57:52.463 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 82.032861068000003, 2024-09-13T14:57:52.464 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.085176267999999999, 2024-09-13T14:57:52.464 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.464 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.464 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.464 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.464 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:57:52.464 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:57:52.464 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 213512 2024-09-13T14:57:52.464 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.464 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.464 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.464 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.464 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.378739+0000", 2024-09-13T14:57:52.464 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.465 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.465 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.465 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.465 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.378739+0000", 2024-09-13T14:57:52.465 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.465 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.465 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.465 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.465 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.378739+0000", 2024-09-13T14:57:52.465 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.6200000000000006e-07 2024-09-13T14:57:52.465 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.465 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.465 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.466 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.378742+0000", 2024-09-13T14:57:52.466 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.757e-06 2024-09-13T14:57:52.466 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.466 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.466 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.466 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.378750+0000", 2024-09-13T14:57:52.466 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.7239999999999992e-06 2024-09-13T14:57:52.466 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.466 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.466 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.466 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.378755+0000", 2024-09-13T14:57:52.466 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5319999999999997e-06 2024-09-13T14:57:52.466 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.466 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.466 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.467 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.380018+0000", 2024-09-13T14:57:52.467 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0012626810000000001 2024-09-13T14:57:52.467 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.467 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.467 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.467 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.380136+0000", 2024-09-13T14:57:52.467 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011803600000000001 2024-09-13T14:57:52.467 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.467 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.467 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:57:52.467 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.380243+0000", 2024-09-13T14:57:52.467 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000106495 2024-09-13T14:57:52.467 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.468 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.468 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.468 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.391589+0000", 2024-09-13T14:57:52.468 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011346781 2024-09-13T14:57:52.468 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.468 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.468 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.468 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.392025+0000", 2024-09-13T14:57:52.468 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00043508199999999999 2024-09-13T14:57:52.468 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.468 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.468 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.468 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393959+0000", 2024-09-13T14:57:52.468 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0019340589999999999 2024-09-13T14:57:52.469 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.469 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.469 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.469 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.463884+0000", 2024-09-13T14:57:52.469 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.069925680000000004 2024-09-13T14:57:52.469 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.469 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.469 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.469 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.463915+0000", 2024-09-13T14:57:52.469 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.0479000000000001e-05 2024-09-13T14:57:52.469 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.469 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.469 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.469 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.470 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.470 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:213518 8.0 8:077dfe8c:::benchmark_data_smithi096_227542_object88123:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T14:57:52.470 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:30.391438+0000", 2024-09-13T14:57:52.470 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 82.020161213999998, 2024-09-13T14:57:52.470 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.077203315999999994, 2024-09-13T14:57:52.470 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.470 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.470 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.470 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.470 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:57:52.470 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:57:52.470 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 213518 2024-09-13T14:57:52.470 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.471 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.471 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.471 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.471 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.391438+0000", 2024-09-13T14:57:52.471 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.471 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.471 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.471 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.471 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.391438+0000", 2024-09-13T14:57:52.471 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.471 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.471 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.471 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.472 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.391440+0000", 2024-09-13T14:57:52.472 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.328e-06 2024-09-13T14:57:52.472 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.472 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.472 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.472 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.391444+0000", 2024-09-13T14:57:52.472 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9829999999999998e-06 2024-09-13T14:57:52.472 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.472 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.472 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.472 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.391453+0000", 2024-09-13T14:57:52.472 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9169999999999995e-06 2024-09-13T14:57:52.472 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.473 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.473 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.473 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.391459+0000", 2024-09-13T14:57:52.473 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.46e-06 2024-09-13T14:57:52.473 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.473 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.473 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.473 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.391565+0000", 2024-09-13T14:57:52.473 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000105851 2024-09-13T14:57:52.473 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.473 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.473 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.473 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.391702+0000", 2024-09-13T14:57:52.473 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000137315 2024-09-13T14:57:52.473 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.474 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.474 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:57:52.474 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.391822+0000", 2024-09-13T14:57:52.474 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000119429 2024-09-13T14:57:52.474 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.474 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.474 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.474 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.468070+0000", 2024-09-13T14:57:52.474 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.076248694000000006 2024-09-13T14:57:52.474 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.474 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.475 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.475 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.468082+0000", 2024-09-13T14:57:52.475 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1270999999999999e-05 2024-09-13T14:57:52.475 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.475 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.476 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.476 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.468541+0000", 2024-09-13T14:57:52.476 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00045908199999999998 2024-09-13T14:57:52.476 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.476 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.476 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.476 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.468599+0000", 2024-09-13T14:57:52.476 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8298999999999999e-05 2024-09-13T14:57:52.476 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.476 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.476 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.476 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.468642+0000", 2024-09-13T14:57:52.476 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2687e-05 2024-09-13T14:57:52.477 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.477 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.477 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.477 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.477 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.477 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:213520 8.0 8:094ba3f6:::benchmark_data_smithi096_227542_object88125:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T14:57:52.477 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:30.392367+0000", 2024-09-13T14:57:52.477 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 82.019232423000005, 2024-09-13T14:57:52.477 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.088299162, 2024-09-13T14:57:52.477 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.477 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.477 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.477 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.477 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:57:52.478 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:57:52.478 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 213520 2024-09-13T14:57:52.478 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.478 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.478 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.478 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.478 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.392367+0000", 2024-09-13T14:57:52.478 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.478 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.478 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.478 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.478 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.392367+0000", 2024-09-13T14:57:52.478 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.479 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.479 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.479 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.479 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.392368+0000", 2024-09-13T14:57:52.479 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.7800000000000001e-07 2024-09-13T14:57:52.479 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.479 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.479 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.479 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.392371+0000", 2024-09-13T14:57:52.479 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4419999999999998e-06 2024-09-13T14:57:52.479 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.479 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.479 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.480 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.392380+0000", 2024-09-13T14:57:52.480 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5150000000000006e-06 2024-09-13T14:57:52.480 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.480 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.480 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.480 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.392386+0000", 2024-09-13T14:57:52.480 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.545e-06 2024-09-13T14:57:52.480 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.480 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.480 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.480 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.464029+0000", 2024-09-13T14:57:52.480 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.071642384000000003 2024-09-13T14:57:52.480 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.480 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.481 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.481 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.464152+0000", 2024-09-13T14:57:52.481 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000123227 2024-09-13T14:57:52.481 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.481 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.481 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:57:52.481 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.464277+0000", 2024-09-13T14:57:52.481 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012476800000000001 2024-09-13T14:57:52.481 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.481 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.481 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.481 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.478447+0000", 2024-09-13T14:57:52.481 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.014170296000000001 2024-09-13T14:57:52.482 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.482 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.482 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.482 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.478600+0000", 2024-09-13T14:57:52.482 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000152693 2024-09-13T14:57:52.482 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.482 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.482 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.482 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.480572+0000", 2024-09-13T14:57:52.482 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001972157 2024-09-13T14:57:52.482 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.482 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.482 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.482 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.480629+0000", 2024-09-13T14:57:52.483 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.694e-05 2024-09-13T14:57:52.483 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.483 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.483 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.483 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.480666+0000", 2024-09-13T14:57:52.483 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7416999999999998e-05 2024-09-13T14:57:52.483 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.483 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.483 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.483 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.483 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.483 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.45749.0:213525 8.7 e293/270 8:ef8fe264:::benchmark_data_smithi096_227542_object88130:head v 293'12974, mlcod=293'12972)", 2024-09-13T14:57:52.483 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:30.393103+0000", 2024-09-13T14:57:52.484 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 82.018496849000002, 2024-09-13T14:57:52.484 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.070785951999999999, 2024-09-13T14:57:52.484 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.484 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.484 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.484 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.484 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.484 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.484 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393103+0000", 2024-09-13T14:57:52.484 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.484 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.484 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.484 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.484 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393103+0000", 2024-09-13T14:57:52.485 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.485 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.485 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.485 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.485 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393103+0000", 2024-09-13T14:57:52.485 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7800000000000001e-07 2024-09-13T14:57:52.485 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.485 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.485 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.485 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393108+0000", 2024-09-13T14:57:52.485 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9899999999999997e-06 2024-09-13T14:57:52.485 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.485 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.486 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.486 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393118+0000", 2024-09-13T14:57:52.486 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.5510000000000005e-06 2024-09-13T14:57:52.486 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.486 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.486 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.486 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393124+0000", 2024-09-13T14:57:52.486 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6489999999999996e-06 2024-09-13T14:57:52.486 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.486 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.486 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.486 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393184+0000", 2024-09-13T14:57:52.486 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0166e-05 2024-09-13T14:57:52.487 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.487 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.487 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.487 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393201+0000", 2024-09-13T14:57:52.487 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7260000000000001e-05 2024-09-13T14:57:52.487 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.487 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.487 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.487 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.463844+0000", 2024-09-13T14:57:52.487 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.070642542000000003 2024-09-13T14:57:52.487 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.487 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.487 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.487 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.463889+0000", 2024-09-13T14:57:52.488 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5116e-05 2024-09-13T14:57:52.488 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.488 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.488 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.488 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.488 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.488 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:213526 8.0 8:0ba0daa6:::benchmark_data_smithi096_227542_object88131:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T14:57:52.488 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:30.393332+0000", 2024-09-13T14:57:52.488 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 82.018267347000005, 2024-09-13T14:57:52.488 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.087465433999999995, 2024-09-13T14:57:52.488 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.488 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.488 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.489 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.489 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:57:52.489 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:57:52.489 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 213526 2024-09-13T14:57:52.489 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.489 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.489 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.489 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.489 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393332+0000", 2024-09-13T14:57:52.489 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.489 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.489 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.489 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.489 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393332+0000", 2024-09-13T14:57:52.490 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.490 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.490 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.490 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.490 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393333+0000", 2024-09-13T14:57:52.490 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2299999999999999e-07 2024-09-13T14:57:52.490 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.490 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.490 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.490 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393337+0000", 2024-09-13T14:57:52.490 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.8310000000000003e-06 2024-09-13T14:57:52.490 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.490 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.491 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.491 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393346+0000", 2024-09-13T14:57:52.491 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2439999999999999e-06 2024-09-13T14:57:52.491 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.491 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.491 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.491 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393352+0000", 2024-09-13T14:57:52.491 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0900000000000001e-06 2024-09-13T14:57:52.491 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.491 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.492 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.492 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.468671+0000", 2024-09-13T14:57:52.492 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.075318199000000002 2024-09-13T14:57:52.492 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.492 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.492 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.492 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.468822+0000", 2024-09-13T14:57:52.492 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00015187499999999999 2024-09-13T14:57:52.492 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.492 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.492 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:57:52.492 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.468928+0000", 2024-09-13T14:57:52.492 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000105412 2024-09-13T14:57:52.493 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.493 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.493 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.493 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.480504+0000", 2024-09-13T14:57:52.493 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011576565 2024-09-13T14:57:52.493 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.493 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.493 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.493 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.480680+0000", 2024-09-13T14:57:52.493 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00017546400000000001 2024-09-13T14:57:52.493 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.493 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.493 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.494 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.480718+0000", 2024-09-13T14:57:52.494 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.8436999999999998e-05 2024-09-13T14:57:52.494 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.494 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.494 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.494 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.480764+0000", 2024-09-13T14:57:52.494 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.621e-05 2024-09-13T14:57:52.494 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.494 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.494 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.494 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.480798+0000", 2024-09-13T14:57:52.494 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.3183999999999999e-05 2024-09-13T14:57:52.494 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.495 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.495 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.495 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.495 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.495 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.45749.0:213512 8.b e293/270 ondisk, result = 0)", 2024-09-13T14:57:52.495 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:30.393738+0000", 2024-09-13T14:57:52.495 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 82.017861456999995, 2024-09-13T14:57:52.495 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.070182069999999999, 2024-09-13T14:57:52.495 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.495 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.495 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:57:52.495 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.496 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.496 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.496 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393738+0000", 2024-09-13T14:57:52.496 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.496 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.496 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.496 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.496 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393738+0000", 2024-09-13T14:57:52.496 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.496 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.496 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.496 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.496 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393738+0000", 2024-09-13T14:57:52.496 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7300000000000002e-07 2024-09-13T14:57:52.497 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.497 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.497 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.497 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393740+0000", 2024-09-13T14:57:52.497 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8989999999999999e-06 2024-09-13T14:57:52.497 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.497 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.497 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.497 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393746+0000", 2024-09-13T14:57:52.497 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5940000000000001e-06 2024-09-13T14:57:52.497 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.497 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.497 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.497 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393750+0000", 2024-09-13T14:57:52.498 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9550000000000002e-06 2024-09-13T14:57:52.498 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.498 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.498 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.498 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393948+0000", 2024-09-13T14:57:52.498 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00019862499999999999 2024-09-13T14:57:52.498 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.498 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.498 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.498 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.393954+0000", 2024-09-13T14:57:52.498 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5210000000000002e-06 2024-09-13T14:57:52.498 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.498 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.498 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.499 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.463920+0000", 2024-09-13T14:57:52.499 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.069966203000000005 2024-09-13T14:57:52.499 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.499 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.499 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.499 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.499 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.499 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.45749.0:213524 8.c e293/270 8:38d0c6cb:::benchmark_data_smithi096_227542_object88129:head v 293'13357, mlcod=293'13355)", 2024-09-13T14:57:52.499 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:30.394019+0000", 2024-09-13T14:57:52.499 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 82.017580749999993, 2024-09-13T14:57:52.499 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.081986524000000005, 2024-09-13T14:57:52.499 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.499 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.499 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.500 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.500 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.500 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.500 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.394019+0000", 2024-09-13T14:57:52.500 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.500 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.500 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.500 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.500 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.394019+0000", 2024-09-13T14:57:52.500 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.500 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.500 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.500 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.501 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.394019+0000", 2024-09-13T14:57:52.501 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1399999999999997e-07 2024-09-13T14:57:52.501 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.501 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.501 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.501 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.394024+0000", 2024-09-13T14:57:52.501 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.938e-06 2024-09-13T14:57:52.501 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.501 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.501 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.501 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.463805+0000", 2024-09-13T14:57:52.501 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.069780735999999996 2024-09-13T14:57:52.501 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.501 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.502 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.502 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.463821+0000", 2024-09-13T14:57:52.502 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6039e-05 2024-09-13T14:57:52.502 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.502 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.502 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.502 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.464274+0000", 2024-09-13T14:57:52.502 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00045299700000000001 2024-09-13T14:57:52.502 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.502 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.502 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.502 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.464293+0000", 2024-09-13T14:57:52.502 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8703e-05 2024-09-13T14:57:52.502 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.503 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.503 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.503 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.475631+0000", 2024-09-13T14:57:52.503 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011338585 2024-09-13T14:57:52.503 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.503 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.503 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.503 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.476005+0000", 2024-09-13T14:57:52.503 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00037391200000000001 2024-09-13T14:57:52.503 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.503 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.503 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.503 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.503 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.503 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.45749.0:213518 8.0 e293/270 ondisk, result = 0)", 2024-09-13T14:57:52.504 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:30.396084+0000", 2024-09-13T14:57:52.504 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 82.015515962999999, 2024-09-13T14:57:52.504 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.071998856999999999, 2024-09-13T14:57:52.504 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.504 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.504 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:57:52.504 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.504 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.504 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.504 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.396084+0000", 2024-09-13T14:57:52.504 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.504 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.504 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.505 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.505 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.396084+0000", 2024-09-13T14:57:52.505 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.505 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.505 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.505 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.505 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.396084+0000", 2024-09-13T14:57:52.505 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9899999999999999e-07 2024-09-13T14:57:52.505 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.505 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.505 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.505 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.396089+0000", 2024-09-13T14:57:52.505 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6879999999999998e-06 2024-09-13T14:57:52.505 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.506 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.506 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.506 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.463940+0000", 2024-09-13T14:57:52.506 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.067851329000000002 2024-09-13T14:57:52.506 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.506 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.506 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.506 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.463947+0000", 2024-09-13T14:57:52.506 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6470000000000001e-06 2024-09-13T14:57:52.506 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.506 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.506 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.506 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.468058+0000", 2024-09-13T14:57:52.506 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0041106229999999999 2024-09-13T14:57:52.507 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.507 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.507 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.507 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.468065+0000", 2024-09-13T14:57:52.507 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.029e-06 2024-09-13T14:57:52.507 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.507 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.507 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.507 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.468082+0000", 2024-09-13T14:57:52.507 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7742000000000001e-05 2024-09-13T14:57:52.507 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.507 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.507 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.507 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.508 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.508 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:238010 8.b 8:d935f84b:::benchmark_data_smithi096_227542_object112615:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T14:57:52.508 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:56.166071+0000", 2024-09-13T14:57:52.508 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 56.245528307999997, 2024-09-13T14:57:52.508 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.098469616999999995, 2024-09-13T14:57:52.508 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.508 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.508 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.508 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.508 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:57:52.508 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:57:52.508 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 238010 2024-09-13T14:57:52.508 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.509 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.509 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.509 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.509 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.166071+0000", 2024-09-13T14:57:52.509 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.509 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.509 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.509 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.509 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.166071+0000", 2024-09-13T14:57:52.509 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.509 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.509 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.509 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.509 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.166072+0000", 2024-09-13T14:57:52.510 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1379999999999999e-06 2024-09-13T14:57:52.510 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.512 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.512 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.512 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.166076+0000", 2024-09-13T14:57:52.512 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4319999999999999e-06 2024-09-13T14:57:52.512 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.512 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.512 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.512 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.166087+0000", 2024-09-13T14:57:52.512 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.132e-05 2024-09-13T14:57:52.513 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.513 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.513 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.513 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.166102+0000", 2024-09-13T14:57:52.513 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5026e-05 2024-09-13T14:57:52.513 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.513 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.513 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.513 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.172042+0000", 2024-09-13T14:57:52.513 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0059396880000000003 2024-09-13T14:57:52.513 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.513 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.513 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.513 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.172151+0000", 2024-09-13T14:57:52.514 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010865 2024-09-13T14:57:52.514 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.514 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.514 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:57:52.514 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.172258+0000", 2024-09-13T14:57:52.514 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000106983 2024-09-13T14:57:52.514 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.514 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.514 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.514 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.176739+0000", 2024-09-13T14:57:52.514 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0044812100000000002 2024-09-13T14:57:52.514 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.514 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.514 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.515 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.185094+0000", 2024-09-13T14:57:52.515 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0083554430000000006 2024-09-13T14:57:52.515 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.515 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.515 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.515 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.264455+0000", 2024-09-13T14:57:52.515 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.079360985999999994 2024-09-13T14:57:52.515 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.515 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.515 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.515 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.264502+0000", 2024-09-13T14:57:52.515 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7185999999999999e-05 2024-09-13T14:57:52.515 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.516 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.516 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.516 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.264541+0000", 2024-09-13T14:57:52.516 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.8554999999999999e-05 2024-09-13T14:57:52.516 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.516 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.516 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.516 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.516 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.516 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:238020 8.b 8:d8b1731e:::benchmark_data_smithi096_227542_object112625:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T14:57:52.516 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:56.168156+0000", 2024-09-13T14:57:52.516 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 56.243444033999999, 2024-09-13T14:57:52.516 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.108313223, 2024-09-13T14:57:52.517 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.517 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.517 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.517 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.517 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:57:52.517 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:57:52.517 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 238020 2024-09-13T14:57:52.517 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.517 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.517 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.517 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.517 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.168156+0000", 2024-09-13T14:57:52.517 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.517 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.518 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.518 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.518 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.168156+0000", 2024-09-13T14:57:52.518 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.518 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.518 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.518 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.518 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.168157+0000", 2024-09-13T14:57:52.518 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.093e-06 2024-09-13T14:57:52.518 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.518 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.518 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.518 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.168161+0000", 2024-09-13T14:57:52.519 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1110000000000001e-06 2024-09-13T14:57:52.519 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.519 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.519 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.519 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.168173+0000", 2024-09-13T14:57:52.519 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2147000000000001e-05 2024-09-13T14:57:52.519 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.519 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.519 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.519 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.168183+0000", 2024-09-13T14:57:52.519 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0553999999999999e-05 2024-09-13T14:57:52.519 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.519 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.519 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.520 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.181297+0000", 2024-09-13T14:57:52.520 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.013113599 2024-09-13T14:57:52.520 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.520 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.520 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.520 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.181424+0000", 2024-09-13T14:57:52.520 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012678599999999999 2024-09-13T14:57:52.520 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.520 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.520 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:57:52.520 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.181534+0000", 2024-09-13T14:57:52.520 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000110047 2024-09-13T14:57:52.520 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.520 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.521 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:52.522 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.522 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.185459+0000", 2024-09-13T14:57:52.522 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.003925591 2024-09-13T14:57:52.522 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.522 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.522 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.522 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.189254+0000", 2024-09-13T14:57:52.522 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0037941279999999999 2024-09-13T14:57:52.523 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.523 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.523 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.523 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.276384+0000", 2024-09-13T14:57:52.523 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.087130330000000006 2024-09-13T14:57:52.523 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.523 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.523 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.523 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.276433+0000", 2024-09-13T14:57:52.523 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8754000000000001e-05 2024-09-13T14:57:52.523 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.523 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.523 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.523 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.276469+0000", 2024-09-13T14:57:52.524 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6083e-05 2024-09-13T14:57:52.524 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.524 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.524 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.524 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.524 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.524 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:238039 8.b 8:d7a254aa:::benchmark_data_smithi096_227542_object112644:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T14:57:52.524 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:56.192357+0000", 2024-09-13T14:57:52.524 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 56.219242764000001, 2024-09-13T14:57:52.524 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.084226101999999997, 2024-09-13T14:57:52.524 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.524 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.524 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.525 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.525 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:57:52.525 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:57:52.525 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 238039 2024-09-13T14:57:52.525 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.525 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.525 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.525 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.525 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.192357+0000", 2024-09-13T14:57:52.525 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.525 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.525 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.525 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.525 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.192357+0000", 2024-09-13T14:57:52.526 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.526 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.526 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.526 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.526 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.192359+0000", 2024-09-13T14:57:52.526 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.86e-06 2024-09-13T14:57:52.526 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.526 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.526 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.526 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.192363+0000", 2024-09-13T14:57:52.526 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7729999999999999e-06 2024-09-13T14:57:52.526 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.526 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.526 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.527 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.192376+0000", 2024-09-13T14:57:52.527 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2234e-05 2024-09-13T14:57:52.527 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.527 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.527 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.527 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.192383+0000", 2024-09-13T14:57:52.527 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3730000000000004e-06 2024-09-13T14:57:52.527 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.527 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.527 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.527 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.192446+0000", 2024-09-13T14:57:52.527 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2675999999999994e-05 2024-09-13T14:57:52.527 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.527 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.528 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.528 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.192568+0000", 2024-09-13T14:57:52.528 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012210199999999999 2024-09-13T14:57:52.528 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.528 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.528 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:57:52.528 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.192683+0000", 2024-09-13T14:57:52.528 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000115523 2024-09-13T14:57:52.528 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.528 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.528 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.528 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.195682+0000", 2024-09-13T14:57:52.528 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.002998279 2024-09-13T14:57:52.528 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.529 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.529 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.529 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.197172+0000", 2024-09-13T14:57:52.529 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00149015 2024-09-13T14:57:52.529 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.529 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.529 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.529 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.276502+0000", 2024-09-13T14:57:52.529 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.079330028999999996 2024-09-13T14:57:52.529 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.529 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.529 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.529 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.276546+0000", 2024-09-13T14:57:52.529 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4302999999999999e-05 2024-09-13T14:57:52.530 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.530 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.530 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.530 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.276583+0000", 2024-09-13T14:57:52.530 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.68e-05 2024-09-13T14:57:52.530 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.530 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.530 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.530 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.530 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.530 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:239218 8.b 8:d782cbd9:::benchmark_data_smithi096_227542_object113823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T14:57:52.530 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:57.669444+0000", 2024-09-13T14:57:52.530 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 54.742155697999998, 2024-09-13T14:57:52.530 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.070935152000000001, 2024-09-13T14:57:52.530 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.531 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.531 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.531 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.531 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:57:52.531 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:57:52.531 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 239218 2024-09-13T14:57:52.531 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.531 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.531 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.531 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.531 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.669444+0000", 2024-09-13T14:57:52.531 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.531 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.531 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.532 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.532 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.669444+0000", 2024-09-13T14:57:52.532 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.532 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.532 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.532 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.532 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.669445+0000", 2024-09-13T14:57:52.532 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.232e-06 2024-09-13T14:57:52.532 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.532 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.532 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.532 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.669449+0000", 2024-09-13T14:57:52.532 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7589999999999999e-06 2024-09-13T14:57:52.532 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.533 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.533 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.533 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.669458+0000", 2024-09-13T14:57:52.533 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3700000000000001e-06 2024-09-13T14:57:52.533 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.533 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.533 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.533 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.669464+0000", 2024-09-13T14:57:52.533 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3770000000000004e-06 2024-09-13T14:57:52.533 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.533 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.533 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.533 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.669507+0000", 2024-09-13T14:57:52.533 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3432e-05 2024-09-13T14:57:52.534 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.534 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.534 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.534 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.669585+0000", 2024-09-13T14:57:52.534 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.7861999999999994e-05 2024-09-13T14:57:52.534 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.534 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.534 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:57:52.534 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.669662+0000", 2024-09-13T14:57:52.534 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.7071999999999997e-05 2024-09-13T14:57:52.534 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.534 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.534 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.534 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.672562+0000", 2024-09-13T14:57:52.535 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0028995729999999999 2024-09-13T14:57:52.535 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.535 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.535 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.535 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.673850+0000", 2024-09-13T14:57:52.535 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001288584 2024-09-13T14:57:52.535 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.535 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.535 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.535 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.740256+0000", 2024-09-13T14:57:52.535 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.066406270000000003 2024-09-13T14:57:52.535 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.535 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.535 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.536 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.740322+0000", 2024-09-13T14:57:52.536 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5643999999999997e-05 2024-09-13T14:57:52.536 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.536 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.536 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.536 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.740379+0000", 2024-09-13T14:57:52.536 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6977e-05 2024-09-13T14:57:52.536 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.536 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.536 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.536 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.536 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.536 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.46040.0:3708 9.2 9:49c73ec9:::benchmark_data_smithi096_256909_object3707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2024-09-13T14:57:52.536 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:57:51.616213+0000", 2024-09-13T14:57:52.537 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.79538616200000001, 2024-09-13T14:57:52.537 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.012693688999999999, 2024-09-13T14:57:52.537 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:57:52.537 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:57:52.537 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:52.537 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:57:52.537 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.46040", 2024-09-13T14:57:52.537 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:57:52.537 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 3708 2024-09-13T14:57:52.537 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.537 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:57:52.537 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.537 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:57:52.537 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:57:51.616213+0000", 2024-09-13T14:57:52.538 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.538 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.538 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.538 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:57:52.538 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:57:51.616213+0000", 2024-09-13T14:57:52.538 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:57:52.538 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.538 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.538 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:57:52.538 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:57:51.616214+0000", 2024-09-13T14:57:52.538 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0999999999999997e-07 2024-09-13T14:57:52.538 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.538 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.538 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:57:52.539 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:57:51.616239+0000", 2024-09-13T14:57:52.539 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4986000000000001e-05 2024-09-13T14:57:52.539 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.539 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.539 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:57:52.539 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:57:51.616252+0000", 2024-09-13T14:57:52.539 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.235e-05 2024-09-13T14:57:52.539 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.539 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.539 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:57:52.539 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:57:51.616259+0000", 2024-09-13T14:57:52.539 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.08e-06 2024-09-13T14:57:52.539 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.539 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.540 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:57:52.540 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:57:51.617492+0000", 2024-09-13T14:57:52.540 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0012334710000000001 2024-09-13T14:57:52.540 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.540 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.540 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:57:52.540 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:57:51.617634+0000", 2024-09-13T14:57:52.540 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00014163000000000001 2024-09-13T14:57:52.540 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.540 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.541 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:57:52.541 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:57:51.618387+0000", 2024-09-13T14:57:52.541 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00075276399999999995 2024-09-13T14:57:52.541 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.541 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.541 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:57:52.541 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:57:51.623861+0000", 2024-09-13T14:57:52.541 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0054742569999999997 2024-09-13T14:57:52.541 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.541 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.541 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.541 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:57:51.624026+0000", 2024-09-13T14:57:52.541 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00016488700000000001 2024-09-13T14:57:52.542 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.542 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.542 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:52.542 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:57:51.628803+0000", 2024-09-13T14:57:52.542 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0047777030000000003 2024-09-13T14:57:52.542 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.542 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.542 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:57:52.542 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:57:51.628873+0000", 2024-09-13T14:57:52.542 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9621000000000003e-05 2024-09-13T14:57:52.542 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:57:52.542 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:57:52.542 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:57:52.543 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:57:51.628907+0000", 2024-09-13T14:57:52.543 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4029999999999998e-05 2024-09-13T14:57:52.543 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.543 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.543 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.543 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:57:52.543 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:57:52.543 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:57:52.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.557 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:52.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:52.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 2 16 7054 7038 219.922 211.375 0.00307476 0.00453713 2024-09-13T14:57:52.658 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:52.698 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:52.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:52.799 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:52.833 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:52.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:52.935 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:52.966 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:52.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:53.068 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:53.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.105 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:53.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:53.206 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:53.239 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:53.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.240 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:53.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:53.341 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:53.374 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:53.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:53.476 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:53.514 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:53.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:53.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 3 16 10156 10140 211.234 193.875 0.00341855 0.00472708 2024-09-13T14:57:53.615 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:53.646 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:53.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:53.749 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:53.786 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:53.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:53.872 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:57:53.889 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:53.927 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:53.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:54.015 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:57:54.029 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:54.064 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:54.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:54.135 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:53 smithi204 ceph-mon[87071]: pgmap v985: 109 pgs: 109 active+clean; 64 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 5.9 MiB/s wr, 94 op/s 2024-09-13T14:57:54.135 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:53 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3722107861' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:54.135 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:53 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:54.167 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:54.204 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:54.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:54.307 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:54.314 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:53 smithi096 ceph-mon[98901]: pgmap v985: 109 pgs: 109 active+clean; 64 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 5.9 MiB/s wr, 94 op/s 2024-09-13T14:57:54.314 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:53 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3722107861' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:54.315 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:53 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:54.315 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:53 smithi096 ceph-mon[102928]: pgmap v985: 109 pgs: 109 active+clean; 64 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 5.9 MiB/s wr, 94 op/s 2024-09-13T14:57:54.315 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:53 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3722107861' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:54.315 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:53 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:54.343 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:54.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:54.445 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:54.482 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:54.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:54.527 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:57:54.584 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:54.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 4 15 13320 13305 207.87 197.812 0.00453612 0.00480357 2024-09-13T14:57:54.620 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:54.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:54.722 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:54.757 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:54.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:54.859 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:54.896 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:54.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:54.954 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:54 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:54.954 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:54 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:54.954 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:54 smithi204 ceph-mon[87071]: osdmap e307: 8 total, 8 up, 8 in 2024-09-13T14:57:54.954 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:54 smithi204 ceph-mon[87071]: pgmap v987: 109 pgs: 109 active+clean; 530 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:57:54.998 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:55.034 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:55.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:54 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:54 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:54 smithi096 ceph-mon[98901]: osdmap e307: 8 total, 8 up, 8 in 2024-09-13T14:57:55.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:54 smithi096 ceph-mon[98901]: pgmap v987: 109 pgs: 109 active+clean; 530 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:57:55.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:54 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:55.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:54 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:55.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:54 smithi096 ceph-mon[102928]: osdmap e307: 8 total, 8 up, 8 in 2024-09-13T14:57:55.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:54 smithi096 ceph-mon[102928]: pgmap v987: 109 pgs: 109 active+clean; 530 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:57:55.137 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:55.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.178 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:55.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:55.279 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:55.314 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:55.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:55.416 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:55.452 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:55.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:55.554 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:55.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.592 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:55.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:55.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 5 15 16284 16269 203.343 185.25 0.00453213 0.00491142 2024-09-13T14:57:55.693 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:55.726 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:55.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:55.828 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:55.864 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:55.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:55.966 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:56.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.004 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:56.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:56.106 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:56.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.140 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:56.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:56.241 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:56.277 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:56.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:56.379 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:56.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.420 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:56.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:56.521 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:56.559 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:56.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:56.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 6 16 18940 18924 197.106 165.938 0.00466628 0.0050678 2024-09-13T14:57:56.661 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:56.696 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:56.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:56.800 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:56.842 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:56.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:56.944 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:56.970 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:56.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:57.072 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:57.108 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:57.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:57.146 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:57:57.146 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:57:57.146 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:57:57.147 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.46040.0:20149 9.c e307/302 9:33430218:::benchmark_data_smithi096_256909_object20148:head v 307'1275, mlcod=307'1272)", 2024-09-13T14:57:57.147 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:57:57.143585+0000", 2024-09-13T14:57:57.147 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.00076772899999999998, 2024-09-13T14:57:57.147 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00079390000000000005, 2024-09-13T14:57:57.147 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:57:57.147 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:57:57.147 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:57:57.147 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:57:57.147 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:57:57.147 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.143585+0000", 2024-09-13T14:57:57.147 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:57:57.147 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:57:57.147 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:57:57.148 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:57:57.148 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.143585+0000", 2024-09-13T14:57:57.148 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:57:57.148 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:57:57.148 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:57:57.148 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:57:57.148 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.143584+0000", 2024-09-13T14:57:57.148 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:57:57.148 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:57:57.148 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:57:57.148 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:57:57.148 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.143719+0000", 2024-09-13T14:57:57.148 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013535399999999999 2024-09-13T14:57:57.148 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:57:57.149 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:57:57.149 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:57:57.149 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.143746+0000", 2024-09-13T14:57:57.149 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.6064999999999999e-05 2024-09-13T14:57:57.149 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:57:57.149 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:57:57.149 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:57:57.149 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.143752+0000", 2024-09-13T14:57:57.149 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0399999999999998e-06 2024-09-13T14:57:57.149 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:57:57.149 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:57:57.149 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:57:57.149 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.143800+0000", 2024-09-13T14:57:57.149 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8687999999999999e-05 2024-09-13T14:57:57.150 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:57:57.150 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:57:57.150 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:57:57.150 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.143825+0000", 2024-09-13T14:57:57.150 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4593e-05 2024-09-13T14:57:57.150 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:57:57.150 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:57:57.150 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:57:57.150 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:57:57.150 INFO:teuthology.orchestra.run.smithi204.stdout: ], 2024-09-13T14:57:57.150 INFO:teuthology.orchestra.run.smithi204.stdout: "num_ops": 1 2024-09-13T14:57:57.150 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:57:57.210 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:57.245 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:57.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:57.346 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:57.386 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:57.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:57.456 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:57 smithi096 ceph-mon[98901]: pgmap v988: 109 pgs: 109 active+clean; 530 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2024-09-13T14:57:57.456 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:57 smithi096 ceph-mon[102928]: pgmap v988: 109 pgs: 109 active+clean; 530 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2024-09-13T14:57:57.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:57 smithi204 ceph-mon[87071]: pgmap v988: 109 pgs: 109 active+clean; 530 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2024-09-13T14:57:57.488 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:57.510 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:57:57.510 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:57:57.515 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:57.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:57.544 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:57:57.544 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:57:57.564 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:57:57.564 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:57:57.580 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:57:57.580 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:57:57.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 7 16 21170 21154 188.857 139.375 0.00161019 0.00524351 2024-09-13T14:57:57.617 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:57.638 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:57.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:57.672 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:57:57.673 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:57:57.703 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:57:57.704 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:57:57.728 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:57:57.728 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:57:57.742 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:57.743 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:57:57.744 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:57:57.765 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:57.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:57.867 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:57.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.901 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:57.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:58.003 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:58.040 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:58.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:57:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:57:57] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.51.0" 2024-09-13T14:57:58.142 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:58.174 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:58.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:58.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:58 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/492390376' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:58.276 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:58.307 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:58.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:58.409 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:58.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.444 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:58.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:58.545 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:58.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:58 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/492390376' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:58.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:58 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/492390376' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:58.579 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:58.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:58.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 8 16 23328 23312 182.108 134.875 0.00667667 0.00548556 2024-09-13T14:57:58.681 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:58.716 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:58.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:58.818 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:58.853 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:58.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:58.955 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:58.977 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:58.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:59.079 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:59.110 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:59.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:59.212 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:59.245 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:59.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:59.305 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:57:59 smithi096 ceph-mon[98901]: pgmap v989: 109 pgs: 109 active+clean; 936 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-13T14:57:59.306 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:57:59 smithi096 ceph-mon[102928]: pgmap v989: 109 pgs: 109 active+clean; 936 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-13T14:57:59.347 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:59.382 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:59.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:59.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:57:59 smithi204 ceph-mon[87071]: pgmap v989: 109 pgs: 109 active+clean; 936 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-13T14:57:59.484 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:59.517 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:59.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:59.607 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 9 16 25849 25833 179.379 157.562 0.00761254 0.00556962 2024-09-13T14:57:59.619 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:59.624 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:57:59.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.651 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:59.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:59.753 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:59.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.792 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:57:59.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:59.893 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:59.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.928 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:57:59.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:00.029 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:00.063 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:00.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:00.078 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:00.160 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T14:58:00.166 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:00.198 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:00.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:00.254 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:58:00.300 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:00.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.342 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:00.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:00.443 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:00 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3245684135' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:58:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:00 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:58:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:58:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:58:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:00.477 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:00.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:00.529 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:00 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3245684135' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:58:00.529 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:00 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:58:00.529 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:00 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3245684135' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:58:00.530 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:00 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:58:00.579 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:00.608 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 10 16 28179 28163 175.999 145.625 0.00477635 0.0056759 2024-09-13T14:58:00.610 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:00.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:00.712 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:00.743 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:00.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:00.845 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:00.881 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:00.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:00.983 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:01.020 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:01.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:01.081 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:58:01.122 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:01.146 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:01.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:01.248 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:01.283 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:01.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:01.385 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:01.421 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:01.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:01 smithi204 ceph-mon[87071]: pgmap v990: 109 pgs: 109 active+clean; 1.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-13T14:58:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:01 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:58:01.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:01 smithi204 ceph-mon[87071]: osdmap e308: 8 total, 8 up, 8 in 2024-09-13T14:58:01.523 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:01.559 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:01.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:01.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:01 smithi096 ceph-mon[98901]: pgmap v990: 109 pgs: 109 active+clean; 1.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-13T14:58:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:01 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:58:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:01 smithi096 ceph-mon[98901]: osdmap e308: 8 total, 8 up, 8 in 2024-09-13T14:58:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:01 smithi096 ceph-mon[102928]: pgmap v990: 109 pgs: 109 active+clean; 1.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-13T14:58:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:01 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:58:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:01 smithi096 ceph-mon[102928]: osdmap e308: 8 total, 8 up, 8 in 2024-09-13T14:58:01.608 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 11 16 30112 30096 170.981 120.812 0.00597486 0.00584297 2024-09-13T14:58:01.661 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:01.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.687 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:01.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:01.788 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:01.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.827 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:01.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:01.928 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:01.957 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:01.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:02.060 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:02.095 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:02.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:02.196 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:02.232 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:02.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:02.334 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:02.364 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:02.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:02.466 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:02.487 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:02.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:02.588 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:02.608 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 12 16 31723 31707 165.123 100.688 0.00784507 0.00605107 2024-09-13T14:58:02.621 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:02.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:02.723 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:02.760 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:02.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:02.862 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:02.883 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:02.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:02.985 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:03.015 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:03.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:03.117 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:03.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.154 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:03.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:03 smithi204 ceph-mon[87071]: pgmap v992: 109 pgs: 109 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s 2024-09-13T14:58:03.255 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:03.291 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:03.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:03.393 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:03.428 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:03.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:03.529 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:03.568 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:03.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:03.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:03 smithi096 ceph-mon[98901]: pgmap v992: 109 pgs: 109 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s 2024-09-13T14:58:03.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:03 smithi096 ceph-mon[102928]: pgmap v992: 109 pgs: 109 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s 2024-09-13T14:58:03.608 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 13 16 33703 33687 161.939 123.75 0.00655527 0.00617062 2024-09-13T14:58:03.670 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:03.709 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:03.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:03.811 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:03.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.833 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:03.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:03.934 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:03.970 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:03.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:04.072 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:04.105 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:04.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:04.207 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:04.256 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:04.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:04.274 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:58:04.275 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:58:04.358 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:04.394 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:04.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:04.496 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:04.528 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:04.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:04.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 14 16 35613 35597 158.898 119.375 0.00518922 0.00628775 2024-09-13T14:58:04.630 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:04.661 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:04.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:04.762 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:04.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.785 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:04.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:04.886 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:04.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.916 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:04.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:04.982 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:58:05.017 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:05.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.041 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:05.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:05.143 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:05.161 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:05 smithi204 ceph-mon[87071]: pgmap v993: 109 pgs: 109 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-09-13T14:58:05.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.187 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:05.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:05.288 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:05.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.323 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:05.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:05.334 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:05.425 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:05.461 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:05.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:05.562 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:05.567 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:05 smithi096 ceph-mon[98901]: pgmap v993: 109 pgs: 109 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-09-13T14:58:05.568 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:05 smithi096 ceph-mon[102928]: pgmap v993: 109 pgs: 109 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-09-13T14:58:05.609 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:05.610 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 15 16 37083 37067 154.43 91.875 0.0158488 0.00647013 2024-09-13T14:58:05.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:05.614 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T14:58:05.660 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:05.679 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:05.712 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:05.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.734 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:05.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:05.835 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:05.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.881 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:05.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:05.982 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:06.021 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:06.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:06.123 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:06.145 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:06.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:06.246 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:06.284 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:06.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:06.386 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:06.421 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:06.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:06.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:06.523 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:06.560 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:06.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:06.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:06.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:06.608 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 16 16 38774 38758 151.383 105.688 0.00563721 0.00660087 2024-09-13T14:58:06.662 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:06.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.702 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:06.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:06.803 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:06.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.842 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:06.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:06.944 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:06.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.984 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:06.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:07.085 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:07.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.124 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:07.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:07.225 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:07.251 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:58:07.252 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:58:07.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.253 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:07.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:07.354 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:07.389 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:07.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:07.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:07 smithi204 ceph-mon[87071]: pgmap v994: 109 pgs: 109 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-09-13T14:58:07.491 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:07.527 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:07.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:07.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:07 smithi096 ceph-mon[98901]: pgmap v994: 109 pgs: 109 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-09-13T14:58:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:07 smithi096 ceph-mon[102928]: pgmap v994: 109 pgs: 109 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-09-13T14:58:07.608 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 17 16 40570 40554 149.08 112.25 0.0065353 0.00670231 2024-09-13T14:58:07.628 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:07.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.654 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:07.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:07.755 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:07.790 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:07.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:07.892 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:07.916 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:07.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:08.018 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:08.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.053 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:08.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:08.154 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:08.187 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:08.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:08.199 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:58:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:58:07] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-13T14:58:08.292 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:08.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.328 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:08.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:08.429 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:08.460 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:08.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:08.562 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:08.597 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:08.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:08.608 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 18 16 42252 42236 146.638 105.125 0.00205212 0.00681156 2024-09-13T14:58:08.699 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:08.733 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:08.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:08.835 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:08.859 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:08.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:08.961 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:08.998 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:08.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:09.006 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:58:09.100 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:09.148 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:09.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:09.250 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:09.285 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:09.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:09.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:09 smithi096 ceph-mon[98901]: pgmap v995: 109 pgs: 109 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 135 MiB/s wr, 2.15k op/s 2024-09-13T14:58:09.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:09 smithi096 ceph-mon[102928]: pgmap v995: 109 pgs: 109 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 135 MiB/s wr, 2.15k op/s 2024-09-13T14:58:09.386 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:09.424 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:09.434 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:09.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:09 smithi204 ceph-mon[87071]: pgmap v995: 109 pgs: 109 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 135 MiB/s wr, 2.15k op/s 2024-09-13T14:58:09.526 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:09.560 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:09.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:09.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 19 16 43997 43981 144.66 109.062 0.0116219 0.00690753 2024-09-13T14:58:09.663 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:09.701 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:09.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:09.803 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:09.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.839 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:09.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:09.940 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:09.961 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:09.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:10.063 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:10.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.087 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:10.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:10.188 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:10.225 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:10.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:10.327 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:10.362 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:10.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:10.463 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:58:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:58:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:10.497 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:10.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:10.599 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:10.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:58:10.608559+0000 min lat: 0.00112813 max lat: 0.278402 avg lat: 0.00704937 2024-09-13T14:58:10.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:58:10.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 20 16 45381 45365 141.752 86.5 0.00672848 0.00704937 2024-09-13T14:58:10.633 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:10.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:10.736 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:10.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.774 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:10.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:10.875 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:10.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.914 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:10.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:11.015 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:11.070 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:11.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:11.173 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:11.212 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:11.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:11.314 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:11.349 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:11.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:11.451 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:11.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:11 smithi204 ceph-mon[87071]: pgmap v996: 109 pgs: 109 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-13T14:58:11.477 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:11 smithi096 ceph-mon[98901]: pgmap v996: 109 pgs: 109 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-13T14:58:11.478 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:11 smithi096 ceph-mon[102928]: pgmap v996: 109 pgs: 109 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-13T14:58:11.491 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:11.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:11.593 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:11.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 21 16 47260 47244 140.593 117.438 0.00822133 0.00710755 2024-09-13T14:58:11.629 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:11.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:11.731 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:11.765 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:11.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:11.867 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:11.899 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:11.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:11.925 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:12.035 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:12.036 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:58:12.036 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:58:12.037 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:58:12.037 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:58:12.037 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:58:12.037 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:58:12.064 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:58:12.064 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:58:12.075 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:12.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:12.092 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:58:12.092 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:58:12.111 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:58:12.111 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:58:12.125 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:58:12.125 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:58:12.155 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:58:12.155 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:58:12.177 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:12.183 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:58:12.183 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:58:12.212 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:12.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:12.314 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:12.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:12 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3613048423' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:12.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:12 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3613048423' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:12.346 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:12.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:12.448 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:12.451 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:12 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3613048423' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:12.488 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:12.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:12.590 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:12.609 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 22 16 48907 48891 138.881 102.938 0.0123266 0.00719544 2024-09-13T14:58:12.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.622 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:12.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:12.723 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:12.757 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:12.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:12.813 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:58:12.859 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:12.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.909 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:12.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:13.010 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:13.053 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:13.156 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:13.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.193 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:13.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:13.223 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:13.295 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:13.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.334 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:13.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:13.435 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:13.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:13 smithi204 ceph-mon[87071]: pgmap v997: 109 pgs: 109 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-13T14:58:13.476 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:13.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:13.520 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:13 smithi096 ceph-mon[98901]: pgmap v997: 109 pgs: 109 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-13T14:58:13.521 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:13 smithi096 ceph-mon[102928]: pgmap v997: 109 pgs: 109 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-13T14:58:13.578 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:13.610 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 23 15 50542 50527 137.281 102.25 0.00638063 0.00727817 2024-09-13T14:58:13.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.627 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:13.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:13.728 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:13.762 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:13.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:13.864 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:13.892 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:13.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:13.994 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:14.022 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:14.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:14.124 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:14.157 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:14.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:14.259 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:14.293 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:14.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:14.396 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:14.428 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:14.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:14.530 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:14.557 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:14.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:14.611 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 24 16 51980 51964 135.303 89.8125 0.00252937 0.00736721 2024-09-13T14:58:14.659 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:14.680 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:14.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:14.782 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:14.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.813 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:14.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:14.915 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:14.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.944 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:14.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:15.045 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:15.081 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:15.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:15.183 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:15.222 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:58:15.241 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:15.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:15.321 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:15 smithi096 ceph-mon[98901]: pgmap v998: 109 pgs: 109 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-09-13T14:58:15.321 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:15 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1489838777' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:58:15.321 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:15 smithi096 ceph-mon[102928]: pgmap v998: 109 pgs: 109 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-09-13T14:58:15.321 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:15 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1489838777' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:58:15.343 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:15.379 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:15.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:15.481 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:15.508 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:15 smithi204 ceph-mon[87071]: pgmap v998: 109 pgs: 109 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-09-13T14:58:15.509 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:15 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1489838777' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:58:15.514 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:15.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:15.568 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:58:15.610 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 25 16 53269 53253 133.113 80.5625 0.00378483 0.0075068 2024-09-13T14:58:15.617 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:15.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.645 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:15.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:15.746 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:15.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.787 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:15.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:15.889 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:15.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.935 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:15.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:16.036 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:16.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.058 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:16.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:16.159 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:16.199 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:16.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:16.301 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:16.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.336 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:16.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:16.437 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:16.441 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:16 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:16.441 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:16 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1489838777' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:58:16.442 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:16 smithi204 ceph-mon[87071]: osdmap e309: 8 total, 8 up, 8 in 2024-09-13T14:58:16.477 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:16.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:16.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:16 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:16.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:16 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1489838777' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:58:16.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:16 smithi096 ceph-mon[98901]: osdmap e309: 8 total, 8 up, 8 in 2024-09-13T14:58:16.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:16 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:16.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:16 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1489838777' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:58:16.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:16 smithi096 ceph-mon[102928]: osdmap e309: 8 total, 8 up, 8 in 2024-09-13T14:58:16.579 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:16.600 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:16.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:16.611 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 26 16 54480 54464 130.904 75.6875 0.0473324 0.00763315 2024-09-13T14:58:16.705 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:16.741 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:16.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:16.843 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:16.874 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:16.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:16.975 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:17.008 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:17.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:17.110 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:17.131 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:58:17.131 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:58:17.131 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T14:58:17.131 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T14:58:17.131 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:58:17.144 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:17.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:17.246 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:17.281 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:17.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:17.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:17 smithi096 ceph-mon[98901]: pgmap v1000: 109 pgs: 109 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-09-13T14:58:17.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:17 smithi096 ceph-mon[102928]: pgmap v1000: 109 pgs: 109 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-09-13T14:58:17.382 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:17.407 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:17.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:17.509 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:17.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.548 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:17.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:17.611 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 27 16 56044 56028 129.676 97.75 0.00289048 0.0077008 2024-09-13T14:58:17.649 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:17.684 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:17.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:17.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:17 smithi204 ceph-mon[87071]: pgmap v1000: 109 pgs: 109 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-09-13T14:58:17.787 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:17.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.820 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:17.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:17.843 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:58:17.921 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:17.970 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:17.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:18.072 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:18.109 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:18.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:18.212 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:18.247 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:18.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:18.295 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:18.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:58:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:58:17] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-13T14:58:18.349 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:18.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.385 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:18.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:18.487 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:18.520 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:18.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:18.611 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 28 16 57493 57477 128.279 90.5625 0.0084133 0.00779019 2024-09-13T14:58:18.622 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:18.662 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:18.764 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:18.795 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:18.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:18.896 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:18.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.938 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:18.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:19.039 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:19.068 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:19.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:19.169 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:19.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.205 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:19.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:19.261 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T14:58:19.306 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:19.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.340 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:19.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:19.441 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:19.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.475 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:19.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:19.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:19 smithi096 ceph-mon[98901]: pgmap v1001: 109 pgs: 109 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2024-09-13T14:58:19.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:19 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3450082247' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:58:19.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:19 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:58:19.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:19 smithi096 ceph-mon[102928]: pgmap v1001: 109 pgs: 109 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2024-09-13T14:58:19.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:19 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3450082247' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:58:19.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:19 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:58:19.577 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:19.611 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 29 16 59028 59012 127.163 95.9375 0.00691767 0.007859 2024-09-13T14:58:19.611 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:19.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:19.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:19.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:19 smithi204 ceph-mon[87071]: pgmap v1001: 109 pgs: 109 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2024-09-13T14:58:19.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:19 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3450082247' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:58:19.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:19 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:58:19.750 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:19.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:19.852 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:19.891 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:19.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:19.993 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:20.030 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:20.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:20.064 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T14:58:20.127 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:20.132 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:20.165 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:20.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:20.266 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:58:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:58:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:20.267 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:20.307 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:20 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:58:20.307 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:20 smithi096 ceph-mon[98901]: osdmap e310: 8 total, 8 up, 8 in 2024-09-13T14:58:20.307 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:20 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:58:20.307 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:20 smithi096 ceph-mon[102928]: osdmap e310: 8 total, 8 up, 8 in 2024-09-13T14:58:20.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.313 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:20.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:20.415 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:20.453 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:20.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:20.555 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:20.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.595 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:20.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:20.611 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 30 16 60507 60491 126.006 92.4375 0.00974925 0.00793054 2024-09-13T14:58:20.697 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:20.707 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:20 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:58:20.707 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:20 smithi204 ceph-mon[87071]: osdmap e310: 8 total, 8 up, 8 in 2024-09-13T14:58:20.734 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:20.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:20.836 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:20.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.876 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:20.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:20.978 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:21.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.015 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:21.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:21.117 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:21.158 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:21.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:21.260 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:21.303 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:21.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:21.405 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:21.439 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:21.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:21.441 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:58:21.540 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:21.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:21 smithi096 ceph-mon[98901]: pgmap v1003: 109 pgs: 109 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-13T14:58:21.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:21 smithi096 ceph-mon[102928]: pgmap v1003: 109 pgs: 109 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-13T14:58:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:21.576 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:21.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:21.601 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:58:21.601 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T14:58:21.601 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T14:58:21.602 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:58:21.602 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.602 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45749.0:211527 8.d 8:bd933b7d:::benchmark_data_smithi096_227542_object86132:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T14:58:21.602 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:56:28.341554+0000", 2024-09-13T14:58:21.602 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 112.52999617099999, 2024-09-13T14:58:21.602 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.124546488, 2024-09-13T14:58:21.602 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.602 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.602 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.602 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45749", 2024-09-13T14:58:21.602 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:58:21.602 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 211527 2024-09-13T14:58:21.602 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.603 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.603 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.603 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.603 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.341554+0000", 2024-09-13T14:58:21.603 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.603 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.603 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.603 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.603 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.341554+0000", 2024-09-13T14:58:21.603 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.603 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.603 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.603 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.603 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.341553+0000", 2024-09-13T14:58:21.604 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:21.604 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.604 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.604 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.604 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.341559+0000", 2024-09-13T14:58:21.604 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5799999999999997e-06 2024-09-13T14:58:21.604 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.604 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.604 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.604 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.341571+0000", 2024-09-13T14:58:21.604 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1613999999999999e-05 2024-09-13T14:58:21.604 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.604 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.605 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.605 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.341577+0000", 2024-09-13T14:58:21.605 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9739999999999999e-06 2024-09-13T14:58:21.605 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.605 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.605 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.605 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.345972+0000", 2024-09-13T14:58:21.605 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.004395047 2024-09-13T14:58:21.605 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.605 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.605 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.605 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.346262+0000", 2024-09-13T14:58:21.605 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00029036699999999998 2024-09-13T14:58:21.606 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.606 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.606 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:58:21.606 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.346609+0000", 2024-09-13T14:58:21.606 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00034643899999999998 2024-09-13T14:58:21.606 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.606 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.606 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.606 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.361544+0000", 2024-09-13T14:58:21.606 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.014934691 2024-09-13T14:58:21.606 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.606 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.606 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.606 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.361549+0000", 2024-09-13T14:58:21.607 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7919999999999998e-06 2024-09-13T14:58:21.607 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.607 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.607 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.607 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.466024+0000", 2024-09-13T14:58:21.607 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.10447506600000001 2024-09-13T14:58:21.607 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.607 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.607 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.607 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.466067+0000", 2024-09-13T14:58:21.607 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2925e-05 2024-09-13T14:58:21.607 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.607 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.607 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.608 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.466101+0000", 2024-09-13T14:58:21.608 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3646999999999998e-05 2024-09-13T14:58:21.608 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.608 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.608 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.608 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.608 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.608 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45749.0:211530 8.d 8:b4e26e56:::benchmark_data_smithi096_227542_object86135:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T14:58:21.608 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:56:28.347118+0000", 2024-09-13T14:58:21.608 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 112.52443295400001, 2024-09-13T14:58:21.608 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.122728462, 2024-09-13T14:58:21.608 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.608 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.609 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.609 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45749", 2024-09-13T14:58:21.609 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:58:21.609 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 211530 2024-09-13T14:58:21.609 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.609 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.609 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.609 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.609 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.347118+0000", 2024-09-13T14:58:21.609 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.609 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.609 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.609 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.609 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.347118+0000", 2024-09-13T14:58:21.610 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.610 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.610 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.610 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.610 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.347116+0000", 2024-09-13T14:58:21.610 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:21.610 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.610 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.610 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.610 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.347124+0000", 2024-09-13T14:58:21.610 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7789999999999996e-06 2024-09-13T14:58:21.610 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.610 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.611 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.611 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.347134+0000", 2024-09-13T14:58:21.611 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5850000000000006e-06 2024-09-13T14:58:21.611 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.611 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.611 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.611 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.347139+0000", 2024-09-13T14:58:21.611 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.904e-06 2024-09-13T14:58:21.611 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.611 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.611 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.611 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.352447+0000", 2024-09-13T14:58:21.611 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0053084050000000004 2024-09-13T14:58:21.611 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.612 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.612 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.612 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.352545+0000", 2024-09-13T14:58:21.612 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7800000000000006e-05 2024-09-13T14:58:21.612 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.612 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.612 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:58:21.612 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.352660+0000", 2024-09-13T14:58:21.612 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011519 2024-09-13T14:58:21.612 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.612 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.612 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.612 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.361554+0000", 2024-09-13T14:58:21.613 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0088936999999999992 2024-09-13T14:58:21.613 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.613 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.613 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.613 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.365383+0000", 2024-09-13T14:58:21.613 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0038291000000000002 2024-09-13T14:58:21.613 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.613 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.613 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.613 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.469768+0000", 2024-09-13T14:58:21.613 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.104385251 2024-09-13T14:58:21.613 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.613 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.614 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.614 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.469812+0000", 2024-09-13T14:58:21.614 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3603999999999998e-05 2024-09-13T14:58:21.614 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.614 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.614 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.614 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.469846+0000", 2024-09-13T14:58:21.614 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4387999999999997e-05 2024-09-13T14:58:21.614 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.614 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.614 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.614 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.614 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.615 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45749.0:211537 8.d 8:b21b7568:::benchmark_data_smithi096_227542_object86142:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T14:58:21.615 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:56:28.351266+0000", 2024-09-13T14:58:21.615 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 112.520285128, 2024-09-13T14:58:21.615 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.142733057, 2024-09-13T14:58:21.615 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.615 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.615 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.615 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45749", 2024-09-13T14:58:21.615 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:58:21.615 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 211537 2024-09-13T14:58:21.615 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.615 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.615 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.616 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.616 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.351266+0000", 2024-09-13T14:58:21.616 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.616 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.616 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.616 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.616 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.351266+0000", 2024-09-13T14:58:21.616 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.616 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.616 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.616 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.616 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.351265+0000", 2024-09-13T14:58:21.617 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:21.617 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.617 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.617 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.617 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.351268+0000", 2024-09-13T14:58:21.617 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.576e-06 2024-09-13T14:58:21.617 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.617 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.617 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.617 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.351273+0000", 2024-09-13T14:58:21.617 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.728e-06 2024-09-13T14:58:21.617 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.617 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.618 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.618 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.351277+0000", 2024-09-13T14:58:21.618 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.675e-06 2024-09-13T14:58:21.618 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.618 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.618 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.618 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.361660+0000", 2024-09-13T14:58:21.618 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010382567000000001 2024-09-13T14:58:21.618 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.618 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.618 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.618 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.361809+0000", 2024-09-13T14:58:21.618 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014912500000000001 2024-09-13T14:58:21.618 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.619 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.619 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:58:21.619 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.361959+0000", 2024-09-13T14:58:21.619 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000150125 2024-09-13T14:58:21.619 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.619 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.619 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.619 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.365532+0000", 2024-09-13T14:58:21.619 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035731479999999999 2024-09-13T14:58:21.619 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.619 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.619 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.619 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.366567+0000", 2024-09-13T14:58:21.620 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0010348340000000001 2024-09-13T14:58:21.620 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.620 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.620 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.620 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.493911+0000", 2024-09-13T14:58:21.620 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.12734378099999999 2024-09-13T14:58:21.620 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.620 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.620 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.620 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.493963+0000", 2024-09-13T14:58:21.620 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2074999999999999e-05 2024-09-13T14:58:21.620 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.620 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.621 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.621 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:28.493999+0000", 2024-09-13T14:58:21.621 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5892000000000001e-05 2024-09-13T14:58:21.621 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.621 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.621 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.621 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.621 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.621 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:19648 9.6 9:6371fbfd:::benchmark_data_smithi096_256909_object19647:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:58:21.621 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:57:56.855281+0000", 2024-09-13T14:58:21.621 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 24.016269855000001, 2024-09-13T14:58:21.621 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.083450249000000004, 2024-09-13T14:58:21.621 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.622 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.622 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.622 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:21.622 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:21.622 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 19648 2024-09-13T14:58:21.622 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.622 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.622 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.623 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.624 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855281+0000", 2024-09-13T14:58:21.624 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.624 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.624 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.624 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.624 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855281+0000", 2024-09-13T14:58:21.624 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.624 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.624 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.624 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.624 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855280+0000", 2024-09-13T14:58:21.624 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:21.624 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.625 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.625 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.625 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855319+0000", 2024-09-13T14:58:21.625 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9269e-05 2024-09-13T14:58:21.625 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.625 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.625 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.625 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855330+0000", 2024-09-13T14:58:21.625 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1316e-05 2024-09-13T14:58:21.625 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.625 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.625 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.625 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855336+0000", 2024-09-13T14:58:21.625 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4330000000000003e-06 2024-09-13T14:58:21.626 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.626 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.626 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.626 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.856224+0000", 2024-09-13T14:58:21.626 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00088800700000000001 2024-09-13T14:58:21.626 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.626 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.626 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.626 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.856346+0000", 2024-09-13T14:58:21.626 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000121767 2024-09-13T14:58:21.626 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.626 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.626 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:58:21.626 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.856468+0000", 2024-09-13T14:58:21.627 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012209699999999999 2024-09-13T14:58:21.627 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.627 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.627 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.627 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.859680+0000", 2024-09-13T14:58:21.627 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003212463 2024-09-13T14:58:21.627 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.627 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.627 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.627 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.860956+0000", 2024-09-13T14:58:21.627 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0012757479999999999 2024-09-13T14:58:21.627 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.627 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.628 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.628 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.938645+0000", 2024-09-13T14:58:21.628 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.077688604999999994 2024-09-13T14:58:21.628 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.628 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.628 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.628 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.938692+0000", 2024-09-13T14:58:21.628 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7475000000000002e-05 2024-09-13T14:58:21.628 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.628 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.628 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.628 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.938731+0000", 2024-09-13T14:58:21.628 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9035999999999997e-05 2024-09-13T14:58:21.629 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.629 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.629 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.629 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.629 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.629 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:19650 9.f 9:f8374e5b:::benchmark_data_smithi096_256909_object19649:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:58:21.629 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:57:56.855351+0000", 2024-09-13T14:58:21.629 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 24.016199687, 2024-09-13T14:58:21.629 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.084667223999999999, 2024-09-13T14:58:21.629 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.629 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.629 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.629 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:21.630 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:21.630 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 19650 2024-09-13T14:58:21.630 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.630 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.630 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.630 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.630 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855351+0000", 2024-09-13T14:58:21.630 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.630 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.630 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.631 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.631 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855351+0000", 2024-09-13T14:58:21.631 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.631 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.631 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.631 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.631 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855350+0000", 2024-09-13T14:58:21.631 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:21.631 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.631 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.631 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.631 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855386+0000", 2024-09-13T14:58:21.631 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6084999999999998e-05 2024-09-13T14:58:21.632 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.632 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.632 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.632 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855398+0000", 2024-09-13T14:58:21.632 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1070000000000001e-05 2024-09-13T14:58:21.632 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.632 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.632 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.632 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855402+0000", 2024-09-13T14:58:21.632 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.702e-06 2024-09-13T14:58:21.632 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.632 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.632 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.633 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.857354+0000", 2024-09-13T14:58:21.633 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0019521569999999999 2024-09-13T14:58:21.633 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.633 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.633 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.633 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.857516+0000", 2024-09-13T14:58:21.633 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016162899999999999 2024-09-13T14:58:21.633 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.633 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.633 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:21.633 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.857665+0000", 2024-09-13T14:58:21.633 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014855399999999999 2024-09-13T14:58:21.633 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.634 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.634 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.634 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.859795+0000", 2024-09-13T14:58:21.634 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002130084 2024-09-13T14:58:21.634 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.634 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.634 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.634 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.860403+0000", 2024-09-13T14:58:21.634 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00060851999999999998 2024-09-13T14:58:21.634 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.634 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.634 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.634 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.939940+0000", 2024-09-13T14:58:21.635 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.079536808000000001 2024-09-13T14:58:21.635 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.635 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.635 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.635 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.939983+0000", 2024-09-13T14:58:21.635 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2871999999999999e-05 2024-09-13T14:58:21.635 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.635 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.635 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.635 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.940018+0000", 2024-09-13T14:58:21.635 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5309999999999999e-05 2024-09-13T14:58:21.635 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.635 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.636 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.636 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.636 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.636 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:19654 9.7 9:e0c67722:::benchmark_data_smithi096_256909_object19653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:58:21.636 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:57:56.855709+0000", 2024-09-13T14:58:21.636 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 24.015841400999999, 2024-09-13T14:58:21.636 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.084325250000000004, 2024-09-13T14:58:21.636 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.636 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.636 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.636 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:21.636 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:21.636 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 19654 2024-09-13T14:58:21.637 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.637 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.637 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.637 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.637 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855709+0000", 2024-09-13T14:58:21.637 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.637 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.637 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.637 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.637 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855709+0000", 2024-09-13T14:58:21.637 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.637 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.637 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.638 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.638 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855708+0000", 2024-09-13T14:58:21.638 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:21.638 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.638 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.638 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.638 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855747+0000", 2024-09-13T14:58:21.638 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8390999999999999e-05 2024-09-13T14:58:21.638 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.638 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.638 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.638 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855756+0000", 2024-09-13T14:58:21.638 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.0480000000000004e-06 2024-09-13T14:58:21.639 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.639 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.639 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.639 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855760+0000", 2024-09-13T14:58:21.639 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1740000000000002e-06 2024-09-13T14:58:21.639 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.639 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.639 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.639 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.857640+0000", 2024-09-13T14:58:21.639 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0018803349999999999 2024-09-13T14:58:21.639 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.639 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.639 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.640 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.857765+0000", 2024-09-13T14:58:21.640 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012434800000000001 2024-09-13T14:58:21.640 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.640 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.640 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:58:21.640 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.857893+0000", 2024-09-13T14:58:21.640 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000128715 2024-09-13T14:58:21.640 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.640 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.640 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.640 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.859822+0000", 2024-09-13T14:58:21.640 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001928489 2024-09-13T14:58:21.640 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.641 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.641 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.641 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.860409+0000", 2024-09-13T14:58:21.641 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00058688800000000004 2024-09-13T14:58:21.641 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.641 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.641 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.641 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.939956+0000", 2024-09-13T14:58:21.641 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.079547598999999997 2024-09-13T14:58:21.641 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.641 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.641 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.641 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.940002+0000", 2024-09-13T14:58:21.642 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5819999999999998e-05 2024-09-13T14:58:21.642 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.642 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.642 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.642 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.940035+0000", 2024-09-13T14:58:21.642 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2223e-05 2024-09-13T14:58:21.642 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.642 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.642 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.642 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.642 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.642 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:19656 9.f 9:fa9800f6:::benchmark_data_smithi096_256909_object19655:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:58:21.642 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:57:56.855765+0000", 2024-09-13T14:58:21.642 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 24.015785682000001, 2024-09-13T14:58:21.643 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.084361773000000001, 2024-09-13T14:58:21.643 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.643 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.643 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.643 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:21.643 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:21.643 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 19656 2024-09-13T14:58:21.643 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.643 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.643 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.643 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.643 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855765+0000", 2024-09-13T14:58:21.643 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.643 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.644 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.644 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.644 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855765+0000", 2024-09-13T14:58:21.644 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.644 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.644 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.644 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.644 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855765+0000", 2024-09-13T14:58:21.644 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:21.644 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.644 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.644 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.644 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855794+0000", 2024-09-13T14:58:21.645 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9051999999999999e-05 2024-09-13T14:58:21.645 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.645 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.645 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.645 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855799+0000", 2024-09-13T14:58:21.645 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.091e-06 2024-09-13T14:58:21.645 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.645 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.645 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.645 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.855802+0000", 2024-09-13T14:58:21.645 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8590000000000001e-06 2024-09-13T14:58:21.645 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.645 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.645 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.646 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.858562+0000", 2024-09-13T14:58:21.646 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0027600760000000002 2024-09-13T14:58:21.646 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.646 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.646 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.646 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.858661+0000", 2024-09-13T14:58:21.646 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9717999999999996e-05 2024-09-13T14:58:21.646 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.646 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.646 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:21.646 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.858773+0000", 2024-09-13T14:58:21.646 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000111551 2024-09-13T14:58:21.646 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.646 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.647 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.647 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.861481+0000", 2024-09-13T14:58:21.647 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0027084000000000001 2024-09-13T14:58:21.647 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.647 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.647 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.647 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.938650+0000", 2024-09-13T14:58:21.647 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.077168451999999998 2024-09-13T14:58:21.647 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.647 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.647 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.647 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.940052+0000", 2024-09-13T14:58:21.647 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0014021280000000001 2024-09-13T14:58:21.647 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.648 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.648 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.648 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.940093+0000", 2024-09-13T14:58:21.648 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1115000000000002e-05 2024-09-13T14:58:21.648 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.648 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.648 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.648 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.940127+0000", 2024-09-13T14:58:21.648 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3683000000000003e-05 2024-09-13T14:58:21.648 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.648 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.648 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.648 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.648 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.649 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:19661 9.6 9:6c9e69b7:::benchmark_data_smithi096_256909_object19660:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:58:21.649 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:57:56.859087+0000", 2024-09-13T14:58:21.649 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 24.012463829000001, 2024-09-13T14:58:21.649 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.080065637999999995, 2024-09-13T14:58:21.649 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.649 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.649 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.649 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:21.649 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:21.649 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 19661 2024-09-13T14:58:21.649 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.649 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.649 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.650 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.650 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.859087+0000", 2024-09-13T14:58:21.650 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.650 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.650 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.650 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.650 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.859087+0000", 2024-09-13T14:58:21.650 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.650 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.650 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.650 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.650 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.859085+0000", 2024-09-13T14:58:21.650 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:21.650 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.651 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.651 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.651 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.859137+0000", 2024-09-13T14:58:21.651 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1860999999999997e-05 2024-09-13T14:58:21.651 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.651 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.651 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.651 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.859160+0000", 2024-09-13T14:58:21.651 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2558e-05 2024-09-13T14:58:21.651 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.651 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.651 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.651 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.859165+0000", 2024-09-13T14:58:21.652 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5809999999999996e-06 2024-09-13T14:58:21.652 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.652 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.652 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.652 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.859798+0000", 2024-09-13T14:58:21.652 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000632078 2024-09-13T14:58:21.652 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.652 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.652 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.652 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.859916+0000", 2024-09-13T14:58:21.652 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000118743 2024-09-13T14:58:21.652 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.652 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.652 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:58:21.653 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.860046+0000", 2024-09-13T14:58:21.653 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012928700000000001 2024-09-13T14:58:21.653 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.653 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.653 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.653 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.861002+0000", 2024-09-13T14:58:21.653 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000956025 2024-09-13T14:58:21.653 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.653 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.653 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.653 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.861460+0000", 2024-09-13T14:58:21.653 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00045875899999999999 2024-09-13T14:58:21.653 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.653 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.653 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.654 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.939068+0000", 2024-09-13T14:58:21.654 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.077608065000000004 2024-09-13T14:58:21.654 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.654 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.654 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.654 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.939116+0000", 2024-09-13T14:58:21.654 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7537999999999998e-05 2024-09-13T14:58:21.654 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.654 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.654 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.654 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.939152+0000", 2024-09-13T14:58:21.654 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6492999999999998e-05 2024-09-13T14:58:21.654 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.654 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.655 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.655 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.655 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.655 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:19664 9.f 9:f3681d91:::benchmark_data_smithi096_256909_object19663:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:58:21.655 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:57:56.860500+0000", 2024-09-13T14:58:21.655 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 24.011050704999999, 2024-09-13T14:58:21.655 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.079867529000000007, 2024-09-13T14:58:21.655 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.655 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.655 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.655 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:21.655 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:21.655 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 19664 2024-09-13T14:58:21.655 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.656 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.656 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.656 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.656 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.860500+0000", 2024-09-13T14:58:21.656 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.656 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.656 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.656 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.656 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.860500+0000", 2024-09-13T14:58:21.656 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.656 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.656 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.656 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.657 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.860499+0000", 2024-09-13T14:58:21.657 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:21.657 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.657 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.657 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.657 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.860534+0000", 2024-09-13T14:58:21.657 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4866999999999997e-05 2024-09-13T14:58:21.657 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.657 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.657 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.657 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.860547+0000", 2024-09-13T14:58:21.657 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3042e-05 2024-09-13T14:58:21.657 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.657 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.658 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.658 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.860553+0000", 2024-09-13T14:58:21.658 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8220000000000003e-06 2024-09-13T14:58:21.658 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.658 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.658 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.658 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.860608+0000", 2024-09-13T14:58:21.658 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5154000000000001e-05 2024-09-13T14:58:21.658 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.658 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.658 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.658 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.860755+0000", 2024-09-13T14:58:21.658 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000147081 2024-09-13T14:58:21.658 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.659 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.659 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:21.659 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.860915+0000", 2024-09-13T14:58:21.659 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016030800000000001 2024-09-13T14:58:21.659 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.659 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.659 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.659 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.861573+0000", 2024-09-13T14:58:21.659 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00065751600000000002 2024-09-13T14:58:21.659 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.659 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.659 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.659 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.938667+0000", 2024-09-13T14:58:21.660 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.077093660999999994 2024-09-13T14:58:21.660 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.660 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.660 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.660 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.940227+0000", 2024-09-13T14:58:21.660 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0015607489999999999 2024-09-13T14:58:21.660 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.660 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.660 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.660 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.940316+0000", 2024-09-13T14:58:21.660 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8732000000000004e-05 2024-09-13T14:58:21.660 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.660 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.661 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.661 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.940367+0000", 2024-09-13T14:58:21.661 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1499000000000003e-05 2024-09-13T14:58:21.661 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.661 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.661 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.661 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.661 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.661 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:51945 9.f 9:fc0db2d0:::benchmark_data_smithi096_256909_object51944:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2024-09-13T14:58:21.661 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:14.513677+0000", 2024-09-13T14:58:21.661 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 6.357873809, 2024-09-13T14:58:21.661 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19199633699999999, 2024-09-13T14:58:21.661 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.661 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.662 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.662 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:21.662 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:21.662 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 51945 2024-09-13T14:58:21.662 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.662 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.662 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.662 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.662 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513677+0000", 2024-09-13T14:58:21.662 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.662 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.662 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.662 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.662 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513677+0000", 2024-09-13T14:58:21.663 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.663 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.663 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.663 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.663 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513676+0000", 2024-09-13T14:58:21.663 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:21.663 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.663 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.663 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.663 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513728+0000", 2024-09-13T14:58:21.663 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2562000000000003e-05 2024-09-13T14:58:21.663 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.663 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.663 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.664 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513739+0000", 2024-09-13T14:58:21.664 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1173e-05 2024-09-13T14:58:21.664 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.664 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.664 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.664 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513745+0000", 2024-09-13T14:58:21.664 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9560000000000002e-06 2024-09-13T14:58:21.664 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.664 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.664 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.664 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513780+0000", 2024-09-13T14:58:21.664 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4693999999999997e-05 2024-09-13T14:58:21.664 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.665 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.665 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.665 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539271+0000", 2024-09-13T14:58:21.665 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.025491362 2024-09-13T14:58:21.665 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.665 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.665 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:21.665 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539413+0000", 2024-09-13T14:58:21.665 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000142058 2024-09-13T14:58:21.665 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.665 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.665 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.665 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.544116+0000", 2024-09-13T14:58:21.666 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0047030259999999999 2024-09-13T14:58:21.666 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.666 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.666 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.666 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701627+0000", 2024-09-13T14:58:21.666 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15751026100000001 2024-09-13T14:58:21.666 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.666 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.666 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.666 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705595+0000", 2024-09-13T14:58:21.666 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0039685709999999997 2024-09-13T14:58:21.666 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.666 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.666 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.667 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705639+0000", 2024-09-13T14:58:21.667 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3959999999999999e-05 2024-09-13T14:58:21.667 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.667 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.667 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.667 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705673+0000", 2024-09-13T14:58:21.667 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3964999999999998e-05 2024-09-13T14:58:21.667 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.667 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.667 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.667 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.667 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.667 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:51946 9.7 9:ecbe616b:::benchmark_data_smithi096_256909_object51945:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2024-09-13T14:58:21.667 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:14.513993+0000", 2024-09-13T14:58:21.668 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 6.3575578249999998, 2024-09-13T14:58:21.668 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.187708018, 2024-09-13T14:58:21.668 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.668 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.668 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.668 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:21.668 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:21.668 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 51946 2024-09-13T14:58:21.668 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.668 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.668 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.668 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.668 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513993+0000", 2024-09-13T14:58:21.669 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.669 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.669 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.669 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.669 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513993+0000", 2024-09-13T14:58:21.669 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.669 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.669 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.669 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.669 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513992+0000", 2024-09-13T14:58:21.669 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:21.669 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.669 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.669 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.670 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.514057+0000", 2024-09-13T14:58:21.670 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5326000000000004e-05 2024-09-13T14:58:21.670 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.670 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.670 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.670 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539161+0000", 2024-09-13T14:58:21.670 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.025103503999999999 2024-09-13T14:58:21.670 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.670 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.670 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.670 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539174+0000", 2024-09-13T14:58:21.670 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2542e-05 2024-09-13T14:58:21.670 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.670 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.671 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.671 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539542+0000", 2024-09-13T14:58:21.671 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00036863999999999999 2024-09-13T14:58:21.671 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.671 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.671 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.671 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539683+0000", 2024-09-13T14:58:21.671 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014123200000000001 2024-09-13T14:58:21.671 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.671 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.671 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:58:21.671 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539825+0000", 2024-09-13T14:58:21.671 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014115699999999999 2024-09-13T14:58:21.672 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.672 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.672 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.672 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.544025+0000", 2024-09-13T14:58:21.672 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0042005430000000002 2024-09-13T14:58:21.672 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.672 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.672 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.672 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.544395+0000", 2024-09-13T14:58:21.672 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000369432 2024-09-13T14:58:21.672 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.672 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.672 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.673 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701625+0000", 2024-09-13T14:58:21.673 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.157230326 2024-09-13T14:58:21.673 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.673 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.673 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.673 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701664+0000", 2024-09-13T14:58:21.673 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8760000000000002e-05 2024-09-13T14:58:21.673 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.673 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.673 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.673 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701701+0000", 2024-09-13T14:58:21.673 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7243999999999998e-05 2024-09-13T14:58:21.673 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.674 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.674 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.674 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.674 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.674 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:51947 9.f 9:f8f9d21b:::benchmark_data_smithi096_256909_object51946:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2024-09-13T14:58:21.674 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:14.539191+0000", 2024-09-13T14:58:21.674 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 6.3323593169999999, 2024-09-13T14:58:21.674 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.166579427, 2024-09-13T14:58:21.674 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.674 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.674 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.674 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:21.674 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:21.675 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 51947 2024-09-13T14:58:21.675 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.675 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.675 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.675 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.675 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539191+0000", 2024-09-13T14:58:21.675 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.675 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.675 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.675 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.675 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539191+0000", 2024-09-13T14:58:21.675 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.675 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.676 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.676 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.676 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539190+0000", 2024-09-13T14:58:21.676 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:21.676 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.676 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.676 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.676 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539238+0000", 2024-09-13T14:58:21.676 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8136000000000001e-05 2024-09-13T14:58:21.676 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.676 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.676 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.676 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539249+0000", 2024-09-13T14:58:21.676 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1051e-05 2024-09-13T14:58:21.677 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.677 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.677 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.677 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539253+0000", 2024-09-13T14:58:21.677 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4120000000000003e-06 2024-09-13T14:58:21.677 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.677 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.677 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.677 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.542175+0000", 2024-09-13T14:58:21.677 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029216730000000001 2024-09-13T14:58:21.677 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.677 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.677 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.677 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.542272+0000", 2024-09-13T14:58:21.678 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6768999999999995e-05 2024-09-13T14:58:21.678 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.678 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.678 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:21.678 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.542363+0000", 2024-09-13T14:58:21.678 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1615999999999998e-05 2024-09-13T14:58:21.678 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.678 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.678 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.678 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.544362+0000", 2024-09-13T14:58:21.678 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0019985200000000002 2024-09-13T14:58:21.678 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.678 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.679 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.679 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701753+0000", 2024-09-13T14:58:21.679 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15739146200000001 2024-09-13T14:58:21.679 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.679 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.679 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.679 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705705+0000", 2024-09-13T14:58:21.679 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0039517579999999997 2024-09-13T14:58:21.679 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.679 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.679 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.679 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705742+0000", 2024-09-13T14:58:21.679 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6715000000000003e-05 2024-09-13T14:58:21.679 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.679 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.680 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.680 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705771+0000", 2024-09-13T14:58:21.680 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8858999999999999e-05 2024-09-13T14:58:21.680 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.680 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.680 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.680 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.680 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.680 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:51948 9.6 9:6c080a85:::benchmark_data_smithi096_256909_object51947:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2024-09-13T14:58:21.680 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:14.539259+0000", 2024-09-13T14:58:21.680 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 6.3322918939999999, 2024-09-13T14:58:21.680 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16478026000000001, 2024-09-13T14:58:21.680 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.681 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.681 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.681 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:21.681 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:21.681 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 51948 2024-09-13T14:58:21.681 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.681 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.681 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.681 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.681 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539259+0000", 2024-09-13T14:58:21.681 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.681 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.681 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.681 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.682 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539259+0000", 2024-09-13T14:58:21.682 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.682 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.682 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.682 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.682 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539258+0000", 2024-09-13T14:58:21.682 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:21.682 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.682 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.682 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.682 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539295+0000", 2024-09-13T14:58:21.682 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7004e-05 2024-09-13T14:58:21.682 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.683 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.683 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.683 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539302+0000", 2024-09-13T14:58:21.683 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0350000000000004e-06 2024-09-13T14:58:21.683 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.683 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.683 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.683 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539306+0000", 2024-09-13T14:58:21.683 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.929e-06 2024-09-13T14:58:21.683 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.683 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.683 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.683 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539804+0000", 2024-09-13T14:58:21.683 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00049790399999999997 2024-09-13T14:58:21.684 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.684 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.684 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.684 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539898+0000", 2024-09-13T14:58:21.684 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.4310999999999999e-05 2024-09-13T14:58:21.684 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.684 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.684 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:58:21.684 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.540041+0000", 2024-09-13T14:58:21.684 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014301200000000001 2024-09-13T14:58:21.684 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.684 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.684 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.685 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.543085+0000", 2024-09-13T14:58:21.685 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0030438230000000002 2024-09-13T14:58:21.685 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.685 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.685 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.685 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.703863+0000", 2024-09-13T14:58:21.685 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16077804600000001 2024-09-13T14:58:21.685 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.685 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.685 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.685 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.703953+0000", 2024-09-13T14:58:21.685 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.9790000000000006e-05 2024-09-13T14:58:21.685 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.685 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.686 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.686 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.704001+0000", 2024-09-13T14:58:21.686 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7555000000000001e-05 2024-09-13T14:58:21.686 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.686 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.686 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.686 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.704039+0000", 2024-09-13T14:58:21.686 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8339e-05 2024-09-13T14:58:21.686 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.686 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.686 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.686 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.686 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.687 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:51960 9.6 9:6ef6cfd1:::benchmark_data_smithi096_256909_object51959:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2024-09-13T14:58:21.687 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:14.546674+0000", 2024-09-13T14:58:21.687 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 6.3248767859999999, 2024-09-13T14:58:21.687 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15746892600000001, 2024-09-13T14:58:21.687 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.687 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.687 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.687 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:21.687 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:21.687 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 51960 2024-09-13T14:58:21.687 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.687 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.688 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.688 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.688 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.546674+0000", 2024-09-13T14:58:21.688 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.688 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.688 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.688 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.688 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.546674+0000", 2024-09-13T14:58:21.688 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.688 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.688 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.688 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.688 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.546668+0000", 2024-09-13T14:58:21.689 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999943 2024-09-13T14:58:21.689 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.689 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.689 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.689 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.546715+0000", 2024-09-13T14:58:21.689 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7095999999999998e-05 2024-09-13T14:58:21.689 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.689 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.689 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.689 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.546724+0000", 2024-09-13T14:58:21.689 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8100000000000004e-06 2024-09-13T14:58:21.689 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.689 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.690 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.690 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.546728+0000", 2024-09-13T14:58:21.690 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5789999999999997e-06 2024-09-13T14:58:21.690 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.690 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.690 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.690 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.546764+0000", 2024-09-13T14:58:21.690 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5602999999999998e-05 2024-09-13T14:58:21.690 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.690 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.690 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.690 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.547609+0000", 2024-09-13T14:58:21.690 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00084498799999999997 2024-09-13T14:58:21.690 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.691 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.691 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:58:21.691 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.547765+0000", 2024-09-13T14:58:21.691 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015567599999999999 2024-09-13T14:58:21.691 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.691 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.691 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.691 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.703297+0000", 2024-09-13T14:58:21.691 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.155531961 2024-09-13T14:58:21.691 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.691 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.691 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.691 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.703893+0000", 2024-09-13T14:58:21.691 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00059641200000000005 2024-09-13T14:58:21.692 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.692 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.692 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.692 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.704068+0000", 2024-09-13T14:58:21.692 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00017456000000000001 2024-09-13T14:58:21.692 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.692 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.692 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.692 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.704110+0000", 2024-09-13T14:58:21.692 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2746e-05 2024-09-13T14:58:21.692 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.692 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.692 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.693 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.704143+0000", 2024-09-13T14:58:21.693 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2428999999999997e-05 2024-09-13T14:58:21.693 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.693 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.693 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.693 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.693 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.693 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:51962 9.6 9:6e44ab85:::benchmark_data_smithi096_256909_object51961:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2024-09-13T14:58:21.693 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:14.546782+0000", 2024-09-13T14:58:21.693 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 6.3247691560000003, 2024-09-13T14:58:21.693 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.157457237, 2024-09-13T14:58:21.693 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.693 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.693 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.694 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:21.694 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:21.694 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 51962 2024-09-13T14:58:21.694 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.694 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.694 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.694 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.694 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.546782+0000", 2024-09-13T14:58:21.694 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.694 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.694 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.694 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.694 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.546782+0000", 2024-09-13T14:58:21.695 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.695 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.695 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.695 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.695 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.546781+0000", 2024-09-13T14:58:21.695 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:21.695 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.695 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.695 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.695 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.546846+0000", 2024-09-13T14:58:21.695 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4975999999999996e-05 2024-09-13T14:58:21.695 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.695 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.695 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.696 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.546855+0000", 2024-09-13T14:58:21.696 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.3489999999999998e-06 2024-09-13T14:58:21.696 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.696 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.696 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.696 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.546860+0000", 2024-09-13T14:58:21.696 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1290000000000004e-06 2024-09-13T14:58:21.696 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.696 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.696 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.696 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.549642+0000", 2024-09-13T14:58:21.696 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0027816859999999998 2024-09-13T14:58:21.696 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.696 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.697 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.697 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.549784+0000", 2024-09-13T14:58:21.697 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000142134 2024-09-13T14:58:21.697 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.697 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.697 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:58:21.697 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.549906+0000", 2024-09-13T14:58:21.697 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000122006 2024-09-13T14:58:21.697 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.697 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.697 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.697 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.703534+0000", 2024-09-13T14:58:21.697 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.153627703 2024-09-13T14:58:21.697 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.698 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.698 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.698 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.703917+0000", 2024-09-13T14:58:21.698 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00038302300000000002 2024-09-13T14:58:21.698 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.698 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.698 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.698 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.704172+0000", 2024-09-13T14:58:21.698 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00025544599999999999 2024-09-13T14:58:21.698 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.698 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.698 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.698 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.704208+0000", 2024-09-13T14:58:21.699 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6300999999999997e-05 2024-09-13T14:58:21.699 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.699 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.699 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.699 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.704239+0000", 2024-09-13T14:58:21.699 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0358999999999999e-05 2024-09-13T14:58:21.699 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.699 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.699 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.699 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.699 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.699 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:51963 9.a 9:5ff4ee6f:::benchmark_data_smithi096_256909_object51962:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2024-09-13T14:58:21.699 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:14.549639+0000", 2024-09-13T14:58:21.699 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 6.3219114809999999, 2024-09-13T14:58:21.700 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15941506899999999, 2024-09-13T14:58:21.700 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.700 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.700 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.700 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:21.700 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:21.700 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 51963 2024-09-13T14:58:21.700 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.700 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.700 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.700 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.700 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.549639+0000", 2024-09-13T14:58:21.700 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.701 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.701 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.701 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.701 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.549639+0000", 2024-09-13T14:58:21.701 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.701 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.701 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.701 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.701 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.549637+0000", 2024-09-13T14:58:21.701 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:58:21.701 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.701 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.701 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.702 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.549669+0000", 2024-09-13T14:58:21.702 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1534e-05 2024-09-13T14:58:21.702 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.702 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.702 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.702 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.549681+0000", 2024-09-13T14:58:21.702 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.167e-05 2024-09-13T14:58:21.702 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.702 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.702 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.702 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.549690+0000", 2024-09-13T14:58:21.702 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.0000000000000002e-06 2024-09-13T14:58:21.702 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.702 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.703 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.703 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.549741+0000", 2024-09-13T14:58:21.703 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1514000000000002e-05 2024-09-13T14:58:21.703 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.703 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.703 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.703 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.549865+0000", 2024-09-13T14:58:21.703 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000123645 2024-09-13T14:58:21.703 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.703 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.703 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:58:21.703 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701497+0000", 2024-09-13T14:58:21.703 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.151632023 2024-09-13T14:58:21.704 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.704 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.704 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.704 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.703538+0000", 2024-09-13T14:58:21.704 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0020411180000000002 2024-09-13T14:58:21.704 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.704 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.704 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.704 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705877+0000", 2024-09-13T14:58:21.704 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0023386959999999999 2024-09-13T14:58:21.704 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.704 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.704 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.705 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.708920+0000", 2024-09-13T14:58:21.705 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0030435929999999998 2024-09-13T14:58:21.705 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.705 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.705 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.705 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.708987+0000", 2024-09-13T14:58:21.705 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6404999999999999e-05 2024-09-13T14:58:21.705 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.705 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.705 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.705 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.709054+0000", 2024-09-13T14:58:21.705 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7614999999999996e-05 2024-09-13T14:58:21.705 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.705 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.706 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.706 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.706 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.706 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.46040.0:51964 9.8 e308/302 9:1191e2bd:::benchmark_data_smithi096_256909_object51963:head v 308'3222, mlcod=308'3220)", 2024-09-13T14:58:21.706 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:14.551434+0000", 2024-09-13T14:58:21.706 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 6.3201163219999996, 2024-09-13T14:58:21.706 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15437509499999999, 2024-09-13T14:58:21.706 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.706 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.706 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.706 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.706 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.706 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.551434+0000", 2024-09-13T14:58:21.706 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.707 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.707 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.707 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.707 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.551434+0000", 2024-09-13T14:58:21.707 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.707 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.707 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.707 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.707 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.551434+0000", 2024-09-13T14:58:21.707 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:21.707 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.707 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.707 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.708 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.551537+0000", 2024-09-13T14:58:21.708 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000103738 2024-09-13T14:58:21.708 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.708 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.708 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.708 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701382+0000", 2024-09-13T14:58:21.708 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.149844753 2024-09-13T14:58:21.708 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.708 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.708 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.708 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701392+0000", 2024-09-13T14:58:21.708 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9769999999999995e-06 2024-09-13T14:58:21.708 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.709 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.709 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.709 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701435+0000", 2024-09-13T14:58:21.709 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2592000000000001e-05 2024-09-13T14:58:21.709 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.709 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.709 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.709 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701460+0000", 2024-09-13T14:58:21.709 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4960000000000002e-05 2024-09-13T14:58:21.709 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.709 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.709 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.709 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705781+0000", 2024-09-13T14:58:21.709 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.004321826 2024-09-13T14:58:21.710 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.710 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.710 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.710 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705809+0000", 2024-09-13T14:58:21.710 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7983e-05 2024-09-13T14:58:21.710 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.710 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.710 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.710 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.710 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.710 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.46040.0:51971 9.b e308/302 9:dd03afcd:::benchmark_data_smithi096_256909_object51970:head v 308'3256, mlcod=308'3254)", 2024-09-13T14:58:21.710 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:14.557990+0000", 2024-09-13T14:58:21.710 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 6.3135609309999996, 2024-09-13T14:58:21.711 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.147070382, 2024-09-13T14:58:21.711 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.711 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.711 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.711 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.711 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.711 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.557990+0000", 2024-09-13T14:58:21.711 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.711 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.711 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.711 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.711 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.557990+0000", 2024-09-13T14:58:21.711 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.711 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.712 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.712 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.712 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.557988+0000", 2024-09-13T14:58:21.712 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:21.712 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.712 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.712 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.712 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.558085+0000", 2024-09-13T14:58:21.712 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6098e-05 2024-09-13T14:58:21.712 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.712 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.712 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.712 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701379+0000", 2024-09-13T14:58:21.713 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.14329465699999999 2024-09-13T14:58:21.713 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.713 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.713 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.713 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701392+0000", 2024-09-13T14:58:21.713 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2539e-05 2024-09-13T14:58:21.713 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.713 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.713 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.713 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701490+0000", 2024-09-13T14:58:21.713 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.8577000000000002e-05 2024-09-13T14:58:21.713 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.713 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.713 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.714 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701511+0000", 2024-09-13T14:58:21.714 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1080000000000001e-05 2024-09-13T14:58:21.714 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.714 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.714 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.714 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705026+0000", 2024-09-13T14:58:21.714 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003514428 2024-09-13T14:58:21.714 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.714 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.714 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.714 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705060+0000", 2024-09-13T14:58:21.714 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4311999999999999e-05 2024-09-13T14:58:21.714 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.715 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.715 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.715 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.715 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.715 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:52272 9.6 9:6e38594b:::benchmark_data_smithi096_256909_object52271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2024-09-13T14:58:21.715 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:14.909141+0000", 2024-09-13T14:58:21.715 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 5.9624095989999999, 2024-09-13T14:58:21.715 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.071542887999999999, 2024-09-13T14:58:21.715 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.715 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.715 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.715 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:21.715 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:21.716 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 52272 2024-09-13T14:58:21.716 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.716 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.716 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.716 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.716 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.909141+0000", 2024-09-13T14:58:21.716 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.716 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.716 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.716 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.716 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.909141+0000", 2024-09-13T14:58:21.716 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.716 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.716 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.717 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.717 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.909140+0000", 2024-09-13T14:58:21.717 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:21.717 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.717 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.717 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.717 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.909211+0000", 2024-09-13T14:58:21.717 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0847999999999997e-05 2024-09-13T14:58:21.717 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.717 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.717 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.717 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.909222+0000", 2024-09-13T14:58:21.717 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1572e-05 2024-09-13T14:58:21.718 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.718 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.718 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.718 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.909228+0000", 2024-09-13T14:58:21.718 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6779999999999997e-06 2024-09-13T14:58:21.718 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.718 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.718 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.718 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.909270+0000", 2024-09-13T14:58:21.718 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1783000000000003e-05 2024-09-13T14:58:21.718 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.718 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.718 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.718 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.909377+0000", 2024-09-13T14:58:21.719 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000107159 2024-09-13T14:58:21.719 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.719 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.719 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:58:21.719 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.909525+0000", 2024-09-13T14:58:21.719 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000148336 2024-09-13T14:58:21.719 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.719 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.719 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.719 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.911401+0000", 2024-09-13T14:58:21.719 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0018752650000000001 2024-09-13T14:58:21.719 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.719 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.719 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.720 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.913469+0000", 2024-09-13T14:58:21.720 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002068336 2024-09-13T14:58:21.720 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.720 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.720 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.720 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.980612+0000", 2024-09-13T14:58:21.720 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.067142779 2024-09-13T14:58:21.720 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.720 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.720 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.720 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.980651+0000", 2024-09-13T14:58:21.720 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8849e-05 2024-09-13T14:58:21.720 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.720 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.721 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.721 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.980684+0000", 2024-09-13T14:58:21.721 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3271e-05 2024-09-13T14:58:21.721 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.721 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.721 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.721 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.721 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.721 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:52280 9.f 9:f8f3936f:::benchmark_data_smithi096_256909_object52279:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2024-09-13T14:58:21.721 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:14.912860+0000", 2024-09-13T14:58:21.721 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 5.9586909810000002, 2024-09-13T14:58:21.721 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.067944401000000001, 2024-09-13T14:58:21.721 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:21.721 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:21.722 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:21.722 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:21.722 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:21.722 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 52280 2024-09-13T14:58:21.722 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.722 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:21.722 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.722 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:21.722 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.912860+0000", 2024-09-13T14:58:21.722 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.722 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.722 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.722 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:21.723 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.912860+0000", 2024-09-13T14:58:21.723 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:21.723 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.723 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.723 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:21.723 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.912858+0000", 2024-09-13T14:58:21.723 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:21.723 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.723 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.723 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:21.723 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.912891+0000", 2024-09-13T14:58:21.723 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3118999999999999e-05 2024-09-13T14:58:21.723 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.723 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.724 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:21.724 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.912903+0000", 2024-09-13T14:58:21.724 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1741e-05 2024-09-13T14:58:21.724 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.724 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.724 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:21.724 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.912908+0000", 2024-09-13T14:58:21.724 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3279999999999996e-06 2024-09-13T14:58:21.724 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.724 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.724 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:21.724 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.912957+0000", 2024-09-13T14:58:21.724 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.846e-05 2024-09-13T14:58:21.724 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.724 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.725 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:21.725 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.913058+0000", 2024-09-13T14:58:21.725 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010137 2024-09-13T14:58:21.725 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.725 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.725 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:21.725 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.913216+0000", 2024-09-13T14:58:21.725 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015835400000000001 2024-09-13T14:58:21.725 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.725 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.725 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:21.725 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.916123+0000", 2024-09-13T14:58:21.725 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029068750000000002 2024-09-13T14:58:21.725 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.725 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.726 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.726 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.918881+0000", 2024-09-13T14:58:21.726 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0027574660000000001 2024-09-13T14:58:21.726 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.726 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.726 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:21.726 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.980739+0000", 2024-09-13T14:58:21.726 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.061857717999999999 2024-09-13T14:58:21.726 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.726 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.726 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:21.726 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.980773+0000", 2024-09-13T14:58:21.726 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4656000000000002e-05 2024-09-13T14:58:21.726 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:21.726 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:21.727 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:21.727 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.980804+0000", 2024-09-13T14:58:21.727 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0882e-05 2024-09-13T14:58:21.727 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.727 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.727 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.727 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:21.727 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:21.727 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:58:21.727 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:21 smithi204 ceph-mon[87071]: pgmap v1003: 109 pgs: 109 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-13T14:58:21.727 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:21.728 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:21.728 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 31 16 61920 61904 124.789 88.3125 0.00801295 0.00800876 2024-09-13T14:58:21.731 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:21.764 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:21.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:21.866 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:21.905 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:21.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:22.007 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:22.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.056 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:22.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:22.157 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:22.191 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:22.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:22.292 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:22.323 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:22.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:22.425 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:22.462 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:22.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:22.555 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:58:22.564 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:22.602 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:22.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:22.611 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 32 16 63311 63295 123.606 86.9375 0.00949066 0.00808434 2024-09-13T14:58:22.704 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:22.736 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:22.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:22.838 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:22.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.884 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:22.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:22.955 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:22.985 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:23.017 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:23.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:23.119 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:23.155 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:23.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:23.257 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:23.288 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:23.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:23.390 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:23.405 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:23 smithi204 ceph-mon[87071]: pgmap v1004: 109 pgs: 109 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:58:23.427 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:23.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:23.529 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:23.571 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:23.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:23.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:23 smithi096 ceph-mon[98901]: pgmap v1004: 109 pgs: 109 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:58:23.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:23 smithi096 ceph-mon[102928]: pgmap v1004: 109 pgs: 109 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:58:23.611 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 33 16 64738 64722 122.563 89.1875 0.0114578 0.0081542 2024-09-13T14:58:23.673 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:23.711 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:23.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:23.813 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:23.850 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:23.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:23.952 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:23.990 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:23.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:24.092 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:24.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.130 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:24.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:24.231 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:24.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.268 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:24.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:24.369 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:24.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.420 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:24.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:24.499 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:58:24.499 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:58:24.511 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:24.521 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:24.523 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:58:24.523 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:58:24.559 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:58:24.560 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:58:24.576 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:24.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:24.593 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:58:24.593 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:58:24.611 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 34 16 66104 66088 121.469 85.375 0.00537683 0.00822693 2024-09-13T14:58:24.615 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:58:24.616 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:58:24.655 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:58:24.656 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:58:24.680 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:24.692 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:58:24.692 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:58:24.719 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:24.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:24.735 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:58:24.735 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:58:24.821 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:24.844 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:24.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:24.945 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:24.979 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:24.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:25.082 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:25.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.117 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:25.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:25.219 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:25.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.261 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:25.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:25.362 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:25.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.394 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:25.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:25.496 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:25.524 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:25.525 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:25 smithi096 ceph-mon[98901]: pgmap v1005: 109 pgs: 109 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-13T14:58:25.525 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:25 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4144834275' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:25.526 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:25 smithi096 ceph-mon[102928]: pgmap v1005: 109 pgs: 109 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-13T14:58:25.526 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:25 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4144834275' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:25.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:25.611 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 35 16 67367 67351 120.254 78.9375 0.0149576 0.00831089 2024-09-13T14:58:25.612 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:58:25.612 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:58:25.612 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:25 smithi204 ceph-mon[87071]: pgmap v1005: 109 pgs: 109 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-13T14:58:25.612 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:25 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4144834275' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:25.628 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:25.663 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:25.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:25.765 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:25.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.801 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:25.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:25.902 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:25.941 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:25.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:26.044 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:26.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.078 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:26.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:26.180 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:26.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.214 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:26.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:26.315 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:26.348 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:26.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:26.450 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:26.484 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:26.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:26.585 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:26.611 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 36 16 68761 68745 119.333 87.125 0.00437581 0.00837475 2024-09-13T14:58:26.620 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:26.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:26.722 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:26.754 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:26.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:26.856 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:26.891 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:26.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:26.993 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:27.030 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:27.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:27.095 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:58:27.132 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:27.182 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:27.284 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:27.311 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T14:58:27.318 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:27.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:27.372 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:27 smithi096 ceph-mon[98901]: pgmap v1006: 109 pgs: 109 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:58:27.372 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:27 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3973904397' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:58:27.372 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:27 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:58:27.372 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:27 smithi096 ceph-mon[102928]: pgmap v1006: 109 pgs: 109 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:58:27.373 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:27 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3973904397' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:58:27.373 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:27 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:58:27.419 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:27.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.473 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:27.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:27.519 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:27.528 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:58:27.574 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:27.612 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 37 16 70154 70138 118.461 87.0625 0.0152251 0.00843613 2024-09-13T14:58:27.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.617 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:27.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:27.652 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:27 smithi204 ceph-mon[87071]: pgmap v1006: 109 pgs: 109 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:58:27.652 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:27 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3973904397' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:58:27.652 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:27 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:58:27.718 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:27.750 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:27.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:27.851 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:27.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.893 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:27.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:27.994 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:28.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.032 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:28.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:28.053 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:58:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:58:27] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-13T14:58:28.100 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T14:58:28.101 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:58:28.133 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:28.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.166 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:28.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:28.267 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:28.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.318 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:28.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:28.379 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:28.419 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:28.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.468 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:28.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:28.569 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:28.607 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:28.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:28.611 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 38 16 71434 71418 117.448 80 0.0135802 0.00850938 2024-09-13T14:58:28.693 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:28 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:58:28.693 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:28 smithi096 ceph-mon[102928]: osdmap e311: 8 total, 8 up, 8 in 2024-09-13T14:58:28.693 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:28 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:58:28.693 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:28 smithi096 ceph-mon[98901]: osdmap e311: 8 total, 8 up, 8 in 2024-09-13T14:58:28.709 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:28 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:58:28.709 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:28 smithi204 ceph-mon[87071]: osdmap e311: 8 total, 8 up, 8 in 2024-09-13T14:58:28.709 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:28.741 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:28.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:28.843 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:28.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.881 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:28.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:28.982 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:29.018 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:29.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:29.119 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:29.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.166 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:29.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:29.267 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:29.303 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:29.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:29.347 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:58:29.405 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:29.428 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T14:58:29.428 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:58:29.434 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:29.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:29.536 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:29.574 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:29.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:29.612 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 39 16 72734 72718 116.52 81.25 0.00641481 0.00857805 2024-09-13T14:58:29.676 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:29.697 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:29 smithi096 ceph-mon[98901]: pgmap v1008: 109 pgs: 109 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T14:58:29.697 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:29 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3477666923' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:29.697 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:29 smithi096 ceph-mon[102928]: pgmap v1008: 109 pgs: 109 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T14:58:29.697 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:29 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3477666923' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:29.705 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:29.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:29.712 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:29 smithi204 ceph-mon[87071]: pgmap v1008: 109 pgs: 109 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T14:58:29.713 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:29 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3477666923' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:29.806 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:29.841 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:29.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:29.943 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:29.982 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:29.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:30.083 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:30.121 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:30.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:30.223 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:30.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.265 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:30.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:30.322 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:58:30.365 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:30.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.402 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:30.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:30.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:30 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:30.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:30 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T14:58:30.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:30 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3477666923' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:30.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:30 smithi204 ceph-mon[87071]: osdmap e312: 8 total, 8 up, 8 in 2024-09-13T14:58:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:58:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:58:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:30.503 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:30.544 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:30.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:30.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:30 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:30.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:30 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T14:58:30.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:30 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3477666923' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:30.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:30 smithi096 ceph-mon[102928]: osdmap e312: 8 total, 8 up, 8 in 2024-09-13T14:58:30.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:30 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:30.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:30 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T14:58:30.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:30 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3477666923' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:30.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:30 smithi096 ceph-mon[98901]: osdmap e312: 8 total, 8 up, 8 in 2024-09-13T14:58:30.612 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:58:30.611740+0000 min lat: 0.00112813 max lat: 0.278402 avg lat: 0.00864473 2024-09-13T14:58:30.612 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:58:30.612 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 40 16 74014 73998 115.607 80 0.0159462 0.00864473 2024-09-13T14:58:30.646 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:30.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.695 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:30.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:30.774 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:30.796 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:30.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.843 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:30.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:30.944 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:30.985 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:30.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:31.087 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:31.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.121 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:31.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:31.222 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:31.257 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:31.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:31.360 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:31.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.397 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:31.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:31.498 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:31.533 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:31.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:31.612 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 41 16 75405 75389 114.908 86.9375 0.0160985 0.00869785 2024-09-13T14:58:31.635 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:31.659 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:31 smithi096 ceph-mon[98901]: pgmap v1010: 109 pgs: 109 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-09-13T14:58:31.659 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:31 smithi096 ceph-mon[102928]: pgmap v1010: 109 pgs: 109 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-09-13T14:58:31.667 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:31.667 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:31 smithi204 ceph-mon[87071]: pgmap v1010: 109 pgs: 109 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-09-13T14:58:31.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:31.769 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:31.806 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:31.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:31.908 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:31.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.956 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:31.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:32.057 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:32.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.093 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:32.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:32.195 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:32.216 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:32.222 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:32.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:32.307 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:58:32.308 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:58:32.324 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:32.342 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:58:32.342 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:58:32.360 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:32.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:32.373 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:58:32.373 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:58:32.405 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:58:32.405 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:58:32.433 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:58:32.433 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:58:32.462 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:32.467 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:58:32.467 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:58:32.467 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:32 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4016918429' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:32.468 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:32 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4016918429' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:32.500 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:58:32.500 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:58:32.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.501 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:32.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:32.529 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:58:32.529 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:58:32.602 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:32.612 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 42 16 76634 76618 114 76.8125 0.0156231 0.00876658 2024-09-13T14:58:32.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.637 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:32.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:32.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:32 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4016918429' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:32.739 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:32.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.773 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:32.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:32.874 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:32.916 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:32.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:33.018 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:33.058 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:33.160 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:33.236 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:33.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:33.338 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:33.379 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:33.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:33.396 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:33 smithi204 ceph-mon[87071]: pgmap v1011: 109 pgs: 109 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2024-09-13T14:58:33.450 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:58:33.450 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T14:58:33.450 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T14:58:33.450 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:58:33.450 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.450 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169247 7.7 7:ee1c7cba:::benchmark_data_smithi096_196313_object49116:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2024-09-13T14:58:33.450 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.032573+0000", 2024-09-13T14:58:33.450 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 443.41080122900001, 2024-09-13T14:58:33.451 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2455696700000001, 2024-09-13T14:58:33.451 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.451 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:58:33.451 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.451 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.451 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.451 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169247 2024-09-13T14:58:33.451 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.451 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.451 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.451 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.451 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.032573+0000", 2024-09-13T14:58:33.451 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.452 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.452 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.452 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.452 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.032573+0000", 2024-09-13T14:58:33.452 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.452 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.452 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.452 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.452 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.032571+0000", 2024-09-13T14:58:33.452 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999976 2024-09-13T14:58:33.452 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.452 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.452 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.452 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.032579+0000", 2024-09-13T14:58:33.453 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.4739999999999999e-06 2024-09-13T14:58:33.453 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.453 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.453 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.453 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.032596+0000", 2024-09-13T14:58:33.453 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6900000000000001e-05 2024-09-13T14:58:33.453 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.453 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.453 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.453 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.032602+0000", 2024-09-13T14:58:33.453 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3289999999999999e-06 2024-09-13T14:58:33.453 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.453 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.454 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.454 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.032655+0000", 2024-09-13T14:58:33.454 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2725000000000001e-05 2024-09-13T14:58:33.454 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.454 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.454 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.454 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.032780+0000", 2024-09-13T14:58:33.454 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000125592 2024-09-13T14:58:33.454 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.454 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.455 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:58:33.455 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.032902+0000", 2024-09-13T14:58:33.455 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000121403 2024-09-13T14:58:33.455 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.455 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.455 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.455 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.036892+0000", 2024-09-13T14:58:33.455 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0039907720000000001 2024-09-13T14:58:33.455 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.455 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.455 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.455 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.043471+0000", 2024-09-13T14:58:33.455 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.006578225 2024-09-13T14:58:33.455 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.456 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.456 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.456 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.057240+0000", 2024-09-13T14:58:33.456 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013769554 2024-09-13T14:58:33.456 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.456 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.456 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.456 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.057251+0000", 2024-09-13T14:58:33.456 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1051e-05 2024-09-13T14:58:33.456 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.456 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.456 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.456 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.052502+0000", 2024-09-13T14:58:33.457 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99525045499999998 2024-09-13T14:58:33.457 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.457 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.457 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.457 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.052510+0000", 2024-09-13T14:58:33.457 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9980000000000003e-06 2024-09-13T14:58:33.457 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.457 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.457 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.457 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.278053+0000", 2024-09-13T14:58:33.457 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.225543257 2024-09-13T14:58:33.457 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.457 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.457 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.458 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.278143+0000", 2024-09-13T14:58:33.458 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.0117999999999993e-05 2024-09-13T14:58:33.458 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.458 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.458 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.458 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.458 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.458 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169251 7.7 7:ea76573f:::benchmark_data_smithi096_196313_object49120:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2024-09-13T14:58:33.458 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.055132+0000", 2024-09-13T14:58:33.458 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 443.38824231900003, 2024-09-13T14:58:33.458 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2340449320000002, 2024-09-13T14:58:33.458 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.458 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.458 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.459 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.459 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.459 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169251 2024-09-13T14:58:33.459 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.459 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.459 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.459 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.459 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.055132+0000", 2024-09-13T14:58:33.459 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.459 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.459 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.459 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.459 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.055132+0000", 2024-09-13T14:58:33.459 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.460 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.460 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.460 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.460 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.055130+0000", 2024-09-13T14:58:33.460 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:58:33.460 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.460 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.460 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.460 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.055139+0000", 2024-09-13T14:58:33.460 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.0459999999999994e-06 2024-09-13T14:58:33.460 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.460 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.460 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.460 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.055158+0000", 2024-09-13T14:58:33.461 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9131e-05 2024-09-13T14:58:33.461 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.461 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.461 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.461 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.055167+0000", 2024-09-13T14:58:33.461 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3489999999999992e-06 2024-09-13T14:58:33.461 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.461 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.461 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.461 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.057539+0000", 2024-09-13T14:58:33.461 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0023720590000000001 2024-09-13T14:58:33.462 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.462 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.462 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.462 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.057546+0000", 2024-09-13T14:58:33.462 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6020000000000002e-06 2024-09-13T14:58:33.462 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.462 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.462 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.462 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.052531+0000", 2024-09-13T14:58:33.462 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99498406800000005 2024-09-13T14:58:33.462 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.462 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.462 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.463 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.052537+0000", 2024-09-13T14:58:33.463 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7000000000000002e-06 2024-09-13T14:58:33.463 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.463 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.463 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.463 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.278444+0000", 2024-09-13T14:58:33.463 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2259063370000001 2024-09-13T14:58:33.463 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.463 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.463 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.463 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.278568+0000", 2024-09-13T14:58:33.463 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000124797 2024-09-13T14:58:33.463 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.463 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.464 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,1,3", 2024-09-13T14:58:33.464 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.278706+0000", 2024-09-13T14:58:33.464 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013759500000000001 2024-09-13T14:58:33.464 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.464 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.464 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.464 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.286116+0000", 2024-09-13T14:58:33.464 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0074099739999999997 2024-09-13T14:58:33.464 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.464 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.464 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.464 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.288362+0000", 2024-09-13T14:58:33.465 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0022460610000000002 2024-09-13T14:58:33.465 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.465 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.465 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.465 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.288772+0000", 2024-09-13T14:58:33.465 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00040957299999999999 2024-09-13T14:58:33.465 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.465 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.465 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.465 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.289075+0000", 2024-09-13T14:58:33.465 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00030354999999999998 2024-09-13T14:58:33.465 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.465 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.465 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.466 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.289107+0000", 2024-09-13T14:58:33.466 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.184e-05 2024-09-13T14:58:33.466 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.466 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.466 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.466 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.289177+0000", 2024-09-13T14:58:33.466 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0240999999999994e-05 2024-09-13T14:58:33.466 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.466 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.466 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.466 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.466 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.466 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169281 7.c 7:34ca89d6:::benchmark_data_smithi096_196313_object49150:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T14:58:33.466 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.109501+0000", 2024-09-13T14:58:33.467 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 443.333873208, 2024-09-13T14:58:33.467 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.176190879, 2024-09-13T14:58:33.467 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.467 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.467 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.467 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.467 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.467 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169281 2024-09-13T14:58:33.467 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.467 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.467 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.467 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.467 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109501+0000", 2024-09-13T14:58:33.467 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.468 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.468 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.468 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.468 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109501+0000", 2024-09-13T14:58:33.468 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.468 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.468 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.468 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.468 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109500+0000", 2024-09-13T14:58:33.468 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:58:33.468 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.468 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.468 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.469 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109509+0000", 2024-09-13T14:58:33.469 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.2450000000000004e-06 2024-09-13T14:58:33.469 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.469 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.469 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.469 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109527+0000", 2024-09-13T14:58:33.469 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8587999999999999e-05 2024-09-13T14:58:33.469 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.469 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.469 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.469 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109534+0000", 2024-09-13T14:58:33.469 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5910000000000001e-06 2024-09-13T14:58:33.469 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.469 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.470 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.470 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109592+0000", 2024-09-13T14:58:33.470 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7549000000000001e-05 2024-09-13T14:58:33.470 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.470 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.470 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.470 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109620+0000", 2024-09-13T14:58:33.470 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8231000000000001e-05 2024-09-13T14:58:33.470 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.470 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.470 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.470 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.051780+0000", 2024-09-13T14:58:33.470 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.942159741 2024-09-13T14:58:33.470 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.471 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.471 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.471 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.051789+0000", 2024-09-13T14:58:33.471 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9389999999999992e-06 2024-09-13T14:58:33.471 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.471 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.471 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.471 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.273314+0000", 2024-09-13T14:58:33.471 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2215250200000001 2024-09-13T14:58:33.471 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.471 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.471 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.471 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.273418+0000", 2024-09-13T14:58:33.471 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000103461 2024-09-13T14:58:33.472 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.472 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.472 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T14:58:33.472 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.273504+0000", 2024-09-13T14:58:33.472 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.6458999999999993e-05 2024-09-13T14:58:33.472 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.472 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.472 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.472 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.285318+0000", 2024-09-13T14:58:33.472 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011813913000000001 2024-09-13T14:58:33.472 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.472 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.472 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.472 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.285371+0000", 2024-09-13T14:58:33.473 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3091999999999999e-05 2024-09-13T14:58:33.473 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.473 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.473 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.473 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.285411+0000", 2024-09-13T14:58:33.473 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0008e-05 2024-09-13T14:58:33.473 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.473 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.473 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.473 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.285597+0000", 2024-09-13T14:58:33.473 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000185401 2024-09-13T14:58:33.473 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.473 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.473 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.474 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.285630+0000", 2024-09-13T14:58:33.474 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2935999999999998e-05 2024-09-13T14:58:33.474 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.474 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.474 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.474 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.285692+0000", 2024-09-13T14:58:33.474 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2508999999999994e-05 2024-09-13T14:58:33.474 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.474 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.474 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.474 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.474 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.474 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169283 7.7 7:e9f817a4:::benchmark_data_smithi096_196313_object49152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T14:58:33.474 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.109887+0000", 2024-09-13T14:58:33.475 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 443.33348773900002, 2024-09-13T14:58:33.475 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1909418829999998, 2024-09-13T14:58:33.475 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.475 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.475 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.475 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.475 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.475 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169283 2024-09-13T14:58:33.475 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.475 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.475 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.475 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.475 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109887+0000", 2024-09-13T14:58:33.476 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.476 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.476 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.476 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.476 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109887+0000", 2024-09-13T14:58:33.476 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.476 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.476 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.476 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.476 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109886+0000", 2024-09-13T14:58:33.476 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:33.476 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.476 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.476 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.477 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109892+0000", 2024-09-13T14:58:33.477 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.015e-06 2024-09-13T14:58:33.477 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.477 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.477 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.477 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109907+0000", 2024-09-13T14:58:33.477 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4922999999999999e-05 2024-09-13T14:58:33.477 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.477 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.477 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.477 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109914+0000", 2024-09-13T14:58:33.477 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.3749999999999997e-06 2024-09-13T14:58:33.477 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.477 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.478 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.478 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109964+0000", 2024-09-13T14:58:33.478 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0061000000000001e-05 2024-09-13T14:58:33.478 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.478 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.478 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.478 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109970+0000", 2024-09-13T14:58:33.478 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3820000000000001e-06 2024-09-13T14:58:33.478 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.478 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.479 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.479 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.052560+0000", 2024-09-13T14:58:33.479 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.94259002599999997 2024-09-13T14:58:33.480 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.480 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.480 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.480 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.052567+0000", 2024-09-13T14:58:33.480 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2550000000000003e-06 2024-09-13T14:58:33.480 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.480 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.480 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.480 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.282603+0000", 2024-09-13T14:58:33.480 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.230036189 2024-09-13T14:58:33.480 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.480 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.480 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.481 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.282728+0000", 2024-09-13T14:58:33.481 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012545000000000001 2024-09-13T14:58:33.481 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.481 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.481 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,1,3", 2024-09-13T14:58:33.481 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.282855+0000", 2024-09-13T14:58:33.481 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012703599999999999 2024-09-13T14:58:33.481 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.481 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.481 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.481 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.286267+0000", 2024-09-13T14:58:33.481 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003411182 2024-09-13T14:58:33.481 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.481 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.482 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.482 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.288812+0000", 2024-09-13T14:58:33.482 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00254507 2024-09-13T14:58:33.482 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.482 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.482 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.482 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.289208+0000", 2024-09-13T14:58:33.482 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00039643099999999999 2024-09-13T14:58:33.482 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.482 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.482 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.482 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.300714+0000", 2024-09-13T14:58:33.482 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011505995 2024-09-13T14:58:33.483 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.483 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.483 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.483 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.300760+0000", 2024-09-13T14:58:33.483 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6356000000000001e-05 2024-09-13T14:58:33.483 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.483 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.483 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.483 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.300829+0000", 2024-09-13T14:58:33.483 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8293999999999994e-05 2024-09-13T14:58:33.483 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.483 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.483 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.483 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.484 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.484 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169288 7.c 7:3f70aeb0:::benchmark_data_smithi096_196313_object49157:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T14:58:33.484 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.117877+0000", 2024-09-13T14:58:33.484 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 443.32549804500002, 2024-09-13T14:58:33.484 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1711432980000001, 2024-09-13T14:58:33.484 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.484 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.484 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.484 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.484 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.484 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169288 2024-09-13T14:58:33.484 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.484 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.485 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.485 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.485 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.117877+0000", 2024-09-13T14:58:33.485 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.485 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.485 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.485 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.485 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.117877+0000", 2024-09-13T14:58:33.485 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.485 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.485 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.485 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.485 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.117875+0000", 2024-09-13T14:58:33.486 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:58:33.486 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.486 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.486 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.486 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.117886+0000", 2024-09-13T14:58:33.486 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1895999999999999e-05 2024-09-13T14:58:33.486 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.486 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.486 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.486 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.117908+0000", 2024-09-13T14:58:33.486 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2079e-05 2024-09-13T14:58:33.486 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.486 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.486 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.487 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.117917+0000", 2024-09-13T14:58:33.487 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7390000000000005e-06 2024-09-13T14:58:33.487 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.487 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.487 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.487 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.117986+0000", 2024-09-13T14:58:33.487 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8750000000000004e-05 2024-09-13T14:58:33.487 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.487 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.487 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.487 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.117998+0000", 2024-09-13T14:58:33.487 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1783999999999999e-05 2024-09-13T14:58:33.487 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.487 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.487 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.488 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.051809+0000", 2024-09-13T14:58:33.488 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93381159599999997 2024-09-13T14:58:33.488 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.488 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.488 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.488 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.051816+0000", 2024-09-13T14:58:33.488 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4010000000000002e-06 2024-09-13T14:58:33.488 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.488 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.488 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.488 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.276884+0000", 2024-09-13T14:58:33.488 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2250681839999999 2024-09-13T14:58:33.488 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.489 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.489 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.489 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.277005+0000", 2024-09-13T14:58:33.489 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000121476 2024-09-13T14:58:33.489 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.489 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.489 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T14:58:33.489 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.277140+0000", 2024-09-13T14:58:33.489 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013439499999999999 2024-09-13T14:58:33.489 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.489 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.489 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.489 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.285422+0000", 2024-09-13T14:58:33.489 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0082821720000000008 2024-09-13T14:58:33.490 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.490 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.490 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.490 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.285735+0000", 2024-09-13T14:58:33.490 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00031304599999999998 2024-09-13T14:58:33.490 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.490 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.490 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.490 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.285763+0000", 2024-09-13T14:58:33.491 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7744000000000001e-05 2024-09-13T14:58:33.491 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.491 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.491 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.491 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.288861+0000", 2024-09-13T14:58:33.491 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0030980280000000001 2024-09-13T14:58:33.491 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.491 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.491 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.491 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.288918+0000", 2024-09-13T14:58:33.491 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6910000000000002e-05 2024-09-13T14:58:33.492 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.492 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.492 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.492 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.289020+0000", 2024-09-13T14:58:33.492 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000102141 2024-09-13T14:58:33.492 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.492 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.492 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.492 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.492 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.492 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169289 7.c 7:331f35d6:::benchmark_data_smithi096_196313_object49158:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T14:58:33.492 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.119144+0000", 2024-09-13T14:58:33.493 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 443.32423056099998, 2024-09-13T14:58:33.493 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1816879309999999, 2024-09-13T14:58:33.493 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.493 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.493 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.493 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.493 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.493 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169289 2024-09-13T14:58:33.493 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.493 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.493 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.493 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.494 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.119144+0000", 2024-09-13T14:58:33.494 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.494 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.494 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.494 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.494 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.119144+0000", 2024-09-13T14:58:33.494 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.494 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.494 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.494 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.494 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.119143+0000", 2024-09-13T14:58:33.494 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:33.494 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.495 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.495 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.495 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.119148+0000", 2024-09-13T14:58:33.495 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.904e-06 2024-09-13T14:58:33.495 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.495 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.495 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.495 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.119159+0000", 2024-09-13T14:58:33.495 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1031e-05 2024-09-13T14:58:33.495 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.495 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.495 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.495 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.119164+0000", 2024-09-13T14:58:33.496 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3340000000000001e-06 2024-09-13T14:58:33.496 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.496 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.496 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.496 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.119206+0000", 2024-09-13T14:58:33.496 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1922000000000003e-05 2024-09-13T14:58:33.496 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.496 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.496 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.496 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.119213+0000", 2024-09-13T14:58:33.496 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8600000000000004e-06 2024-09-13T14:58:33.496 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.496 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.496 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.497 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.051834+0000", 2024-09-13T14:58:33.497 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93262118199999999 2024-09-13T14:58:33.497 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.497 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.497 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.497 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.051841+0000", 2024-09-13T14:58:33.497 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4799999999999998e-06 2024-09-13T14:58:33.498 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.498 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.498 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.498 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.281133+0000", 2024-09-13T14:58:33.498 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.229291898 2024-09-13T14:58:33.498 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.498 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.498 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.498 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.281267+0000", 2024-09-13T14:58:33.498 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013379399999999999 2024-09-13T14:58:33.498 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.498 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.498 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T14:58:33.498 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.281403+0000", 2024-09-13T14:58:33.499 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000136728 2024-09-13T14:58:33.499 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.499 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.499 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.499 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.285770+0000", 2024-09-13T14:58:33.499 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0043663119999999998 2024-09-13T14:58:33.499 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.499 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.499 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.499 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.285794+0000", 2024-09-13T14:58:33.499 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.3918e-05 2024-09-13T14:58:33.499 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.499 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.499 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.500 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.285821+0000", 2024-09-13T14:58:33.500 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7926999999999999e-05 2024-09-13T14:58:33.500 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.500 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.500 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.500 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.300691+0000", 2024-09-13T14:58:33.500 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.014869284 2024-09-13T14:58:33.500 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.500 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.500 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.500 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.300755+0000", 2024-09-13T14:58:33.500 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4288999999999994e-05 2024-09-13T14:58:33.500 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.500 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.501 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.501 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.300832+0000", 2024-09-13T14:58:33.501 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6948999999999998e-05 2024-09-13T14:58:33.501 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.501 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.501 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.501 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.501 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.501 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193672 7.c 7:3c3448d3:::benchmark_data_smithi096_196313_object73541:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-09-13T14:58:33.501 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.177979+0000", 2024-09-13T14:58:33.501 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 408.265395963, 2024-09-13T14:58:33.501 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.032648639, 2024-09-13T14:58:33.501 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.502 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:58:33.502 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.502 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.502 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.502 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193672 2024-09-13T14:58:33.502 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.502 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.502 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.502 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.502 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.177979+0000", 2024-09-13T14:58:33.502 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.502 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.502 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.502 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.503 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.177979+0000", 2024-09-13T14:58:33.503 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.503 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.503 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.503 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.503 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.177977+0000", 2024-09-13T14:58:33.503 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:58:33.503 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.503 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.503 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.503 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.177984+0000", 2024-09-13T14:58:33.503 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9689999999999997e-06 2024-09-13T14:58:33.504 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.504 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.504 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.504 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.177997+0000", 2024-09-13T14:58:33.504 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3138e-05 2024-09-13T14:58:33.504 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.504 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.504 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.504 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.178003+0000", 2024-09-13T14:58:33.504 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.84e-06 2024-09-13T14:58:33.504 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.504 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.504 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.505 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.178051+0000", 2024-09-13T14:58:33.505 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8118000000000002e-05 2024-09-13T14:58:33.505 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.505 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.505 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.505 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.178189+0000", 2024-09-13T14:58:33.505 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000137632 2024-09-13T14:58:33.505 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.505 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.505 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T14:58:33.505 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.178324+0000", 2024-09-13T14:58:33.505 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013502499999999999 2024-09-13T14:58:33.505 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.505 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.506 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.506 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.186082+0000", 2024-09-13T14:58:33.506 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0077580089999999997 2024-09-13T14:58:33.506 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.506 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.506 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.506 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.186093+0000", 2024-09-13T14:58:33.506 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1022000000000001e-05 2024-09-13T14:58:33.506 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.506 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.506 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.506 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.189796+0000", 2024-09-13T14:58:33.506 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0037034949999999998 2024-09-13T14:58:33.506 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.507 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.507 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.507 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.189805+0000", 2024-09-13T14:58:33.507 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.037e-06 2024-09-13T14:58:33.507 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.507 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.507 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.507 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.210554+0000", 2024-09-13T14:58:33.507 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0207485650000001 2024-09-13T14:58:33.507 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.507 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.507 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.507 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.210627+0000", 2024-09-13T14:58:33.507 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.3478000000000004e-05 2024-09-13T14:58:33.508 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.508 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.508 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.508 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.508 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.508 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193675 7.c 7:3f91f1c4:::benchmark_data_smithi096_196313_object73544:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-09-13T14:58:33.508 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.181545+0000", 2024-09-13T14:58:33.508 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 408.261829245, 2024-09-13T14:58:33.508 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.029564718, 2024-09-13T14:58:33.508 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.508 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:58:33.508 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.508 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.508 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.509 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193675 2024-09-13T14:58:33.509 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.509 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.509 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.509 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.509 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.181545+0000", 2024-09-13T14:58:33.509 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.509 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.509 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.509 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.509 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.181545+0000", 2024-09-13T14:58:33.509 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.509 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.510 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.510 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.510 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.181544+0000", 2024-09-13T14:58:33.510 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:33.510 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.510 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.510 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.510 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.181550+0000", 2024-09-13T14:58:33.510 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4370000000000004e-06 2024-09-13T14:58:33.510 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.510 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.510 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.510 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.181564+0000", 2024-09-13T14:58:33.510 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3356999999999999e-05 2024-09-13T14:58:33.511 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.511 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.511 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.511 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.181569+0000", 2024-09-13T14:58:33.511 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.446e-06 2024-09-13T14:58:33.511 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.511 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.511 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.511 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.182087+0000", 2024-09-13T14:58:33.511 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00051824300000000005 2024-09-13T14:58:33.511 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.511 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.511 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.511 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.182198+0000", 2024-09-13T14:58:33.512 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000111106 2024-09-13T14:58:33.512 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.512 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.512 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T14:58:33.512 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.182297+0000", 2024-09-13T14:58:33.512 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9020999999999999e-05 2024-09-13T14:58:33.512 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.512 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.512 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.512 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.187301+0000", 2024-09-13T14:58:33.512 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0050036910000000002 2024-09-13T14:58:33.512 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.512 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.512 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.513 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.189901+0000", 2024-09-13T14:58:33.513 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0025996700000000001 2024-09-13T14:58:33.513 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.513 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.513 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.513 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.189908+0000", 2024-09-13T14:58:33.513 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5830000000000001e-06 2024-09-13T14:58:33.513 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.513 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.513 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.513 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.211040+0000", 2024-09-13T14:58:33.513 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0211311439999999 2024-09-13T14:58:33.513 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.513 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.514 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.514 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.211110+0000", 2024-09-13T14:58:33.514 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0531999999999995e-05 2024-09-13T14:58:33.514 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.514 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.514 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.514 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.514 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.514 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193694 7.c 7:375370fa:::benchmark_data_smithi096_196313_object73563:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2024-09-13T14:58:33.514 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.212908+0000", 2024-09-13T14:58:33.514 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 408.23046627399998, 2024-09-13T14:58:33.514 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0150302760000001, 2024-09-13T14:58:33.514 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.514 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.515 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.515 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.515 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.515 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193694 2024-09-13T14:58:33.515 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.515 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.515 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.515 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.515 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.212908+0000", 2024-09-13T14:58:33.515 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.515 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.515 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.515 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.515 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.212908+0000", 2024-09-13T14:58:33.516 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.516 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.516 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.516 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.516 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.212907+0000", 2024-09-13T14:58:33.516 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:33.516 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.516 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.516 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.516 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.212914+0000", 2024-09-13T14:58:33.516 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7660000000000001e-06 2024-09-13T14:58:33.516 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.516 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.516 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.517 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.212945+0000", 2024-09-13T14:58:33.517 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1584000000000001e-05 2024-09-13T14:58:33.517 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.517 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.517 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.518 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.212952+0000", 2024-09-13T14:58:33.518 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6139999999999997e-06 2024-09-13T14:58:33.518 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.518 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.518 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.519 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.213009+0000", 2024-09-13T14:58:33.519 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7293999999999997e-05 2024-09-13T14:58:33.519 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.519 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.519 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.519 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.213018+0000", 2024-09-13T14:58:33.520 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5369999999999997e-06 2024-09-13T14:58:33.520 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.520 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.520 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.520 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.211146+0000", 2024-09-13T14:58:33.521 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99812826099999996 2024-09-13T14:58:33.521 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.521 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.521 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.521 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.211234+0000", 2024-09-13T14:58:33.521 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7675000000000003e-05 2024-09-13T14:58:33.521 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.521 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.521 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:58:33.521 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.211304+0000", 2024-09-13T14:58:33.521 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0656999999999998e-05 2024-09-13T14:58:33.521 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.521 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.522 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.522 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.223283+0000", 2024-09-13T14:58:33.522 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011978277000000001 2024-09-13T14:58:33.522 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.522 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.522 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.522 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.223334+0000", 2024-09-13T14:58:33.522 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0895e-05 2024-09-13T14:58:33.522 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.522 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.522 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.523 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.227842+0000", 2024-09-13T14:58:33.523 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0045089420000000002 2024-09-13T14:58:33.523 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.523 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.523 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.523 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.227891+0000", 2024-09-13T14:58:33.523 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8998000000000001e-05 2024-09-13T14:58:33.523 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.523 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.523 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.523 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.227939+0000", 2024-09-13T14:58:33.523 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7102999999999999e-05 2024-09-13T14:58:33.523 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.523 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.524 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.524 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.524 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.524 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193715 7.c 7:311e1db2:::benchmark_data_smithi096_196313_object73584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2024-09-13T14:58:33.524 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.238753+0000", 2024-09-13T14:58:33.524 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 408.20462123200002, 2024-09-13T14:58:33.524 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99428842900000003, 2024-09-13T14:58:33.524 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.524 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.524 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.524 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.525 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.525 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193715 2024-09-13T14:58:33.525 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.525 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.525 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.525 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.525 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238753+0000", 2024-09-13T14:58:33.525 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.525 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.525 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:33.530 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:33 smithi096 ceph-mon[98901]: pgmap v1011: 109 pgs: 109 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2024-09-13T14:58:33.530 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:33 smithi096 ceph-mon[102928]: pgmap v1011: 109 pgs: 109 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2024-09-13T14:58:33.531 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.531 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.531 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238753+0000", 2024-09-13T14:58:33.531 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.531 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.531 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.532 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.532 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238752+0000", 2024-09-13T14:58:33.532 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:33.532 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.532 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.532 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.532 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238760+0000", 2024-09-13T14:58:33.532 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0830000000000004e-06 2024-09-13T14:58:33.532 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.532 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.532 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.533 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238777+0000", 2024-09-13T14:58:33.533 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6670000000000001e-05 2024-09-13T14:58:33.533 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.533 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.533 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.533 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238785+0000", 2024-09-13T14:58:33.533 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7689999999999996e-06 2024-09-13T14:58:33.533 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.533 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.534 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.534 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238828+0000", 2024-09-13T14:58:33.534 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3096999999999998e-05 2024-09-13T14:58:33.534 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.534 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.534 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.534 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238836+0000", 2024-09-13T14:58:33.534 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9459999999999998e-06 2024-09-13T14:58:33.534 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.534 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.534 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.534 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.215529+0000", 2024-09-13T14:58:33.535 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97669314500000004 2024-09-13T14:58:33.535 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.535 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.535 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.535 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.215644+0000", 2024-09-13T14:58:33.535 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000115549 2024-09-13T14:58:33.536 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.536 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.536 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:58:33.536 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.215770+0000", 2024-09-13T14:58:33.536 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012607300000000001 2024-09-13T14:58:33.536 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.536 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.536 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.536 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.227949+0000", 2024-09-13T14:58:33.536 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012179024 2024-09-13T14:58:33.536 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.537 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.537 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.537 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.232885+0000", 2024-09-13T14:58:33.537 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0049357089999999999 2024-09-13T14:58:33.537 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.537 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.537 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.537 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.232945+0000", 2024-09-13T14:58:33.537 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0398e-05 2024-09-13T14:58:33.537 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.537 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.537 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.537 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.232996+0000", 2024-09-13T14:58:33.537 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0985000000000001e-05 2024-09-13T14:58:33.538 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.538 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.538 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.538 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233042+0000", 2024-09-13T14:58:33.538 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5321000000000002e-05 2024-09-13T14:58:33.538 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.538 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.538 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.538 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.538 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.538 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193716 7.c 7:3f878838:::benchmark_data_smithi096_196313_object73585:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2024-09-13T14:58:33.538 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.239711+0000", 2024-09-13T14:58:33.539 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 408.20366350099999, 2024-09-13T14:58:33.539 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99352322699999995, 2024-09-13T14:58:33.539 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.539 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.539 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.539 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.539 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.539 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193716 2024-09-13T14:58:33.539 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.539 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.539 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.540 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.540 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239711+0000", 2024-09-13T14:58:33.540 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.540 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.540 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.540 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.540 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239711+0000", 2024-09-13T14:58:33.540 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.541 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.541 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.541 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.541 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239710+0000", 2024-09-13T14:58:33.541 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:33.541 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.541 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.541 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.541 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239715+0000", 2024-09-13T14:58:33.541 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4009999999999999e-06 2024-09-13T14:58:33.541 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.541 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.541 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.542 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239723+0000", 2024-09-13T14:58:33.542 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5990000000000002e-06 2024-09-13T14:58:33.542 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.542 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.542 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.542 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239729+0000", 2024-09-13T14:58:33.542 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1780000000000002e-06 2024-09-13T14:58:33.542 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.542 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.542 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.542 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239755+0000", 2024-09-13T14:58:33.542 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.6869e-05 2024-09-13T14:58:33.542 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.542 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.543 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.543 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239761+0000", 2024-09-13T14:58:33.543 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3530000000000002e-06 2024-09-13T14:58:33.543 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.543 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.543 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.543 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.219460+0000", 2024-09-13T14:58:33.543 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97969935600000002 2024-09-13T14:58:33.543 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.544 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.544 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.544 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.219555+0000", 2024-09-13T14:58:33.544 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.4474000000000004e-05 2024-09-13T14:58:33.544 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.545 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.545 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:58:33.545 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.219662+0000", 2024-09-13T14:58:33.545 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000107624 2024-09-13T14:58:33.545 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.545 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.545 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.545 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233063+0000", 2024-09-13T14:58:33.545 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013401135999999999 2024-09-13T14:58:33.546 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.546 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.546 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.546 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233107+0000", 2024-09-13T14:58:33.546 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3887999999999997e-05 2024-09-13T14:58:33.546 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.547 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.547 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.547 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233151+0000", 2024-09-13T14:58:33.547 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3993999999999999e-05 2024-09-13T14:58:33.547 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.547 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.547 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.547 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233197+0000", 2024-09-13T14:58:33.547 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5535999999999999e-05 2024-09-13T14:58:33.547 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.547 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.548 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.548 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233234+0000", 2024-09-13T14:58:33.548 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7459999999999997e-05 2024-09-13T14:58:33.548 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.548 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.548 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.548 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.548 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.548 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193744 7.c 7:3cb38b24:::benchmark_data_smithi096_196313_object73613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2024-09-13T14:58:33.548 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.277160+0000", 2024-09-13T14:58:33.548 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 408.16621504099999, 2024-09-13T14:58:33.548 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95624868799999996, 2024-09-13T14:58:33.548 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.549 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.549 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.549 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.549 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.549 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193744 2024-09-13T14:58:33.549 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.549 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.549 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.549 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.549 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277160+0000", 2024-09-13T14:58:33.550 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.550 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.550 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.550 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.550 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277160+0000", 2024-09-13T14:58:33.550 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.550 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.550 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.550 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.551 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277158+0000", 2024-09-13T14:58:33.551 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:33.551 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.551 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.551 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.551 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277164+0000", 2024-09-13T14:58:33.551 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5260000000000001e-06 2024-09-13T14:58:33.551 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.551 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.551 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.551 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277176+0000", 2024-09-13T14:58:33.551 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2353e-05 2024-09-13T14:58:33.552 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.552 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.552 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.552 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277182+0000", 2024-09-13T14:58:33.552 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.818e-06 2024-09-13T14:58:33.552 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.552 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.552 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.552 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277218+0000", 2024-09-13T14:58:33.552 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.578e-05 2024-09-13T14:58:33.552 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.552 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.552 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.552 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277225+0000", 2024-09-13T14:58:33.553 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.4869999999999996e-06 2024-09-13T14:58:33.553 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.553 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.553 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.553 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.223362+0000", 2024-09-13T14:58:33.553 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.94613646500000004 2024-09-13T14:58:33.553 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.553 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.553 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.553 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.223463+0000", 2024-09-13T14:58:33.554 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000101635 2024-09-13T14:58:33.554 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.554 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.554 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:58:33.554 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.223569+0000", 2024-09-13T14:58:33.554 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000106063 2024-09-13T14:58:33.554 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.554 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.555 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.555 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233078+0000", 2024-09-13T14:58:33.555 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0095081460000000003 2024-09-13T14:58:33.555 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.555 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.555 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.556 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233289+0000", 2024-09-13T14:58:33.556 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00021173899999999999 2024-09-13T14:58:33.556 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.556 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.556 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.557 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233326+0000", 2024-09-13T14:58:33.557 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6884999999999997e-05 2024-09-13T14:58:33.558 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.558 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.558 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.558 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233373+0000", 2024-09-13T14:58:33.558 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6882999999999998e-05 2024-09-13T14:58:33.558 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.558 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.559 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.559 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233408+0000", 2024-09-13T14:58:33.559 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5098999999999997e-05 2024-09-13T14:58:33.559 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.559 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.559 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.559 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.559 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.559 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193768 7.c 7:3b3e0e52:::benchmark_data_smithi096_196313_object73637:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2024-09-13T14:58:33.559 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.313978+0000", 2024-09-13T14:58:33.559 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 408.12939668799999, 2024-09-13T14:58:33.560 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93176736800000004, 2024-09-13T14:58:33.560 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.560 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.560 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.560 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.560 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.560 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193768 2024-09-13T14:58:33.560 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.560 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.560 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.560 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.561 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.313978+0000", 2024-09-13T14:58:33.561 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.561 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.561 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.561 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.561 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.313978+0000", 2024-09-13T14:58:33.561 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.561 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.561 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.561 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.561 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.313976+0000", 2024-09-13T14:58:33.561 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:33.561 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.562 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.562 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.562 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.313983+0000", 2024-09-13T14:58:33.562 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6590000000000001e-06 2024-09-13T14:58:33.562 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.562 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.562 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.562 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.314003+0000", 2024-09-13T14:58:33.562 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9842000000000001e-05 2024-09-13T14:58:33.562 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.562 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.562 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.562 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.314010+0000", 2024-09-13T14:58:33.562 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0260000000000002e-06 2024-09-13T14:58:33.563 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.563 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.563 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.563 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.314057+0000", 2024-09-13T14:58:33.563 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7092000000000002e-05 2024-09-13T14:58:33.563 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.563 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.563 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.563 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.314065+0000", 2024-09-13T14:58:33.563 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9869999999999999e-06 2024-09-13T14:58:33.563 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.563 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.563 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.564 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.227993+0000", 2024-09-13T14:58:33.564 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.91392780200000001 2024-09-13T14:58:33.564 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.564 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.564 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.564 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.228101+0000", 2024-09-13T14:58:33.564 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000108186 2024-09-13T14:58:33.564 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.564 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.564 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:58:33.564 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.228225+0000", 2024-09-13T14:58:33.564 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012376100000000001 2024-09-13T14:58:33.565 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.565 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.565 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.565 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233244+0000", 2024-09-13T14:58:33.565 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0050197490000000004 2024-09-13T14:58:33.565 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.565 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.565 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.565 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233450+0000", 2024-09-13T14:58:33.565 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00020564499999999999 2024-09-13T14:58:33.566 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.566 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.566 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.566 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.245654+0000", 2024-09-13T14:58:33.566 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012204281000000001 2024-09-13T14:58:33.566 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.566 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.566 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.566 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.245706+0000", 2024-09-13T14:58:33.566 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1480000000000002e-05 2024-09-13T14:58:33.567 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.567 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.567 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.567 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.245745+0000", 2024-09-13T14:58:33.567 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9524999999999999e-05 2024-09-13T14:58:33.567 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.567 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.567 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.567 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.567 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.567 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193782 7.c 7:3bff5279:::benchmark_data_smithi096_196313_object73651:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2024-09-13T14:58:33.567 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.329589+0000", 2024-09-13T14:58:33.567 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 408.113786049, 2024-09-13T14:58:33.568 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.92642909600000001, 2024-09-13T14:58:33.568 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.568 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.568 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.568 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.568 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.568 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193782 2024-09-13T14:58:33.568 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.568 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.568 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.569 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.569 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329589+0000", 2024-09-13T14:58:33.569 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.569 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.569 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.569 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.569 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329589+0000", 2024-09-13T14:58:33.569 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.569 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.569 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.569 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.570 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329587+0000", 2024-09-13T14:58:33.570 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:33.570 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.570 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.570 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.570 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329594+0000", 2024-09-13T14:58:33.570 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9299999999999997e-06 2024-09-13T14:58:33.570 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.570 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.570 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.570 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329604+0000", 2024-09-13T14:58:33.570 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0658e-05 2024-09-13T14:58:33.570 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.571 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.571 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.571 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329609+0000", 2024-09-13T14:58:33.571 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3019999999999996e-06 2024-09-13T14:58:33.571 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.571 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.571 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.571 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329653+0000", 2024-09-13T14:58:33.571 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4045999999999998e-05 2024-09-13T14:58:33.571 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.571 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.571 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.571 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329662+0000", 2024-09-13T14:58:33.571 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.3740000000000004e-06 2024-09-13T14:58:33.572 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.572 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.572 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.572 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233483+0000", 2024-09-13T14:58:33.572 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.90382105800000001 2024-09-13T14:58:33.572 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.572 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.572 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.572 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233646+0000", 2024-09-13T14:58:33.572 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016245300000000001 2024-09-13T14:58:33.572 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.572 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.572 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:58:33.573 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233764+0000", 2024-09-13T14:58:33.573 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000118293 2024-09-13T14:58:33.573 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.573 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.573 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.573 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.245755+0000", 2024-09-13T14:58:33.573 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011990806 2024-09-13T14:58:33.573 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.573 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.573 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.573 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.245856+0000", 2024-09-13T14:58:33.573 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000100739 2024-09-13T14:58:33.573 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.573 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.573 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.574 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.255938+0000", 2024-09-13T14:58:33.574 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010082559 2024-09-13T14:58:33.574 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.574 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.574 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.574 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.255978+0000", 2024-09-13T14:58:33.574 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9461e-05 2024-09-13T14:58:33.574 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.574 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.574 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.575 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.256018+0000", 2024-09-13T14:58:33.575 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0059999999999999e-05 2024-09-13T14:58:33.575 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.575 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.575 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.575 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.575 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.575 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193788 7.c 7:37a34649:::benchmark_data_smithi096_196313_object73657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2024-09-13T14:58:33.575 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.334453+0000", 2024-09-13T14:58:33.575 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 408.10892124100002, 2024-09-13T14:58:33.575 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.92167261300000003, 2024-09-13T14:58:33.575 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.575 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.575 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.576 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.576 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.576 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193788 2024-09-13T14:58:33.576 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.576 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.576 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.576 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.576 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334453+0000", 2024-09-13T14:58:33.576 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.576 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.577 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.577 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.577 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334453+0000", 2024-09-13T14:58:33.577 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.577 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.577 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.577 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.577 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334452+0000", 2024-09-13T14:58:33.577 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:33.577 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.577 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.577 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.577 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334459+0000", 2024-09-13T14:58:33.578 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1539999999999996e-06 2024-09-13T14:58:33.578 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.578 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.578 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.578 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334475+0000", 2024-09-13T14:58:33.578 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5577e-05 2024-09-13T14:58:33.578 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.578 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.578 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.578 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334482+0000", 2024-09-13T14:58:33.578 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9879999999999998e-06 2024-09-13T14:58:33.578 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.579 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.579 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.579 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334526+0000", 2024-09-13T14:58:33.579 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4279000000000001e-05 2024-09-13T14:58:33.579 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.579 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.579 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.579 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334535+0000", 2024-09-13T14:58:33.579 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.3430000000000002e-06 2024-09-13T14:58:33.579 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.579 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.579 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.579 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.237838+0000", 2024-09-13T14:58:33.580 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.90330297299999995 2024-09-13T14:58:33.580 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.580 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.580 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.580 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.237958+0000", 2024-09-13T14:58:33.580 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119401 2024-09-13T14:58:33.580 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.580 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.580 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:58:33.580 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.238068+0000", 2024-09-13T14:58:33.580 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000110418 2024-09-13T14:58:33.580 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.581 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.581 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.581 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.245761+0000", 2024-09-13T14:58:33.581 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0076925450000000003 2024-09-13T14:58:33.581 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.581 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.581 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.581 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.255890+0000", 2024-09-13T14:58:33.581 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010129712000000001 2024-09-13T14:58:33.581 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.581 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.581 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.581 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.256071+0000", 2024-09-13T14:58:33.582 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000180512 2024-09-13T14:58:33.582 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.582 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.582 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.582 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.256095+0000", 2024-09-13T14:58:33.582 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4224e-05 2024-09-13T14:58:33.582 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.582 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.582 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.582 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.256126+0000", 2024-09-13T14:58:33.582 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0917999999999998e-05 2024-09-13T14:58:33.582 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.582 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.583 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.583 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.583 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.583 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:226690 7.7 7:ee5af816:::benchmark_data_smithi096_196313_object106559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2024-09-13T14:58:33.583 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:52:31.424483+0000", 2024-09-13T14:58:33.583 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 362.01889203299999, 2024-09-13T14:58:33.583 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0532642480000001, 2024-09-13T14:58:33.583 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.583 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T14:58:33.583 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.583 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.583 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.584 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 226690 2024-09-13T14:58:33.584 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.584 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.584 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.584 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.584 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424483+0000", 2024-09-13T14:58:33.584 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.584 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.584 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.584 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.584 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424483+0000", 2024-09-13T14:58:33.584 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.585 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.585 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.585 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.585 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424482+0000", 2024-09-13T14:58:33.585 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:33.585 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.585 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.585 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.585 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424485+0000", 2024-09-13T14:58:33.585 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.3690000000000001e-06 2024-09-13T14:58:33.585 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.585 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.585 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.585 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424491+0000", 2024-09-13T14:58:33.586 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.251e-06 2024-09-13T14:58:33.586 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.586 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.586 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.586 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424495+0000", 2024-09-13T14:58:33.586 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8940000000000003e-06 2024-09-13T14:58:33.586 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.586 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.586 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.586 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424537+0000", 2024-09-13T14:58:33.586 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1993999999999998e-05 2024-09-13T14:58:33.586 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.586 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.587 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.587 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424658+0000", 2024-09-13T14:58:33.587 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000120875 2024-09-13T14:58:33.587 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.587 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.587 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,1,3", 2024-09-13T14:58:33.587 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424769+0000", 2024-09-13T14:58:33.587 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000111619 2024-09-13T14:58:33.587 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.587 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.587 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.587 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.430664+0000", 2024-09-13T14:58:33.587 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0058946800000000002 2024-09-13T14:58:33.587 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.587 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.588 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.588 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.439757+0000", 2024-09-13T14:58:33.588 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0090930149999999994 2024-09-13T14:58:33.588 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.588 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.588 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.589 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.441422+0000", 2024-09-13T14:58:33.589 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0016653869999999999 2024-09-13T14:58:33.589 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.589 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.589 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.589 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.445501+0000", 2024-09-13T14:58:33.589 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0040786319999999996 2024-09-13T14:58:33.589 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.589 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.589 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.590 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.445511+0000", 2024-09-13T14:58:33.590 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0197000000000001e-05 2024-09-13T14:58:33.590 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.590 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.590 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.590 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.477698+0000", 2024-09-13T14:58:33.590 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0321867280000001 2024-09-13T14:58:33.590 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.591 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.591 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.591 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.477747+0000", 2024-09-13T14:58:33.591 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8998000000000001e-05 2024-09-13T14:58:33.591 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.591 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.591 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.591 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.591 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.591 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:226698 7.7 7:ed1c0bd5:::benchmark_data_smithi096_196313_object106567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2024-09-13T14:58:33.592 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:52:31.443183+0000", 2024-09-13T14:58:33.592 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 362.00019169000001, 2024-09-13T14:58:33.592 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0511919729999999, 2024-09-13T14:58:33.592 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.592 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.592 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.592 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.592 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.592 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 226698 2024-09-13T14:58:33.592 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.593 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.593 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.593 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.593 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.443183+0000", 2024-09-13T14:58:33.593 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.593 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.593 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.593 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.593 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.443183+0000", 2024-09-13T14:58:33.593 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.593 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.593 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.594 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.594 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.443182+0000", 2024-09-13T14:58:33.594 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:33.594 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.594 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.594 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.594 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.443185+0000", 2024-09-13T14:58:33.595 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4839999999999998e-06 2024-09-13T14:58:33.595 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.595 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.595 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.596 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.443190+0000", 2024-09-13T14:58:33.596 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5079999999999996e-06 2024-09-13T14:58:33.596 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.596 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.596 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.596 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.443194+0000", 2024-09-13T14:58:33.597 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6389999999999998e-06 2024-09-13T14:58:33.597 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.597 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.597 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.597 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.445536+0000", 2024-09-13T14:58:33.597 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0023414709999999999 2024-09-13T14:58:33.598 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.598 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.598 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.598 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.445544+0000", 2024-09-13T14:58:33.598 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3000000000000002e-06 2024-09-13T14:58:33.598 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.599 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.599 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.599 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.478073+0000", 2024-09-13T14:58:33.599 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0325288770000001 2024-09-13T14:58:33.599 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.599 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.600 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.600 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.478143+0000", 2024-09-13T14:58:33.600 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9918000000000004e-05 2024-09-13T14:58:33.600 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.600 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.600 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,3", 2024-09-13T14:58:33.601 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.478219+0000", 2024-09-13T14:58:33.601 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6754000000000004e-05 2024-09-13T14:58:33.601 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.601 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.601 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.602 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.487596+0000", 2024-09-13T14:58:33.602 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0093767369999999996 2024-09-13T14:58:33.602 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.602 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.602 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.602 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.487601+0000", 2024-09-13T14:58:33.603 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4780000000000002e-06 2024-09-13T14:58:33.603 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.603 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.603 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.603 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494280+0000", 2024-09-13T14:58:33.604 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0066789900000000001 2024-09-13T14:58:33.604 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.604 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.604 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.604 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494334+0000", 2024-09-13T14:58:33.604 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4627999999999999e-05 2024-09-13T14:58:33.605 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.605 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.605 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.605 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494375+0000", 2024-09-13T14:58:33.605 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0593999999999998e-05 2024-09-13T14:58:33.606 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.606 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.606 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.606 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.606 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.606 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:226705 7.7 7:e639073c:::benchmark_data_smithi096_196313_object106574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-13T14:58:33.607 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:52:31.458774+0000", 2024-09-13T14:58:33.607 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 361.98460016000001, 2024-09-13T14:58:33.607 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0357834050000001, 2024-09-13T14:58:33.607 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.607 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.607 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.608 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.609 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.609 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 226705 2024-09-13T14:58:33.609 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.609 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.609 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.609 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.609 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458774+0000", 2024-09-13T14:58:33.609 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.609 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.609 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.609 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.609 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458774+0000", 2024-09-13T14:58:33.610 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.610 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.610 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.610 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.610 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458773+0000", 2024-09-13T14:58:33.610 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:33.610 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.610 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.610 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.610 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458780+0000", 2024-09-13T14:58:33.610 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7290000000000003e-06 2024-09-13T14:58:33.611 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.611 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.611 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.611 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458791+0000", 2024-09-13T14:58:33.611 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1192999999999999e-05 2024-09-13T14:58:33.611 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.611 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.612 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.612 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458797+0000", 2024-09-13T14:58:33.612 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9800000000000003e-06 2024-09-13T14:58:33.612 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.612 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.612 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.613 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458838+0000", 2024-09-13T14:58:33.613 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1474000000000003e-05 2024-09-13T14:58:33.613 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.613 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.613 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.613 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458845+0000", 2024-09-13T14:58:33.614 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7780000000000002e-06 2024-09-13T14:58:33.614 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.614 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.614 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.614 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.481450+0000", 2024-09-13T14:58:33.614 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.022604946 2024-09-13T14:58:33.614 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.615 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.615 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.615 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.481527+0000", 2024-09-13T14:58:33.615 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6643000000000005e-05 2024-09-13T14:58:33.615 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.615 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.616 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,3", 2024-09-13T14:58:33.616 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.481607+0000", 2024-09-13T14:58:33.616 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9943000000000004e-05 2024-09-13T14:58:33.616 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.616 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.617 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.617 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.487604+0000", 2024-09-13T14:58:33.617 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0059977119999999997 2024-09-13T14:58:33.617 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.617 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.617 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.617 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.487652+0000", 2024-09-13T14:58:33.618 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7268999999999998e-05 2024-09-13T14:58:33.618 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.618 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.618 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.618 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494471+0000", 2024-09-13T14:58:33.618 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0068191359999999999 2024-09-13T14:58:33.619 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.619 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.619 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.619 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494519+0000", 2024-09-13T14:58:33.619 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8155999999999998e-05 2024-09-13T14:58:33.619 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.620 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.620 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.620 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494558+0000", 2024-09-13T14:58:33.620 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8822999999999997e-05 2024-09-13T14:58:33.620 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.621 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.621 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.621 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.621 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.621 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:226760 7.7 7:ead9d959:::benchmark_data_smithi096_196313_object106629:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-13T14:58:33.621 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:52:31.515678+0000", 2024-09-13T14:58:33.622 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 361.92769627899997, 2024-09-13T14:58:33.622 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99782514, 2024-09-13T14:58:33.622 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.622 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.622 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.623 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.623 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.623 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 226760 2024-09-13T14:58:33.623 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.623 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.623 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.624 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.624 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515678+0000", 2024-09-13T14:58:33.624 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.624 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.624 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.624 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.624 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515678+0000", 2024-09-13T14:58:33.625 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.625 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.625 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.625 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.625 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515677+0000", 2024-09-13T14:58:33.625 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:33.625 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.625 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.625 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.625 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515684+0000", 2024-09-13T14:58:33.625 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.866e-06 2024-09-13T14:58:33.625 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.625 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.625 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.626 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515699+0000", 2024-09-13T14:58:33.626 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5417000000000002e-05 2024-09-13T14:58:33.626 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.626 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.626 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.626 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515706+0000", 2024-09-13T14:58:33.626 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4520000000000002e-06 2024-09-13T14:58:33.626 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.626 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.626 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.626 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515754+0000", 2024-09-13T14:58:33.626 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8773999999999998e-05 2024-09-13T14:58:33.626 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.626 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.626 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.627 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515765+0000", 2024-09-13T14:58:33.627 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0103e-05 2024-09-13T14:58:33.627 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.627 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.627 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.627 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.484331+0000", 2024-09-13T14:58:33.627 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.968566869 2024-09-13T14:58:33.627 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.627 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.627 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.627 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.484438+0000", 2024-09-13T14:58:33.627 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010631700000000001 2024-09-13T14:58:33.627 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.627 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.628 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,3", 2024-09-13T14:58:33.628 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.484551+0000", 2024-09-13T14:58:33.628 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000113021 2024-09-13T14:58:33.628 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.628 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.628 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.628 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494224+0000", 2024-09-13T14:58:33.628 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0096733240000000005 2024-09-13T14:58:33.628 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.628 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.628 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.628 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494388+0000", 2024-09-13T14:58:33.628 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000163726 2024-09-13T14:58:33.628 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.629 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.629 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.629 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.513421+0000", 2024-09-13T14:58:33.629 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.019033042 2024-09-13T14:58:33.629 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.629 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.630 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.630 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.513463+0000", 2024-09-13T14:58:33.630 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2015999999999999e-05 2024-09-13T14:58:33.630 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.630 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.630 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.630 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.513503+0000", 2024-09-13T14:58:33.630 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0602000000000002e-05 2024-09-13T14:58:33.630 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.630 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.630 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.630 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.630 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.631 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:226835 7.7 7:eb0e6d5e:::benchmark_data_smithi096_196313_object106704:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-13T14:58:33.631 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:52:31.597667+0000", 2024-09-13T14:58:33.631 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 361.84570777300002, 2024-09-13T14:58:33.631 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.92256422400000004, 2024-09-13T14:58:33.631 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:33.631 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:33.631 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:33.631 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:33.631 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:33.631 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 226835 2024-09-13T14:58:33.631 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.631 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:33.631 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.631 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:33.632 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597667+0000", 2024-09-13T14:58:33.632 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.632 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.632 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.632 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:33.632 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597667+0000", 2024-09-13T14:58:33.632 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:33.632 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.632 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.632 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:33.632 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597666+0000", 2024-09-13T14:58:33.632 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:33.632 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.633 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.633 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:33.633 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597671+0000", 2024-09-13T14:58:33.633 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3989999999999997e-06 2024-09-13T14:58:33.633 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.633 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.633 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:33.633 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597679+0000", 2024-09-13T14:58:33.633 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8910000000000001e-06 2024-09-13T14:58:33.633 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.633 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.633 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:33.634 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597683+0000", 2024-09-13T14:58:33.634 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4740000000000001e-06 2024-09-13T14:58:33.634 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.634 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.634 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.634 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597725+0000", 2024-09-13T14:58:33.634 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1904999999999999e-05 2024-09-13T14:58:33.634 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.634 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.634 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T14:58:33.634 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597732+0000", 2024-09-13T14:58:33.634 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.4869999999999996e-06 2024-09-13T14:58:33.634 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.634 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.635 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:33.635 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.487698+0000", 2024-09-13T14:58:33.635 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.88996593099999999 2024-09-13T14:58:33.635 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.635 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.635 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:33.635 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.487827+0000", 2024-09-13T14:58:33.635 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000129014 2024-09-13T14:58:33.635 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.635 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.635 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,3", 2024-09-13T14:58:33.635 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.487893+0000", 2024-09-13T14:58:33.635 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5758000000000003e-05 2024-09-13T14:58:33.635 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.635 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.636 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:33.636 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.497981+0000", 2024-09-13T14:58:33.636 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010087646 2024-09-13T14:58:33.636 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.636 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.636 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.636 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.500910+0000", 2024-09-13T14:58:33.636 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029293510000000002 2024-09-13T14:58:33.636 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.636 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.636 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:33.636 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.520141+0000", 2024-09-13T14:58:33.636 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.019230550999999999 2024-09-13T14:58:33.636 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.637 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.637 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:33.637 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.520189+0000", 2024-09-13T14:58:33.637 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8591999999999998e-05 2024-09-13T14:58:33.637 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:33.637 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:33.637 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:33.637 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.520231+0000", 2024-09-13T14:58:33.637 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1937000000000002e-05 2024-09-13T14:58:33.637 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.637 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.637 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.637 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:33.638 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:33.638 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:58:33.638 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 43 16 77927 77911 113.228 80.8125 0.00649966 0.00882608 2024-09-13T14:58:33.643 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:33.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:33.707 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:58:33.746 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:33.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.778 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:33.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:33.879 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:33.906 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:33.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:34.010 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:34.043 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:34.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:34.145 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:34.184 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:34.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:34.286 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:34.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.322 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:34.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:34.423 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:34.462 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:34.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:34.564 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:34.582 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:58:34.596 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:34.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:34.612 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 44 16 79125 79109 112.356 74.875 0.00651682 0.00889549 2024-09-13T14:58:34.697 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:34.730 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:34.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:34.832 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:34.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.874 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:34.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:34.935 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:34.976 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:35.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.009 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:35.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:35.111 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:35.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.145 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:35.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:35.247 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:35.269 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:35.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:35.371 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:35.391 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:35.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:35.493 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:35.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.533 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:35.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:35.612 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 45 16 80014 79998 111.094 55.5625 0.00501378 0.00899615 2024-09-13T14:58:35.634 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:35.662 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:35.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:35.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:35 smithi204 ceph-mon[87071]: pgmap v1012: 109 pgs: 109 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:58:35.764 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:35.803 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:35.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:35.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:35 smithi096 ceph-mon[98901]: pgmap v1012: 109 pgs: 109 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:58:35.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:35 smithi096 ceph-mon[102928]: pgmap v1012: 109 pgs: 109 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:58:35.905 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:35.942 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:35.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:36.044 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:36.080 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:36.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:36.182 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:36.211 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:36.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:36.313 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:36.349 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:36.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:36.451 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:36.479 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:36.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:36.582 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:36.614 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 46 16 81258 81242 110.369 77.75 0.0113703 0.00905551 2024-09-13T14:58:36.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.616 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:36.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:36.717 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:36.760 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:36.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:36.861 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:36.892 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:36.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:36.994 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:37.032 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:37.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:37.134 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:37.164 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:58:37.165 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:37.169 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:37.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:37.232 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:58:37.232 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:37.232 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:58:37.232 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:37.271 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:37.305 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:37.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:37.407 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:37.447 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:37.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:37.470 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:58:37.470 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:58:37.470 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.470 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:82291 9.3 9:ca40a6ba:::benchmark_data_smithi096_256909_object82290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T14:58:37.470 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:37.459084+0000", 2024-09-13T14:58:37.470 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0086836889999999996, 2024-09-13T14:58:37.470 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0087164040000000005, 2024-09-13T14:58:37.470 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:37.471 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:58:37.471 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:37.471 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:37.471 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:37.471 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 82291 2024-09-13T14:58:37.471 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.471 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:37.471 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.471 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:37.471 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459084+0000", 2024-09-13T14:58:37.471 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.471 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.471 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.472 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:37.472 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459084+0000", 2024-09-13T14:58:37.472 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.472 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.472 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.472 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:37.472 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459083+0000", 2024-09-13T14:58:37.472 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:37.472 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.472 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.472 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:37.472 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459196+0000", 2024-09-13T14:58:37.472 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000113554 2024-09-13T14:58:37.473 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.473 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.473 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:37.473 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459219+0000", 2024-09-13T14:58:37.473 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.3280000000000001e-05 2024-09-13T14:58:37.473 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.473 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.473 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:37.473 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459225+0000", 2024-09-13T14:58:37.473 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8259999999999998e-06 2024-09-13T14:58:37.473 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.473 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.473 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:37.474 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459323+0000", 2024-09-13T14:58:37.474 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7918000000000007e-05 2024-09-13T14:58:37.474 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.474 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.474 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:37.474 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459473+0000", 2024-09-13T14:58:37.474 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000149338 2024-09-13T14:58:37.474 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.474 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.474 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:37.474 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459651+0000", 2024-09-13T14:58:37.475 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00017835400000000001 2024-09-13T14:58:37.475 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.475 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.475 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:37.475 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.465549+0000", 2024-09-13T14:58:37.475 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0058982749999999997 2024-09-13T14:58:37.475 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.475 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.475 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:37.475 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.465561+0000", 2024-09-13T14:58:37.476 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2349e-05 2024-09-13T14:58:37.476 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.476 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:37.476 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.476 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.477 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.477 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:82292 9.3 9:c9b136c7:::benchmark_data_smithi096_256909_object82291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T14:58:37.477 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:37.459240+0000", 2024-09-13T14:58:37.477 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0085273480000000006, 2024-09-13T14:58:37.477 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0086473629999999999, 2024-09-13T14:58:37.477 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:37.477 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:58:37.477 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:37.477 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:37.477 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:37.477 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 82292 2024-09-13T14:58:37.477 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.477 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:37.478 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.478 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:37.478 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459240+0000", 2024-09-13T14:58:37.478 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.478 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.478 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.478 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:37.478 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459240+0000", 2024-09-13T14:58:37.478 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.478 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.478 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.478 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:37.478 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459239+0000", 2024-09-13T14:58:37.478 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:37.479 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.479 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.479 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:37.479 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459266+0000", 2024-09-13T14:58:37.479 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7229999999999998e-05 2024-09-13T14:58:37.479 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.479 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.479 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:37.479 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459273+0000", 2024-09-13T14:58:37.479 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0849999999999999e-06 2024-09-13T14:58:37.480 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.480 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.480 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:37.480 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459277+0000", 2024-09-13T14:58:37.480 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0069999999999999e-06 2024-09-13T14:58:37.480 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.480 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.480 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:37.480 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.462545+0000", 2024-09-13T14:58:37.480 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0032677330000000001 2024-09-13T14:58:37.481 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.481 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.481 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:37.481 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.462708+0000", 2024-09-13T14:58:37.481 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016300400000000001 2024-09-13T14:58:37.481 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.481 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.481 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:37.481 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.462831+0000", 2024-09-13T14:58:37.481 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012320199999999999 2024-09-13T14:58:37.481 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.481 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:37.481 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.481 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.482 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.482 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.46040.0:82290 9.1 e312/302 9:89e26d9d:::benchmark_data_smithi096_256909_object82289:head v 312'5114, mlcod=312'5112)", 2024-09-13T14:58:37.482 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:37.459835+0000", 2024-09-13T14:58:37.482 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0079325750000000007, 2024-09-13T14:58:37.482 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0081265409999999993, 2024-09-13T14:58:37.482 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:37.482 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:58:37.482 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:37.482 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.482 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:37.482 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459835+0000", 2024-09-13T14:58:37.482 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.482 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.482 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.483 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:37.483 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459835+0000", 2024-09-13T14:58:37.483 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.483 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.483 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.483 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:37.483 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.459834+0000", 2024-09-13T14:58:37.483 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:37.483 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.483 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.483 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:37.483 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.462430+0000", 2024-09-13T14:58:37.483 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0025958800000000001 2024-09-13T14:58:37.483 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.484 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.484 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:37.484 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.462461+0000", 2024-09-13T14:58:37.484 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1553000000000001e-05 2024-09-13T14:58:37.484 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.484 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.484 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:37.484 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.462470+0000", 2024-09-13T14:58:37.484 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.67e-06 2024-09-13T14:58:37.484 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.484 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.484 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:37.484 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.463303+0000", 2024-09-13T14:58:37.485 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00083275500000000002 2024-09-13T14:58:37.485 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.485 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.485 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:37.485 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.463321+0000", 2024-09-13T14:58:37.485 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8275999999999999e-05 2024-09-13T14:58:37.485 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.485 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:37.485 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.485 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.485 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.485 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.46040.0:82288 9.7 e312/302 9:e55f87b4:::benchmark_data_smithi096_256909_object82287:head v 312'5082, mlcod=312'5077)", 2024-09-13T14:58:37.486 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:37.463085+0000", 2024-09-13T14:58:37.486 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0046826489999999997, 2024-09-13T14:58:37.486 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0049320659999999997, 2024-09-13T14:58:37.486 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:37.486 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:58:37.486 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:37.486 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.487 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:37.487 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.463085+0000", 2024-09-13T14:58:37.487 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.487 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.487 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.487 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:37.487 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.463085+0000", 2024-09-13T14:58:37.487 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.487 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.487 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.487 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:37.487 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.463084+0000", 2024-09-13T14:58:37.488 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:37.488 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.488 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.488 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:37.488 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.463108+0000", 2024-09-13T14:58:37.488 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4666000000000001e-05 2024-09-13T14:58:37.488 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.488 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.488 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:37.488 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.463120+0000", 2024-09-13T14:58:37.488 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1181e-05 2024-09-13T14:58:37.488 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.488 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.489 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:37.489 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.463125+0000", 2024-09-13T14:58:37.489 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.989e-06 2024-09-13T14:58:37.489 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.489 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.489 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:37.489 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.464504+0000", 2024-09-13T14:58:37.489 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0013794479999999999 2024-09-13T14:58:37.489 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.489 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.489 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:37.489 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.464539+0000", 2024-09-13T14:58:37.489 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.502e-05 2024-09-13T14:58:37.489 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.490 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:37.490 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.490 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.490 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.490 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.46040.0:82298 9.9 e312/302 9:92805d5f:::benchmark_data_smithi096_256909_object82297:head v 312'5034, mlcod=312'5032)", 2024-09-13T14:58:37.490 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:37.464882+0000", 2024-09-13T14:58:37.490 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0028849069999999999, 2024-09-13T14:58:37.490 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0031873090000000001, 2024-09-13T14:58:37.490 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:37.490 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:58:37.490 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:37.491 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.491 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:37.491 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.464882+0000", 2024-09-13T14:58:37.491 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.491 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.491 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.491 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:37.491 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.464882+0000", 2024-09-13T14:58:37.491 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.491 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.491 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.491 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:37.491 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.464871+0000", 2024-09-13T14:58:37.492 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999886 2024-09-13T14:58:37.492 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.492 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.492 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:37.492 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.464915+0000", 2024-09-13T14:58:37.492 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4611999999999999e-05 2024-09-13T14:58:37.492 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.492 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.492 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:37.492 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.464936+0000", 2024-09-13T14:58:37.492 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0151000000000001e-05 2024-09-13T14:58:37.492 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.492 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.493 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:37.493 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.464942+0000", 2024-09-13T14:58:37.493 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.246e-06 2024-09-13T14:58:37.493 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.493 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.493 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:37.493 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.464994+0000", 2024-09-13T14:58:37.493 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1926999999999999e-05 2024-09-13T14:58:37.493 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.493 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.493 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:37.493 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.465012+0000", 2024-09-13T14:58:37.493 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8607e-05 2024-09-13T14:58:37.493 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.494 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:37.494 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.494 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.494 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.494 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:82299 9.e 9:7d79ad72:::benchmark_data_smithi096_256909_object82298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T14:58:37.494 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:37.465874+0000", 2024-09-13T14:58:37.494 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0018932269999999999, 2024-09-13T14:58:37.494 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0022506000000000002, 2024-09-13T14:58:37.494 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:37.494 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:58:37.494 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:37.494 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:37.494 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:37.494 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 82299 2024-09-13T14:58:37.495 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.495 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:37.495 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.495 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:37.495 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.465874+0000", 2024-09-13T14:58:37.495 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.495 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.495 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.495 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:37.495 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.465874+0000", 2024-09-13T14:58:37.495 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.496 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.496 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.496 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:37.496 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.465872+0000", 2024-09-13T14:58:37.496 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:58:37.496 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.496 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.496 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:37.496 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.465903+0000", 2024-09-13T14:58:37.496 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0886999999999998e-05 2024-09-13T14:58:37.496 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.497 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.497 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:37.497 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.465936+0000", 2024-09-13T14:58:37.497 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2591e-05 2024-09-13T14:58:37.497 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.497 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.497 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:37.497 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.465942+0000", 2024-09-13T14:58:37.497 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0510000000000002e-06 2024-09-13T14:58:37.497 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.497 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.497 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:37.498 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.465981+0000", 2024-09-13T14:58:37.498 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8892e-05 2024-09-13T14:58:37.498 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.498 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.498 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:37.498 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.466090+0000", 2024-09-13T14:58:37.498 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010908600000000001 2024-09-13T14:58:37.498 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.498 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.498 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:58:37.498 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.466209+0000", 2024-09-13T14:58:37.498 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011913600000000001 2024-09-13T14:58:37.498 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.498 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:37.499 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.499 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.499 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.499 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.46040.0:82289 9.7 e312/302 9:eaddff7e:::benchmark_data_smithi096_256909_object82288:head v 312'5083, mlcod=312'5077)", 2024-09-13T14:58:37.499 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:37.466841+0000", 2024-09-13T14:58:37.499 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.00092630499999999999, 2024-09-13T14:58:37.499 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001345904, 2024-09-13T14:58:37.499 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:37.499 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T14:58:37.499 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:37.499 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.499 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:37.499 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.466841+0000", 2024-09-13T14:58:37.500 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.500 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.500 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.500 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:37.500 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.466841+0000", 2024-09-13T14:58:37.500 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.501 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.501 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.501 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:37.501 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.466840+0000", 2024-09-13T14:58:37.501 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:37.501 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.501 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.501 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:37.501 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.466863+0000", 2024-09-13T14:58:37.501 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2861999999999999e-05 2024-09-13T14:58:37.501 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.501 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.501 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:37.502 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.466875+0000", 2024-09-13T14:58:37.502 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1936000000000001e-05 2024-09-13T14:58:37.502 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.502 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.502 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:37.502 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.466880+0000", 2024-09-13T14:58:37.502 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0830000000000003e-06 2024-09-13T14:58:37.502 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.502 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.502 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:37.502 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.466918+0000", 2024-09-13T14:58:37.502 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7645999999999998e-05 2024-09-13T14:58:37.502 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.503 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.503 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:37.503 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.466946+0000", 2024-09-13T14:58:37.503 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7766000000000001e-05 2024-09-13T14:58:37.503 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.503 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:37.503 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.503 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.503 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.503 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:82301 9.3 9:cfce9fbb:::benchmark_data_smithi096_256909_object82300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T14:58:37.503 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:37.467173+0000", 2024-09-13T14:58:37.503 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.000594109, 2024-09-13T14:58:37.503 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0010708, 2024-09-13T14:58:37.503 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:37.504 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:58:37.504 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:37.504 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:37.504 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:37.504 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 82301 2024-09-13T14:58:37.504 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.504 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:37.504 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.504 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:37.504 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.467173+0000", 2024-09-13T14:58:37.504 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.504 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.504 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.504 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:37.505 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.467173+0000", 2024-09-13T14:58:37.505 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:37.505 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.505 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.505 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:37.505 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.467172+0000", 2024-09-13T14:58:37.505 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:37.505 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.505 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.505 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:37.505 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.467220+0000", 2024-09-13T14:58:37.505 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7627000000000003e-05 2024-09-13T14:58:37.506 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.506 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.506 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:37.506 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.467231+0000", 2024-09-13T14:58:37.506 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1039999999999999e-05 2024-09-13T14:58:37.506 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.506 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.506 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:37.506 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.467235+0000", 2024-09-13T14:58:37.506 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2880000000000003e-06 2024-09-13T14:58:37.506 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.506 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.506 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:37.506 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.467265+0000", 2024-09-13T14:58:37.507 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9561e-05 2024-09-13T14:58:37.507 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.507 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.507 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:37.507 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.467338+0000", 2024-09-13T14:58:37.507 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.3378000000000001e-05 2024-09-13T14:58:37.507 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:37.507 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:37.507 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:37.507 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:37.467420+0000", 2024-09-13T14:58:37.507 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1374999999999999e-05 2024-09-13T14:58:37.507 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.507 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:37.507 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.508 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:37.508 INFO:teuthology.orchestra.run.smithi204.stdout: ], 2024-09-13T14:58:37.508 INFO:teuthology.orchestra.run.smithi204.stdout: "num_ops": 8 2024-09-13T14:58:37.508 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:58:37.549 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:37.580 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:37.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:37.613 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 47 16 82454 82438 109.611 74.75 0.0152554 0.0091182 2024-09-13T14:58:37.681 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:37.712 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:37.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:37.814 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:37.844 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:37.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:37.946 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:37.952 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:37.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.995 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:37.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:38.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:37 smithi096 ceph-mon[98901]: pgmap v1013: 109 pgs: 109 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:58:38.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:37 smithi096 ceph-mon[102928]: pgmap v1013: 109 pgs: 109 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:58:38.074 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:58:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:58:37] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.51.0" 2024-09-13T14:58:38.096 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:38.137 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:38.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:38.174 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:37 smithi204 ceph-mon[87071]: pgmap v1013: 109 pgs: 109 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:58:38.187 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:58:38.192 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:58:38.240 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:38.262 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:58:38.278 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:38.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:38.380 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:38.417 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:38.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:38.519 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:38.550 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:38.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:38.613 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 48 16 83634 83618 108.864 73.75 0.0105387 0.00918029 2024-09-13T14:58:38.653 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:38.666 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:38.689 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:38.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:38.791 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:38.828 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:38.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:38.930 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:38.963 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:38.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:39.065 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:39.098 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:39.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:39.200 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:39.234 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:39.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:39.267 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:58:39.336 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:39.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.384 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:39.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:39.485 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:39.529 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:39.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:39.613 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 49 16 84868 84852 108.216 77.125 0.0117244 0.00923574 2024-09-13T14:58:39.632 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:39.675 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:39.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:39.777 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:39.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.814 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:39.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:39.915 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:39.948 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:40.050 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:40.092 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:40.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:40.128 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:39 smithi204 ceph-mon[87071]: pgmap v1014: 109 pgs: 109 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-09-13T14:58:40.129 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:39 smithi204 ceph-mon[87071]: from='client.46100 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:58:40.165 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:39 smithi096 ceph-mon[98901]: pgmap v1014: 109 pgs: 109 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-09-13T14:58:40.165 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:39 smithi096 ceph-mon[98901]: from='client.46100 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:58:40.165 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:39 smithi096 ceph-mon[102928]: pgmap v1014: 109 pgs: 109 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-09-13T14:58:40.165 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:39 smithi096 ceph-mon[102928]: from='client.46100 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:58:40.195 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:40.206 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":1014,"stamp":"2024-09-13T14:58:38.075223+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5237512141,"num_objects":79873,"num_object_clones":0,"num_object_copies":239619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":79873,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":80752,"num_write_kb":5122703,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":8404818,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82977,"ondisk_log_size":82977,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21263240,"kb_used_data":15344328,"kb_used_omap":35,"kb_used_meta":5918812,"kb_avail":728697976,"statfs":{"total":767960285184,"available":746186727424,"internally_reserved":0,"allocated":15712591872,"data_stored":15695534936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36830,"internal_metadata":6060863522},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,5,1,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":843317248,"num_objects":12868,"num_object_clones":0,"num_object_copies":38604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12868,"num_write_kb":823552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003461"},"pg_stats":[{"pgid":"1.11","version":"82'100","reported_seq":215,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.363976+0000","last_change":"2024-09-13T14:56:01.088057+0000","last_active":"2024-09-13T14:58:29.363976+0000","last_peered":"2024-09-13T14:58:29.363976+0000","last_clean":"2024-09-13T14:58:29.363976+0000","last_became_active":"2024-09-13T14:55:55.203197+0000","last_became_peered":"2024-09-13T14:55:55.203197+0000","last_unstale":"2024-09-13T14:58:29.363976+0000","last_undegraded":"2024-09-13T14:58:29.363976+0000","last_fullsized":"2024-09-13T14:58:29.363976+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.087994+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.087994+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:55.677897+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":495,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.364183+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:58:29.364183+0000","last_peered":"2024-09-13T14:58:29.364183+0000","last_clean":"2024-09-13T14:58:29.364183+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:58:29.364183+0000","last_undegraded":"2024-09-13T14:58:29.364183+0000","last_fullsized":"2024-09-13T14:58:29.364183+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":278,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.319229+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:58:27.319229+0000","last_peered":"2024-09-13T14:58:27.319229+0000","last_clean":"2024-09-13T14:58:27.319229+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:58:27.319229+0000","last_undegraded":"2024-09-13T14:58:27.319229+0000","last_fullsized":"2024-09-13T14:58:27.319229+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":42,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.324711+0000","last_change":"2024-09-13T14:56:04.643672+0000","last_active":"2024-09-13T14:58:27.324711+0000","last_peered":"2024-09-13T14:58:27.324711+0000","last_clean":"2024-09-13T14:58:27.324711+0000","last_became_active":"2024-09-13T14:55:55.185995+0000","last_became_peered":"2024-09-13T14:55:55.185995+0000","last_unstale":"2024-09-13T14:58:27.324711+0000","last_undegraded":"2024-09-13T14:58:27.324711+0000","last_fullsized":"2024-09-13T14:58:27.324711+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.643614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.643614+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:36.495546+0000","scrub_duration":0.001441681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":275,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.325544+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:58:27.325544+0000","last_peered":"2024-09-13T14:58:27.325544+0000","last_clean":"2024-09-13T14:58:27.325544+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:58:27.325544+0000","last_undegraded":"2024-09-13T14:58:27.325544+0000","last_fullsized":"2024-09-13T14:58:27.325544+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":509,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.362416+0000","last_change":"2024-09-13T14:51:44.196384+0000","last_active":"2024-09-13T14:58:29.362416+0000","last_peered":"2024-09-13T14:58:29.362416+0000","last_clean":"2024-09-13T14:58:29.362416+0000","last_became_active":"2024-09-13T14:51:44.196082+0000","last_became_peered":"2024-09-13T14:51:44.196082+0000","last_unstale":"2024-09-13T14:58:29.362416+0000","last_undegraded":"2024-09-13T14:58:29.362416+0000","last_fullsized":"2024-09-13T14:58:29.362416+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:11:03.757803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":646,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.356297+0000","last_change":"2024-09-13T14:56:03.240962+0000","last_active":"2024-09-13T14:58:29.356297+0000","last_peered":"2024-09-13T14:58:29.356297+0000","last_clean":"2024-09-13T14:58:29.356297+0000","last_became_active":"2024-09-13T14:55:55.229584+0000","last_became_peered":"2024-09-13T14:55:55.229584+0000","last_unstale":"2024-09-13T14:58:29.356297+0000","last_undegraded":"2024-09-13T14:58:29.356297+0000","last_fullsized":"2024-09-13T14:58:29.356297+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.240882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.240882+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:23.556276+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":503,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.373503+0000","last_change":"2024-09-13T14:51:44.199134+0000","last_active":"2024-09-13T14:58:29.373503+0000","last_peered":"2024-09-13T14:58:29.373503+0000","last_clean":"2024-09-13T14:58:29.373503+0000","last_became_active":"2024-09-13T14:51:44.198525+0000","last_became_peered":"2024-09-13T14:51:44.198525+0000","last_unstale":"2024-09-13T14:58:29.373503+0000","last_undegraded":"2024-09-13T14:58:29.373503+0000","last_fullsized":"2024-09-13T14:58:29.373503+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:44.009127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":279,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.319109+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T14:58:27.319109+0000","last_peered":"2024-09-13T14:58:27.319109+0000","last_clean":"2024-09-13T14:58:27.319109+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T14:58:27.319109+0000","last_undegraded":"2024-09-13T14:58:27.319109+0000","last_fullsized":"2024-09-13T14:58:27.319109+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":42,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.323807+0000","last_change":"2024-09-13T14:56:03.605447+0000","last_active":"2024-09-13T14:58:27.323807+0000","last_peered":"2024-09-13T14:58:27.323807+0000","last_clean":"2024-09-13T14:58:27.323807+0000","last_became_active":"2024-09-13T14:55:55.218939+0000","last_became_peered":"2024-09-13T14:55:55.218939+0000","last_unstale":"2024-09-13T14:58:27.323807+0000","last_undegraded":"2024-09-13T14:58:27.323807+0000","last_fullsized":"2024-09-13T14:58:27.323807+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.605356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.605356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:12.467371+0000","scrub_duration":0.0020269620000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":156,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.370432+0000","last_change":"2024-09-13T14:51:44.207246+0000","last_active":"2024-09-13T14:58:29.370432+0000","last_peered":"2024-09-13T14:58:29.370432+0000","last_clean":"2024-09-13T14:58:29.370432+0000","last_became_active":"2024-09-13T14:51:44.206637+0000","last_became_peered":"2024-09-13T14:51:44.206637+0000","last_unstale":"2024-09-13T14:58:29.370432+0000","last_undegraded":"2024-09-13T14:58:29.370432+0000","last_fullsized":"2024-09-13T14:58:29.370432+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:42.020301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":106,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.336403+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T14:58:27.336403+0000","last_peered":"2024-09-13T14:58:27.336403+0000","last_clean":"2024-09-13T14:58:27.336403+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T14:58:27.336403+0000","last_undegraded":"2024-09-13T14:58:27.336403+0000","last_fullsized":"2024-09-13T14:58:27.336403+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"312'4969","reported_seq":5000,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:33.933518+0000","last_change":"2024-09-13T14:57:34.564761+0000","last_active":"2024-09-13T14:58:33.933518+0000","last_peered":"2024-09-13T14:58:33.933518+0000","last_clean":"2024-09-13T14:58:33.933518+0000","last_became_active":"2024-09-13T14:57:34.564455+0000","last_became_peered":"2024-09-13T14:57:34.564455+0000","last_unstale":"2024-09-13T14:58:33.933518+0000","last_undegraded":"2024-09-13T14:58:33.933518+0000","last_fullsized":"2024-09-13T14:58:33.933518+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":4969,"log_dups_size":0,"ondisk_log_size":4969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:06.596934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325648384,"num_objects":4969,"num_object_clones":0,"num_object_copies":14907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4969,"num_write_kb":318016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":42,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.319190+0000","last_change":"2024-09-13T14:56:01.654803+0000","last_active":"2024-09-13T14:58:27.319190+0000","last_peered":"2024-09-13T14:58:27.319190+0000","last_clean":"2024-09-13T14:58:27.319190+0000","last_became_active":"2024-09-13T14:55:55.188788+0000","last_became_peered":"2024-09-13T14:55:55.188788+0000","last_unstale":"2024-09-13T14:58:27.319190+0000","last_undegraded":"2024-09-13T14:58:27.319190+0000","last_fullsized":"2024-09-13T14:58:27.319190+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.654717+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.654717+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:38.649187+0000","scrub_duration":0.0060902070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":480,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.355679+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:58:29.355679+0000","last_peered":"2024-09-13T14:58:29.355679+0000","last_clean":"2024-09-13T14:58:29.355679+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:58:29.355679+0000","last_undegraded":"2024-09-13T14:58:29.355679+0000","last_fullsized":"2024-09-13T14:58:29.355679+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":298,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.324848+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T14:58:27.324848+0000","last_peered":"2024-09-13T14:58:27.324848+0000","last_clean":"2024-09-13T14:58:27.324848+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T14:58:27.324848+0000","last_undegraded":"2024-09-13T14:58:27.324848+0000","last_fullsized":"2024-09-13T14:58:27.324848+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"312'4636","reported_seq":4667,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:33.531224+0000","last_change":"2024-09-13T14:57:34.842329+0000","last_active":"2024-09-13T14:58:33.531224+0000","last_peered":"2024-09-13T14:58:33.531224+0000","last_clean":"2024-09-13T14:58:33.531224+0000","last_became_active":"2024-09-13T14:57:34.842023+0000","last_became_peered":"2024-09-13T14:57:34.842023+0000","last_unstale":"2024-09-13T14:58:33.531224+0000","last_undegraded":"2024-09-13T14:58:33.531224+0000","last_fullsized":"2024-09-13T14:58:33.531224+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":4636,"log_dups_size":0,"ondisk_log_size":4636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:43:26.434623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303824896,"num_objects":4636,"num_object_clones":0,"num_object_copies":13908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4636,"num_write_kb":296704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":459,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.363747+0000","last_change":"2024-09-13T14:56:01.843753+0000","last_active":"2024-09-13T14:58:29.363747+0000","last_peered":"2024-09-13T14:58:29.363747+0000","last_clean":"2024-09-13T14:58:29.363747+0000","last_became_active":"2024-09-13T14:55:55.188079+0000","last_became_peered":"2024-09-13T14:55:55.188079+0000","last_unstale":"2024-09-13T14:58:29.363747+0000","last_undegraded":"2024-09-13T14:58:29.363747+0000","last_fullsized":"2024-09-13T14:58:29.363747+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.843498+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.843498+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:54.013271+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":278,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.318687+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:58:27.318687+0000","last_peered":"2024-09-13T14:58:27.318687+0000","last_clean":"2024-09-13T14:58:27.318687+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:58:27.318687+0000","last_undegraded":"2024-09-13T14:58:27.318687+0000","last_fullsized":"2024-09-13T14:58:27.318687+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":333,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.333882+0000","last_change":"2024-09-13T14:51:11.059483+0000","last_active":"2024-09-13T14:58:27.333882+0000","last_peered":"2024-09-13T14:58:27.333882+0000","last_clean":"2024-09-13T14:58:27.333882+0000","last_became_active":"2024-09-13T14:51:11.059262+0000","last_became_peered":"2024-09-13T14:51:11.059262+0000","last_unstale":"2024-09-13T14:58:27.333882+0000","last_undegraded":"2024-09-13T14:58:27.333882+0000","last_fullsized":"2024-09-13T14:58:27.333882+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:45:35.152631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"312'5075","reported_seq":5093,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:37.218461+0000","last_change":"2024-09-13T14:57:34.560948+0000","last_active":"2024-09-13T14:58:37.218461+0000","last_peered":"2024-09-13T14:58:37.218461+0000","last_clean":"2024-09-13T14:58:37.218461+0000","last_became_active":"2024-09-13T14:57:34.560657+0000","last_became_peered":"2024-09-13T14:57:34.560657+0000","last_unstale":"2024-09-13T14:58:37.218461+0000","last_undegraded":"2024-09-13T14:58:37.218461+0000","last_fullsized":"2024-09-13T14:58:37.218461+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5075,"log_dups_size":0,"ondisk_log_size":5075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:21:56.540221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332595200,"num_objects":5075,"num_object_clones":0,"num_object_copies":15225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5075,"num_write_kb":324800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":145,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.336402+0000","last_change":"2024-09-13T14:56:08.889974+0000","last_active":"2024-09-13T14:58:27.336402+0000","last_peered":"2024-09-13T14:58:27.336402+0000","last_clean":"2024-09-13T14:58:27.336402+0000","last_became_active":"2024-09-13T14:55:55.185755+0000","last_became_peered":"2024-09-13T14:55:55.185755+0000","last_unstale":"2024-09-13T14:58:27.336402+0000","last_undegraded":"2024-09-13T14:58:27.336402+0000","last_fullsized":"2024-09-13T14:58:27.336402+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:08.889814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:08.889814+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:02.911564+0000","scrub_duration":0.0053731569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":288,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.319408+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T14:58:27.319408+0000","last_peered":"2024-09-13T14:58:27.319408+0000","last_clean":"2024-09-13T14:58:27.319408+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T14:58:27.319408+0000","last_undegraded":"2024-09-13T14:58:27.319408+0000","last_fullsized":"2024-09-13T14:58:27.319408+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":106,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.337360+0000","last_change":"2024-09-13T14:51:44.190879+0000","last_active":"2024-09-13T14:58:27.337360+0000","last_peered":"2024-09-13T14:58:27.337360+0000","last_clean":"2024-09-13T14:58:27.337360+0000","last_became_active":"2024-09-13T14:51:44.190617+0000","last_became_peered":"2024-09-13T14:51:44.190617+0000","last_unstale":"2024-09-13T14:58:27.337360+0000","last_undegraded":"2024-09-13T14:58:27.337360+0000","last_fullsized":"2024-09-13T14:58:27.337360+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:19.536298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"312'5225","reported_seq":5242,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:37.280381+0000","last_change":"2024-09-13T14:57:34.556168+0000","last_active":"2024-09-13T14:58:37.280381+0000","last_peered":"2024-09-13T14:58:37.280381+0000","last_clean":"2024-09-13T14:58:37.280381+0000","last_became_active":"2024-09-13T14:57:34.555882+0000","last_became_peered":"2024-09-13T14:57:34.555882+0000","last_unstale":"2024-09-13T14:58:37.280381+0000","last_undegraded":"2024-09-13T14:58:37.280381+0000","last_fullsized":"2024-09-13T14:58:37.280381+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5225,"log_dups_size":0,"ondisk_log_size":5225,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:05:49.922717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342425600,"num_objects":5225,"num_object_clones":0,"num_object_copies":15675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5225,"num_write_kb":334400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":42,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.335474+0000","last_change":"2024-09-13T14:56:02.325723+0000","last_active":"2024-09-13T14:58:27.335474+0000","last_peered":"2024-09-13T14:58:27.335474+0000","last_clean":"2024-09-13T14:58:27.335474+0000","last_became_active":"2024-09-13T14:55:55.182743+0000","last_became_peered":"2024-09-13T14:55:55.182743+0000","last_unstale":"2024-09-13T14:58:27.335474+0000","last_undegraded":"2024-09-13T14:58:27.335474+0000","last_fullsized":"2024-09-13T14:58:27.335474+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:02.325629+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:02.325629+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:47.868641+0000","scrub_duration":0.0049176200000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":296,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.323809+0000","last_change":"2024-09-13T14:48:32.206148+0000","last_active":"2024-09-13T14:58:27.323809+0000","last_peered":"2024-09-13T14:58:27.323809+0000","last_clean":"2024-09-13T14:58:27.323809+0000","last_became_active":"2024-09-13T14:48:32.205832+0000","last_became_peered":"2024-09-13T14:48:32.205832+0000","last_unstale":"2024-09-13T14:58:27.323809+0000","last_undegraded":"2024-09-13T14:58:27.323809+0000","last_fullsized":"2024-09-13T14:58:27.323809+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:55.026094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":490,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.366526+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:58:29.366526+0000","last_peered":"2024-09-13T14:58:29.366526+0000","last_clean":"2024-09-13T14:58:29.366526+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:58:29.366526+0000","last_undegraded":"2024-09-13T14:58:29.366526+0000","last_fullsized":"2024-09-13T14:58:29.366526+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"312'4848","reported_seq":4879,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:33.909117+0000","last_change":"2024-09-13T14:57:34.835792+0000","last_active":"2024-09-13T14:58:33.909117+0000","last_peered":"2024-09-13T14:58:33.909117+0000","last_clean":"2024-09-13T14:58:33.909117+0000","last_became_active":"2024-09-13T14:57:34.835446+0000","last_became_peered":"2024-09-13T14:57:34.835446+0000","last_unstale":"2024-09-13T14:58:33.909117+0000","last_undegraded":"2024-09-13T14:58:33.909117+0000","last_fullsized":"2024-09-13T14:58:33.909117+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":4848,"log_dups_size":0,"ondisk_log_size":4848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:48.872041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317718528,"num_objects":4848,"num_object_clones":0,"num_object_copies":14544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4848,"num_write_kb":310272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":459,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.318983+0000","last_change":"2024-09-13T14:51:01.996634+0000","last_active":"2024-09-13T14:58:27.318983+0000","last_peered":"2024-09-13T14:58:27.318983+0000","last_clean":"2024-09-13T14:58:27.318983+0000","last_became_active":"2024-09-13T14:51:01.996264+0000","last_became_peered":"2024-09-13T14:51:01.996264+0000","last_unstale":"2024-09-13T14:58:27.318983+0000","last_undegraded":"2024-09-13T14:58:27.318983+0000","last_fullsized":"2024-09-13T14:58:27.318983+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:30.083115+0000","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":493,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.363093+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:58:29.363093+0000","last_peered":"2024-09-13T14:58:29.363093+0000","last_clean":"2024-09-13T14:58:29.363093+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:58:29.363093+0000","last_undegraded":"2024-09-13T14:58:29.363093+0000","last_fullsized":"2024-09-13T14:58:29.363093+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":296,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.333047+0000","last_change":"2024-09-13T14:48:32.196993+0000","last_active":"2024-09-13T14:58:27.333047+0000","last_peered":"2024-09-13T14:58:27.333047+0000","last_clean":"2024-09-13T14:58:27.333047+0000","last_became_active":"2024-09-13T14:48:32.196681+0000","last_became_peered":"2024-09-13T14:48:32.196681+0000","last_unstale":"2024-09-13T14:58:27.333047+0000","last_undegraded":"2024-09-13T14:58:27.333047+0000","last_fullsized":"2024-09-13T14:58:27.333047+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:56.040494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"312'4936","reported_seq":4967,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:33.544032+0000","last_change":"2024-09-13T14:57:34.835630+0000","last_active":"2024-09-13T14:58:33.544032+0000","last_peered":"2024-09-13T14:58:33.544032+0000","last_clean":"2024-09-13T14:58:33.544032+0000","last_became_active":"2024-09-13T14:57:34.835209+0000","last_became_peered":"2024-09-13T14:57:34.835209+0000","last_unstale":"2024-09-13T14:58:33.544032+0000","last_undegraded":"2024-09-13T14:58:33.544032+0000","last_fullsized":"2024-09-13T14:58:33.544032+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":4936,"log_dups_size":0,"ondisk_log_size":4936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:50.159454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323485696,"num_objects":4936,"num_object_clones":0,"num_object_copies":14808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4936,"num_write_kb":315904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":219,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.324706+0000","last_change":"2024-09-13T14:40:27.868402+0000","last_active":"2024-09-13T14:58:27.324706+0000","last_peered":"2024-09-13T14:58:27.324706+0000","last_clean":"2024-09-13T14:58:27.324706+0000","last_became_active":"2024-09-13T14:40:22.966301+0000","last_became_peered":"2024-09-13T14:40:22.966301+0000","last_unstale":"2024-09-13T14:58:27.324706+0000","last_undegraded":"2024-09-13T14:58:27.324706+0000","last_fullsized":"2024-09-13T14:58:27.324706+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:18.660186+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":96,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.337466+0000","last_change":"2024-09-13T14:48:32.200521+0000","last_active":"2024-09-13T14:58:27.337466+0000","last_peered":"2024-09-13T14:58:27.337466+0000","last_clean":"2024-09-13T14:58:27.337466+0000","last_became_active":"2024-09-13T14:48:32.200256+0000","last_became_peered":"2024-09-13T14:48:32.200256+0000","last_unstale":"2024-09-13T14:58:27.337466+0000","last_undegraded":"2024-09-13T14:58:27.337466+0000","last_fullsized":"2024-09-13T14:58:27.337466+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:38.899629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":293,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.333703+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:58:27.333703+0000","last_peered":"2024-09-13T14:58:27.333703+0000","last_clean":"2024-09-13T14:58:27.333703+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:58:27.333703+0000","last_undegraded":"2024-09-13T14:58:27.333703+0000","last_fullsized":"2024-09-13T14:58:27.333703+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"312'5107","reported_seq":5125,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:37.049227+0000","last_change":"2024-09-13T14:57:34.563262+0000","last_active":"2024-09-13T14:58:37.049227+0000","last_peered":"2024-09-13T14:58:37.049227+0000","last_clean":"2024-09-13T14:58:37.049227+0000","last_became_active":"2024-09-13T14:57:34.563081+0000","last_became_peered":"2024-09-13T14:57:34.563081+0000","last_unstale":"2024-09-13T14:58:37.049227+0000","last_undegraded":"2024-09-13T14:58:37.049227+0000","last_fullsized":"2024-09-13T14:58:37.049227+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5107,"log_dups_size":0,"ondisk_log_size":5107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:40.861229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334692352,"num_objects":5107,"num_object_clones":0,"num_object_copies":15321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5107,"num_write_kb":326848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":646,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.356559+0000","last_change":"2024-09-13T14:56:01.254863+0000","last_active":"2024-09-13T14:58:29.356559+0000","last_peered":"2024-09-13T14:58:29.356559+0000","last_clean":"2024-09-13T14:58:29.356559+0000","last_became_active":"2024-09-13T14:55:55.214258+0000","last_became_peered":"2024-09-13T14:55:55.214258+0000","last_unstale":"2024-09-13T14:58:29.356559+0000","last_undegraded":"2024-09-13T14:58:29.356559+0000","last_fullsized":"2024-09-13T14:58:29.356559+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.254778+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.254778+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:26.654454+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":476,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.363851+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:58:29.363851+0000","last_peered":"2024-09-13T14:58:29.363851+0000","last_clean":"2024-09-13T14:58:29.363851+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:58:29.363851+0000","last_undegraded":"2024-09-13T14:58:29.363851+0000","last_fullsized":"2024-09-13T14:58:29.363851+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":296,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.335574+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T14:58:27.335574+0000","last_peered":"2024-09-13T14:58:27.335574+0000","last_clean":"2024-09-13T14:58:27.335574+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T14:58:27.335574+0000","last_undegraded":"2024-09-13T14:58:27.335574+0000","last_fullsized":"2024-09-13T14:58:27.335574+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"312'4859","reported_seq":4890,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:33.921846+0000","last_change":"2024-09-13T14:57:34.562198+0000","last_active":"2024-09-13T14:58:33.921846+0000","last_peered":"2024-09-13T14:58:33.921846+0000","last_clean":"2024-09-13T14:58:33.921846+0000","last_became_active":"2024-09-13T14:57:34.561893+0000","last_became_peered":"2024-09-13T14:57:34.561893+0000","last_unstale":"2024-09-13T14:58:33.921846+0000","last_undegraded":"2024-09-13T14:58:33.921846+0000","last_fullsized":"2024-09-13T14:58:33.921846+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":4859,"log_dups_size":0,"ondisk_log_size":4859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:31:18.516029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318439424,"num_objects":4859,"num_object_clones":0,"num_object_copies":14577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4859,"num_write_kb":310976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":222,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.323676+0000","last_change":"2024-09-13T14:48:32.205288+0000","last_active":"2024-09-13T14:58:27.323676+0000","last_peered":"2024-09-13T14:58:27.323676+0000","last_clean":"2024-09-13T14:58:27.323676+0000","last_became_active":"2024-09-13T14:48:32.204657+0000","last_became_peered":"2024-09-13T14:48:32.204657+0000","last_unstale":"2024-09-13T14:58:27.323676+0000","last_undegraded":"2024-09-13T14:58:27.323676+0000","last_fullsized":"2024-09-13T14:58:27.323676+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:52.513329+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":149,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.362718+0000","last_change":"2024-09-13T14:51:11.059644+0000","last_active":"2024-09-13T14:58:29.362718+0000","last_peered":"2024-09-13T14:58:29.362718+0000","last_clean":"2024-09-13T14:58:29.362718+0000","last_became_active":"2024-09-13T14:51:11.058874+0000","last_became_peered":"2024-09-13T14:51:11.058874+0000","last_unstale":"2024-09-13T14:58:29.362718+0000","last_undegraded":"2024-09-13T14:58:29.362718+0000","last_fullsized":"2024-09-13T14:58:29.362718+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:15:14.736476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":477,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.355598+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:58:29.355598+0000","last_peered":"2024-09-13T14:58:29.355598+0000","last_clean":"2024-09-13T14:58:29.355598+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:58:29.355598+0000","last_undegraded":"2024-09-13T14:58:29.355598+0000","last_fullsized":"2024-09-13T14:58:29.355598+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"312'4945","reported_seq":4976,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:33.902713+0000","last_change":"2024-09-13T14:57:34.562334+0000","last_active":"2024-09-13T14:58:33.902713+0000","last_peered":"2024-09-13T14:58:33.902713+0000","last_clean":"2024-09-13T14:58:33.902713+0000","last_became_active":"2024-09-13T14:57:34.562006+0000","last_became_peered":"2024-09-13T14:57:34.562006+0000","last_unstale":"2024-09-13T14:58:33.902713+0000","last_undegraded":"2024-09-13T14:58:33.902713+0000","last_fullsized":"2024-09-13T14:58:33.902713+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":4945,"log_dups_size":0,"ondisk_log_size":4945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:48.077892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324075520,"num_objects":4945,"num_object_clones":0,"num_object_copies":14835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4945,"num_write_kb":316480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":460,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.319157+0000","last_change":"2024-09-13T14:55:58.568852+0000","last_active":"2024-09-13T14:58:27.319157+0000","last_peered":"2024-09-13T14:58:27.319157+0000","last_clean":"2024-09-13T14:58:27.319157+0000","last_became_active":"2024-09-13T14:55:51.169987+0000","last_became_peered":"2024-09-13T14:55:51.169987+0000","last_unstale":"2024-09-13T14:58:27.319157+0000","last_undegraded":"2024-09-13T14:58:27.319157+0000","last_fullsized":"2024-09-13T14:58:27.319157+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.568750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.568750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:56.273429+0000","scrub_duration":0.0074886340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":838,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.323998+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:58:27.323998+0000","last_peered":"2024-09-13T14:58:27.323998+0000","last_clean":"2024-09-13T14:58:27.323998+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:58:27.323998+0000","last_undegraded":"2024-09-13T14:58:27.323998+0000","last_fullsized":"2024-09-13T14:58:27.323998+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":149,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.363121+0000","last_change":"2024-09-13T14:51:11.059953+0000","last_active":"2024-09-13T14:58:29.363121+0000","last_peered":"2024-09-13T14:58:29.363121+0000","last_clean":"2024-09-13T14:58:29.363121+0000","last_became_active":"2024-09-13T14:51:11.059396+0000","last_became_peered":"2024-09-13T14:51:11.059396+0000","last_unstale":"2024-09-13T14:58:29.363121+0000","last_undegraded":"2024-09-13T14:58:29.363121+0000","last_fullsized":"2024-09-13T14:58:29.363121+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:11:51.119142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":96,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.336710+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T14:58:27.336710+0000","last_peered":"2024-09-13T14:58:27.336710+0000","last_clean":"2024-09-13T14:58:27.336710+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T14:58:27.336710+0000","last_undegraded":"2024-09-13T14:58:27.336710+0000","last_fullsized":"2024-09-13T14:58:27.336710+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"312'4885","reported_seq":4916,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:33.855590+0000","last_change":"2024-09-13T14:57:34.836468+0000","last_active":"2024-09-13T14:58:33.855590+0000","last_peered":"2024-09-13T14:58:33.855590+0000","last_clean":"2024-09-13T14:58:33.855590+0000","last_became_active":"2024-09-13T14:57:34.836144+0000","last_became_peered":"2024-09-13T14:57:34.836144+0000","last_unstale":"2024-09-13T14:58:33.855590+0000","last_undegraded":"2024-09-13T14:58:33.855590+0000","last_fullsized":"2024-09-13T14:58:33.855590+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":4885,"log_dups_size":0,"ondisk_log_size":4885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:40:48.019738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320143360,"num_objects":4885,"num_object_clones":0,"num_object_copies":14655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4885,"num_write_kb":312640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":640,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.353896+0000","last_change":"2024-09-13T14:55:59.245841+0000","last_active":"2024-09-13T14:58:29.353896+0000","last_peered":"2024-09-13T14:58:29.353896+0000","last_clean":"2024-09-13T14:58:29.353896+0000","last_became_active":"2024-09-13T14:55:51.159487+0000","last_became_peered":"2024-09-13T14:55:51.159487+0000","last_unstale":"2024-09-13T14:58:29.353896+0000","last_undegraded":"2024-09-13T14:58:29.353896+0000","last_fullsized":"2024-09-13T14:58:29.353896+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.245213+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.245213+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:58:06.592954+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":523,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.319072+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:58:27.319072+0000","last_peered":"2024-09-13T14:58:27.319072+0000","last_clean":"2024-09-13T14:58:27.319072+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:58:27.319072+0000","last_undegraded":"2024-09-13T14:58:27.319072+0000","last_fullsized":"2024-09-13T14:58:27.319072+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":511,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.363888+0000","last_change":"2024-09-13T14:51:11.061677+0000","last_active":"2024-09-13T14:58:29.363888+0000","last_peered":"2024-09-13T14:58:29.363888+0000","last_clean":"2024-09-13T14:58:29.363888+0000","last_became_active":"2024-09-13T14:51:11.061205+0000","last_became_peered":"2024-09-13T14:51:11.061205+0000","last_unstale":"2024-09-13T14:58:29.363888+0000","last_undegraded":"2024-09-13T14:58:29.363888+0000","last_fullsized":"2024-09-13T14:58:29.363888+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:38.521784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":515,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.373723+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T14:58:29.373723+0000","last_peered":"2024-09-13T14:58:29.373723+0000","last_clean":"2024-09-13T14:58:29.373723+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T14:58:29.373723+0000","last_undegraded":"2024-09-13T14:58:29.373723+0000","last_fullsized":"2024-09-13T14:58:29.373723+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"312'5208","reported_seq":5226,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:37.285063+0000","last_change":"2024-09-13T14:57:34.559226+0000","last_active":"2024-09-13T14:58:37.285063+0000","last_peered":"2024-09-13T14:58:37.285063+0000","last_clean":"2024-09-13T14:58:37.285063+0000","last_became_active":"2024-09-13T14:57:34.558860+0000","last_became_peered":"2024-09-13T14:57:34.558860+0000","last_unstale":"2024-09-13T14:58:37.285063+0000","last_undegraded":"2024-09-13T14:58:37.285063+0000","last_fullsized":"2024-09-13T14:58:37.285063+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5208,"log_dups_size":0,"ondisk_log_size":5208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:56:32.571916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341311488,"num_objects":5208,"num_object_clones":0,"num_object_copies":15624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5208,"num_write_kb":333312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":89,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.318735+0000","last_change":"2024-09-13T14:55:59.574827+0000","last_active":"2024-09-13T14:58:27.318735+0000","last_peered":"2024-09-13T14:58:27.318735+0000","last_clean":"2024-09-13T14:58:27.318735+0000","last_became_active":"2024-09-13T14:55:51.162903+0000","last_became_peered":"2024-09-13T14:55:51.162903+0000","last_unstale":"2024-09-13T14:58:27.318735+0000","last_undegraded":"2024-09-13T14:58:27.318735+0000","last_fullsized":"2024-09-13T14:58:27.318735+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.574773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.574773+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:39.074072+0000","scrub_duration":0.001553708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":839,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.363410+0000","last_change":"2024-09-13T14:56:41.406888+0000","last_active":"2024-09-13T14:58:29.363410+0000","last_peered":"2024-09-13T14:58:29.363410+0000","last_clean":"2024-09-13T14:58:29.363410+0000","last_became_active":"2024-09-13T14:56:41.406299+0000","last_became_peered":"2024-09-13T14:56:41.406299+0000","last_unstale":"2024-09-13T14:58:29.363410+0000","last_undegraded":"2024-09-13T14:58:29.363410+0000","last_fullsized":"2024-09-13T14:58:29.363410+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:53:00.535322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":492,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.353728+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T14:58:29.353728+0000","last_peered":"2024-09-13T14:58:29.353728+0000","last_clean":"2024-09-13T14:58:29.353728+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T14:58:29.353728+0000","last_undegraded":"2024-09-13T14:58:29.353728+0000","last_fullsized":"2024-09-13T14:58:29.353728+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":510,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.355500+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:58:29.355500+0000","last_peered":"2024-09-13T14:58:29.355500+0000","last_clean":"2024-09-13T14:58:29.355500+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:58:29.355500+0000","last_undegraded":"2024-09-13T14:58:29.355500+0000","last_fullsized":"2024-09-13T14:58:29.355500+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"312'4797","reported_seq":4826,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:33.940146+0000","last_change":"2024-09-13T14:57:34.564910+0000","last_active":"2024-09-13T14:58:33.940146+0000","last_peered":"2024-09-13T14:58:33.940146+0000","last_clean":"2024-09-13T14:58:33.940146+0000","last_became_active":"2024-09-13T14:57:34.564633+0000","last_became_peered":"2024-09-13T14:57:34.564633+0000","last_unstale":"2024-09-13T14:58:33.940146+0000","last_undegraded":"2024-09-13T14:58:33.940146+0000","last_fullsized":"2024-09-13T14:58:33.940146+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":4797,"log_dups_size":0,"ondisk_log_size":4797,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:53:38.983345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314376192,"num_objects":4797,"num_object_clones":0,"num_object_copies":14391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4797,"num_write_kb":307008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":232,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.323812+0000","last_change":"2024-09-13T14:56:01.658240+0000","last_active":"2024-09-13T14:58:27.323812+0000","last_peered":"2024-09-13T14:58:27.323812+0000","last_clean":"2024-09-13T14:58:27.323812+0000","last_became_active":"2024-09-13T14:55:51.160480+0000","last_became_peered":"2024-09-13T14:55:51.160480+0000","last_unstale":"2024-09-13T14:58:27.323812+0000","last_undegraded":"2024-09-13T14:58:27.323812+0000","last_fullsized":"2024-09-13T14:58:27.323812+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.658143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.658143+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:19.262955+0000","scrub_duration":0.0038903079999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":938,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.323946+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T14:58:27.323946+0000","last_peered":"2024-09-13T14:58:27.323946+0000","last_clean":"2024-09-13T14:58:27.323946+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T14:58:27.323946+0000","last_undegraded":"2024-09-13T14:58:27.323946+0000","last_fullsized":"2024-09-13T14:58:27.323946+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":297,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.333304+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T14:58:27.333304+0000","last_peered":"2024-09-13T14:58:27.333304+0000","last_clean":"2024-09-13T14:58:27.333304+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T14:58:27.333304+0000","last_undegraded":"2024-09-13T14:58:27.333304+0000","last_fullsized":"2024-09-13T14:58:27.333304+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":510,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.352560+0000","last_change":"2024-09-13T14:51:11.060994+0000","last_active":"2024-09-13T14:58:29.352560+0000","last_peered":"2024-09-13T14:58:29.352560+0000","last_clean":"2024-09-13T14:58:29.352560+0000","last_became_active":"2024-09-13T14:51:11.060666+0000","last_became_peered":"2024-09-13T14:51:11.060666+0000","last_unstale":"2024-09-13T14:58:29.352560+0000","last_undegraded":"2024-09-13T14:58:29.352560+0000","last_fullsized":"2024-09-13T14:58:29.352560+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:40:15.828766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"312'5119","reported_seq":5137,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:37.269785+0000","last_change":"2024-09-13T14:57:34.560777+0000","last_active":"2024-09-13T14:58:37.269785+0000","last_peered":"2024-09-13T14:58:37.269785+0000","last_clean":"2024-09-13T14:58:37.269785+0000","last_became_active":"2024-09-13T14:57:34.560516+0000","last_became_peered":"2024-09-13T14:57:34.560516+0000","last_unstale":"2024-09-13T14:58:37.269785+0000","last_undegraded":"2024-09-13T14:58:37.269785+0000","last_fullsized":"2024-09-13T14:58:37.269785+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5119,"log_dups_size":0,"ondisk_log_size":5119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:06.505880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335478784,"num_objects":5119,"num_object_clones":0,"num_object_copies":15357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5119,"num_write_kb":327616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":136,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.336306+0000","last_change":"2024-09-13T14:56:01.916851+0000","last_active":"2024-09-13T14:58:27.336306+0000","last_peered":"2024-09-13T14:58:27.336306+0000","last_clean":"2024-09-13T14:58:27.336306+0000","last_became_active":"2024-09-13T14:55:51.195404+0000","last_became_peered":"2024-09-13T14:55:51.195404+0000","last_unstale":"2024-09-13T14:58:27.336306+0000","last_undegraded":"2024-09-13T14:58:27.336306+0000","last_fullsized":"2024-09-13T14:58:27.336306+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.916764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.916764+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:47:35.114766+0000","scrub_duration":0.0035298819999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":293,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.325618+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:58:27.325618+0000","last_peered":"2024-09-13T14:58:27.325618+0000","last_clean":"2024-09-13T14:58:27.325618+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:58:27.325618+0000","last_undegraded":"2024-09-13T14:58:27.325618+0000","last_fullsized":"2024-09-13T14:58:27.325618+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":995,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.352738+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:58:29.352738+0000","last_peered":"2024-09-13T14:58:29.352738+0000","last_clean":"2024-09-13T14:58:29.352738+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:58:29.352738+0000","last_undegraded":"2024-09-13T14:58:29.352738+0000","last_fullsized":"2024-09-13T14:58:29.352738+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":467,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.362017+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:58:29.362017+0000","last_peered":"2024-09-13T14:58:29.362017+0000","last_clean":"2024-09-13T14:58:29.362017+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:58:29.362017+0000","last_undegraded":"2024-09-13T14:58:29.362017+0000","last_fullsized":"2024-09-13T14:58:29.362017+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"312'5066","reported_seq":5084,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:37.063543+0000","last_change":"2024-09-13T14:57:34.841879+0000","last_active":"2024-09-13T14:58:37.063543+0000","last_peered":"2024-09-13T14:58:37.063543+0000","last_clean":"2024-09-13T14:58:37.063543+0000","last_became_active":"2024-09-13T14:57:34.841546+0000","last_became_peered":"2024-09-13T14:57:34.841546+0000","last_unstale":"2024-09-13T14:58:37.063543+0000","last_undegraded":"2024-09-13T14:58:37.063543+0000","last_fullsized":"2024-09-13T14:58:37.063543+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5066,"log_dups_size":0,"ondisk_log_size":5066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:36.453841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332005376,"num_objects":5066,"num_object_clones":0,"num_object_copies":15198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5066,"num_write_kb":324224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":126,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.337038+0000","last_change":"2024-09-13T14:56:04.997125+0000","last_active":"2024-09-13T14:58:27.337038+0000","last_peered":"2024-09-13T14:58:27.337038+0000","last_clean":"2024-09-13T14:58:27.337038+0000","last_became_active":"2024-09-13T14:55:51.164957+0000","last_became_peered":"2024-09-13T14:55:51.164957+0000","last_unstale":"2024-09-13T14:58:27.337038+0000","last_undegraded":"2024-09-13T14:58:27.337038+0000","last_fullsized":"2024-09-13T14:58:27.337038+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.997001+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.997001+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:58:48.818249+0000","scrub_duration":0.019401547000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":492,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.352649+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T14:58:29.352649+0000","last_peered":"2024-09-13T14:58:29.352649+0000","last_clean":"2024-09-13T14:58:29.352649+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T14:58:29.352649+0000","last_undegraded":"2024-09-13T14:58:29.352649+0000","last_fullsized":"2024-09-13T14:58:29.352649+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":525,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.319450+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:58:27.319450+0000","last_peered":"2024-09-13T14:58:27.319450+0000","last_clean":"2024-09-13T14:58:27.319450+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:58:27.319450+0000","last_undegraded":"2024-09-13T14:58:27.319450+0000","last_fullsized":"2024-09-13T14:58:27.319450+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":479,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.366408+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:58:29.366408+0000","last_peered":"2024-09-13T14:58:29.366408+0000","last_clean":"2024-09-13T14:58:29.366408+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:58:29.366408+0000","last_undegraded":"2024-09-13T14:58:29.366408+0000","last_fullsized":"2024-09-13T14:58:29.366408+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"312'5124","reported_seq":5142,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:37.032507+0000","last_change":"2024-09-13T14:57:34.562351+0000","last_active":"2024-09-13T14:58:37.032507+0000","last_peered":"2024-09-13T14:58:37.032507+0000","last_clean":"2024-09-13T14:58:37.032507+0000","last_became_active":"2024-09-13T14:57:34.561412+0000","last_became_peered":"2024-09-13T14:57:34.561412+0000","last_unstale":"2024-09-13T14:58:37.032507+0000","last_undegraded":"2024-09-13T14:58:37.032507+0000","last_fullsized":"2024-09-13T14:58:37.032507+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5124,"log_dups_size":0,"ondisk_log_size":5124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:37:07.655839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335806464,"num_objects":5124,"num_object_clones":0,"num_object_copies":15372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5124,"num_write_kb":327936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":263,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.333405+0000","last_change":"2024-09-13T14:55:58.308418+0000","last_active":"2024-09-13T14:58:27.333405+0000","last_peered":"2024-09-13T14:58:27.333405+0000","last_clean":"2024-09-13T14:58:27.333405+0000","last_became_active":"2024-09-13T14:55:51.160771+0000","last_became_peered":"2024-09-13T14:55:51.160771+0000","last_unstale":"2024-09-13T14:58:27.333405+0000","last_undegraded":"2024-09-13T14:58:27.333405+0000","last_fullsized":"2024-09-13T14:58:27.333405+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.308240+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.308240+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:05.665088+0000","scrub_duration":0.0063105979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":493,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.370523+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:58:29.370523+0000","last_peered":"2024-09-13T14:58:29.370523+0000","last_clean":"2024-09-13T14:58:29.370523+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:58:29.370523+0000","last_undegraded":"2024-09-13T14:58:29.370523+0000","last_fullsized":"2024-09-13T14:58:29.370523+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":849,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.354426+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T14:58:29.354426+0000","last_peered":"2024-09-13T14:58:29.354426+0000","last_clean":"2024-09-13T14:58:29.354426+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T14:58:29.354426+0000","last_undegraded":"2024-09-13T14:58:29.354426+0000","last_fullsized":"2024-09-13T14:58:29.354426+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":156,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.362821+0000","last_change":"2024-09-13T14:51:44.203186+0000","last_active":"2024-09-13T14:58:29.362821+0000","last_peered":"2024-09-13T14:58:29.362821+0000","last_clean":"2024-09-13T14:58:29.362821+0000","last_became_active":"2024-09-13T14:51:44.202851+0000","last_became_peered":"2024-09-13T14:51:44.202851+0000","last_unstale":"2024-09-13T14:58:29.362821+0000","last_undegraded":"2024-09-13T14:58:29.362821+0000","last_fullsized":"2024-09-13T14:58:29.362821+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:05:49.173340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"312'5046","reported_seq":5077,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:33.923428+0000","last_change":"2024-09-13T14:57:34.842873+0000","last_active":"2024-09-13T14:58:33.923428+0000","last_peered":"2024-09-13T14:58:33.923428+0000","last_clean":"2024-09-13T14:58:33.923428+0000","last_became_active":"2024-09-13T14:57:34.842571+0000","last_became_peered":"2024-09-13T14:57:34.842571+0000","last_unstale":"2024-09-13T14:58:33.923428+0000","last_undegraded":"2024-09-13T14:58:33.923428+0000","last_fullsized":"2024-09-13T14:58:33.923428+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5046,"log_dups_size":0,"ondisk_log_size":5046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:29:23.565563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330694656,"num_objects":5046,"num_object_clones":0,"num_object_copies":15138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5046,"num_write_kb":322944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":447,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.363988+0000","last_change":"2024-09-13T14:55:59.914869+0000","last_active":"2024-09-13T14:58:29.363988+0000","last_peered":"2024-09-13T14:58:29.363988+0000","last_clean":"2024-09-13T14:58:29.363988+0000","last_became_active":"2024-09-13T14:55:51.162703+0000","last_became_peered":"2024-09-13T14:55:51.162703+0000","last_unstale":"2024-09-13T14:58:29.363988+0000","last_undegraded":"2024-09-13T14:58:29.363988+0000","last_fullsized":"2024-09-13T14:58:29.363988+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.914801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.914801+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:17.053641+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":501,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.373654+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:58:29.373654+0000","last_peered":"2024-09-13T14:58:29.373654+0000","last_clean":"2024-09-13T14:58:29.373654+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:58:29.373654+0000","last_undegraded":"2024-09-13T14:58:29.373654+0000","last_fullsized":"2024-09-13T14:58:29.373654+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1125,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.337169+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T14:58:27.337169+0000","last_peered":"2024-09-13T14:58:27.337169+0000","last_clean":"2024-09-13T14:58:27.337169+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T14:58:27.337169+0000","last_undegraded":"2024-09-13T14:58:27.337169+0000","last_fullsized":"2024-09-13T14:58:27.337169+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":279,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.331377+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T14:58:27.331377+0000","last_peered":"2024-09-13T14:58:27.331377+0000","last_clean":"2024-09-13T14:58:27.331377+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T14:58:27.331377+0000","last_undegraded":"2024-09-13T14:58:27.331377+0000","last_fullsized":"2024-09-13T14:58:27.331377+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":296,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.333220+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T14:58:27.333220+0000","last_peered":"2024-09-13T14:58:27.333220+0000","last_clean":"2024-09-13T14:58:27.333220+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T14:58:27.333220+0000","last_undegraded":"2024-09-13T14:58:27.333220+0000","last_fullsized":"2024-09-13T14:58:27.333220+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":508,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.362422+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T14:58:29.362422+0000","last_peered":"2024-09-13T14:58:29.362422+0000","last_clean":"2024-09-13T14:58:29.362422+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T14:58:29.362422+0000","last_undegraded":"2024-09-13T14:58:29.362422+0000","last_fullsized":"2024-09-13T14:58:29.362422+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":491,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.361763+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T14:58:29.361763+0000","last_peered":"2024-09-13T14:58:29.361763+0000","last_clean":"2024-09-13T14:58:29.361763+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T14:58:29.361763+0000","last_undegraded":"2024-09-13T14:58:29.361763+0000","last_fullsized":"2024-09-13T14:58:29.361763+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":292,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.319401+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:58:27.319401+0000","last_peered":"2024-09-13T14:58:27.319401+0000","last_clean":"2024-09-13T14:58:27.319401+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:58:27.319401+0000","last_undegraded":"2024-09-13T14:58:27.319401+0000","last_fullsized":"2024-09-13T14:58:27.319401+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":69,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.354894+0000","last_change":"2024-09-13T14:56:01.235655+0000","last_active":"2024-09-13T14:58:29.354894+0000","last_peered":"2024-09-13T14:58:29.354894+0000","last_clean":"2024-09-13T14:58:29.354894+0000","last_became_active":"2024-09-13T14:55:55.195703+0000","last_became_peered":"2024-09-13T14:55:55.195703+0000","last_unstale":"2024-09-13T14:58:29.354894+0000","last_undegraded":"2024-09-13T14:58:29.354894+0000","last_fullsized":"2024-09-13T14:58:29.354894+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.235497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.235497+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:55.951927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":276,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.323677+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:58:27.323677+0000","last_peered":"2024-09-13T14:58:27.323677+0000","last_clean":"2024-09-13T14:58:27.323677+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:58:27.323677+0000","last_undegraded":"2024-09-13T14:58:27.323677+0000","last_fullsized":"2024-09-13T14:58:27.323677+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":460,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.363645+0000","last_change":"2024-09-13T14:48:32.200953+0000","last_active":"2024-09-13T14:58:29.363645+0000","last_peered":"2024-09-13T14:58:29.363645+0000","last_clean":"2024-09-13T14:58:29.363645+0000","last_became_active":"2024-09-13T14:48:32.200634+0000","last_became_peered":"2024-09-13T14:48:32.200634+0000","last_unstale":"2024-09-13T14:58:29.363645+0000","last_undegraded":"2024-09-13T14:58:29.363645+0000","last_fullsized":"2024-09-13T14:58:29.363645+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:18.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":488,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.364458+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:58:29.364458+0000","last_peered":"2024-09-13T14:58:29.364458+0000","last_clean":"2024-09-13T14:58:29.364458+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:58:29.364458+0000","last_undegraded":"2024-09-13T14:58:29.364458+0000","last_fullsized":"2024-09-13T14:58:29.364458+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":509,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.363396+0000","last_change":"2024-09-13T14:48:32.201812+0000","last_active":"2024-09-13T14:58:29.363396+0000","last_peered":"2024-09-13T14:58:29.363396+0000","last_clean":"2024-09-13T14:58:29.363396+0000","last_became_active":"2024-09-13T14:48:32.201486+0000","last_became_peered":"2024-09-13T14:48:32.201486+0000","last_unstale":"2024-09-13T14:58:29.363396+0000","last_undegraded":"2024-09-13T14:58:29.363396+0000","last_fullsized":"2024-09-13T14:58:29.363396+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:23:29.693906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":96,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.337135+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T14:58:27.337135+0000","last_peered":"2024-09-13T14:58:27.337135+0000","last_clean":"2024-09-13T14:58:27.337135+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T14:58:27.337135+0000","last_undegraded":"2024-09-13T14:58:27.337135+0000","last_fullsized":"2024-09-13T14:58:27.337135+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":513,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.363317+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:58:29.363317+0000","last_peered":"2024-09-13T14:58:29.363317+0000","last_clean":"2024-09-13T14:58:29.363317+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:58:29.363317+0000","last_undegraded":"2024-09-13T14:58:29.363317+0000","last_fullsized":"2024-09-13T14:58:29.363317+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":512,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.373636+0000","last_change":"2024-09-13T14:51:44.198771+0000","last_active":"2024-09-13T14:58:29.373636+0000","last_peered":"2024-09-13T14:58:29.373636+0000","last_clean":"2024-09-13T14:58:29.373636+0000","last_became_active":"2024-09-13T14:51:44.198466+0000","last_became_peered":"2024-09-13T14:51:44.198466+0000","last_unstale":"2024-09-13T14:58:29.373636+0000","last_undegraded":"2024-09-13T14:58:29.373636+0000","last_fullsized":"2024-09-13T14:58:29.373636+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:41:16.099954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":476,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.351951+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:58:29.351951+0000","last_peered":"2024-09-13T14:58:29.351951+0000","last_clean":"2024-09-13T14:58:29.351951+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:58:29.351951+0000","last_undegraded":"2024-09-13T14:58:29.351951+0000","last_fullsized":"2024-09-13T14:58:29.351951+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":509,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.363599+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:58:29.363599+0000","last_peered":"2024-09-13T14:58:29.363599+0000","last_clean":"2024-09-13T14:58:29.363599+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:58:29.363599+0000","last_undegraded":"2024-09-13T14:58:29.363599+0000","last_fullsized":"2024-09-13T14:58:29.363599+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":295,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.332673+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:58:27.332673+0000","last_peered":"2024-09-13T14:58:27.332673+0000","last_clean":"2024-09-13T14:58:27.332673+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:58:27.332673+0000","last_undegraded":"2024-09-13T14:58:27.332673+0000","last_fullsized":"2024-09-13T14:58:27.332673+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":310,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.352872+0000","last_change":"2024-09-13T14:56:41.388016+0000","last_active":"2024-09-13T14:58:29.352872+0000","last_peered":"2024-09-13T14:58:29.352872+0000","last_clean":"2024-09-13T14:58:29.352872+0000","last_became_active":"2024-09-13T14:56:41.387745+0000","last_became_peered":"2024-09-13T14:56:41.387745+0000","last_unstale":"2024-09-13T14:58:29.352872+0000","last_undegraded":"2024-09-13T14:58:29.352872+0000","last_fullsized":"2024-09-13T14:58:29.352872+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:55.236441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":481,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.366315+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:58:29.366315+0000","last_peered":"2024-09-13T14:58:29.366315+0000","last_clean":"2024-09-13T14:58:29.366315+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:58:29.366315+0000","last_undegraded":"2024-09-13T14:58:29.366315+0000","last_fullsized":"2024-09-13T14:58:29.366315+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":293,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.333142+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:58:27.333142+0000","last_peered":"2024-09-13T14:58:27.333142+0000","last_clean":"2024-09-13T14:58:27.333142+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:58:27.333142+0000","last_undegraded":"2024-09-13T14:58:27.333142+0000","last_fullsized":"2024-09-13T14:58:27.333142+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":278,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.323734+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:58:27.323734+0000","last_peered":"2024-09-13T14:58:27.323734+0000","last_clean":"2024-09-13T14:58:27.323734+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:58:27.323734+0000","last_undegraded":"2024-09-13T14:58:27.323734+0000","last_fullsized":"2024-09-13T14:58:27.323734+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":293,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.332589+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:58:27.332589+0000","last_peered":"2024-09-13T14:58:27.332589+0000","last_clean":"2024-09-13T14:58:27.332589+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:58:27.332589+0000","last_undegraded":"2024-09-13T14:58:27.332589+0000","last_fullsized":"2024-09-13T14:58:27.332589+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":317,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.325583+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T14:58:27.325583+0000","last_peered":"2024-09-13T14:58:27.325583+0000","last_clean":"2024-09-13T14:58:27.325583+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T14:58:27.325583+0000","last_undegraded":"2024-09-13T14:58:27.325583+0000","last_fullsized":"2024-09-13T14:58:27.325583+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":267,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.325074+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:58:27.325074+0000","last_peered":"2024-09-13T14:58:27.325074+0000","last_clean":"2024-09-13T14:58:27.325074+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:58:27.325074+0000","last_undegraded":"2024-09-13T14:58:27.325074+0000","last_fullsized":"2024-09-13T14:58:27.325074+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":517,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.363479+0000","last_change":"2024-09-13T14:48:32.202404+0000","last_active":"2024-09-13T14:58:29.363479+0000","last_peered":"2024-09-13T14:58:29.363479+0000","last_clean":"2024-09-13T14:58:29.363479+0000","last_became_active":"2024-09-13T14:48:32.202064+0000","last_became_peered":"2024-09-13T14:48:32.202064+0000","last_unstale":"2024-09-13T14:58:29.363479+0000","last_undegraded":"2024-09-13T14:58:29.363479+0000","last_fullsized":"2024-09-13T14:58:29.363479+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:25.674819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":299,"reported_epoch":311,"state":"active+clean","last_fresh":"2024-09-13T14:58:27.331300+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T14:58:27.331300+0000","last_peered":"2024-09-13T14:58:27.331300+0000","last_clean":"2024-09-13T14:58:27.331300+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T14:58:27.331300+0000","last_undegraded":"2024-09-13T14:58:27.331300+0000","last_fullsized":"2024-09-13T14:58:27.331300+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":532,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.364355+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T14:58:29.364355+0000","last_peered":"2024-09-13T14:58:29.364355+0000","last_clean":"2024-09-13T14:58:29.364355+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T14:58:29.364355+0000","last_undegraded":"2024-09-13T14:58:29.364355+0000","last_fullsized":"2024-09-13T14:58:29.364355+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5232721920,"num_objects":79845,"num_object_clones":0,"num_object_copies":239535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":79845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":79845,"num_write_kb":5110080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15731589120,"data_stored":15731589120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79845,"ondisk_log_size":79845,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":16172,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2100,"ondisk_log_size":2100,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397896,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3834612,"kb_used_data":2953516,"kb_used_omap":1,"kb_used_meta":881086,"kb_avail":89910540,"statfs":{"total":95995035648,"available":92068392960,"internally_reserved":0,"allocated":3024400384,"data_stored":3022259780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":902232408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,0,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594125,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1284076,"kb_used_data":986724,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":92461076,"statfs":{"total":95995035648,"available":94680141824,"internally_reserved":0,"allocated":1010405376,"data_stored":1008272536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790354,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3846056,"kb_used_data":2953184,"kb_used_omap":1,"kb_used_meta":892862,"kb_avail":89899096,"statfs":{"total":95995035648,"available":92056674304,"internally_reserved":0,"allocated":3024060416,"data_stored":3021928113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":914291032},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,4,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986583,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1878312,"kb_used_data":976928,"kb_used_omap":1,"kb_used_meta":901374,"kb_avail":91866840,"statfs":{"total":95995035648,"available":94071644160,"internally_reserved":0,"allocated":1000374272,"data_stored":998241156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923007320},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676599030,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3266396,"kb_used_data":2490452,"kb_used_omap":6,"kb_used_meta":775929,"kb_avail":90478756,"statfs":{"total":95995035648,"available":92650246144,"internally_reserved":0,"allocated":2550222848,"data_stored":2548102202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":794552288},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762557,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1717160,"kb_used_data":940512,"kb_used_omap":8,"kb_used_meta":776631,"kb_avail":92027992,"statfs":{"total":95995035648,"available":94236663808,"internally_reserved":0,"allocated":963084288,"data_stored":960951980,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":795270602},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926083,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3553248,"kb_used_data":2801496,"kb_used_omap":7,"kb_used_meta":751736,"kb_avail":90191904,"statfs":{"total":95995035648,"available":92356509696,"internally_reserved":0,"allocated":2868731904,"data_stored":2866606881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":769778377},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122323,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1883380,"kb_used_data":1241516,"kb_used_omap":7,"kb_used_meta":641848,"kb_avail":91861772,"statfs":{"total":95995035648,"available":94066454528,"internally_reserved":0,"allocated":1271312384,"data_stored":1269172288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":657252943},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1284440064,"data_stored":1284440064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2867265536,"data_stored":2867265536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":969736192,"data_stored":969736192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2554920960,"data_stored":2554920960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":996671488,"data_stored":996671488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3034710016,"data_stored":3034710016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1002962944,"data_stored":1002962944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3020881920,"data_stored":3020881920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:58:40.211 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:58:40.230 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:40.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:40.332 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:40.365 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:40.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:40.462 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:58:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:58:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:40.466 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:40.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.510 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:40.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:40.597 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:58:40.597 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T14:58:40.597 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T14:58:40.597 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T14:58:40.598 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:58:40.611 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:40.613 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 50 16 86035 86019 107.51 72.9375 0.00451861 0.00929332 2024-09-13T14:58:40.657 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:40.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:40.759 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:40.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.810 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:40.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:40.877 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:40.897 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:40.911 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:40.940 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:40.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:41.042 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:41.086 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:41.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:41.189 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:41.237 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:41.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:41.333 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:58:41.339 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:41.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.424 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:41.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:41.525 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:41.561 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:41.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:41.613 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 51 16 87160 87144 106.78 70.3125 0.0228777 0.00936004 2024-09-13T14:58:41.663 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:41.698 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:41.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:41.761 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:41.800 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:41.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.842 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:41.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:41.890 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:58:41.942 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:41.976 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:41.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:42.018 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:41 smithi096 ceph-mon[98901]: pgmap v1015: 109 pgs: 109 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:58:42.018 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:41 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2913536253' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:58:42.018 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:41 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:58:42.018 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:41 smithi096 ceph-mon[102928]: pgmap v1015: 109 pgs: 109 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:58:42.018 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:41 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2913536253' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:58:42.018 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:41 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:58:42.078 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:42.114 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:42.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:42.215 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:42.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:41 smithi204 ceph-mon[87071]: pgmap v1015: 109 pgs: 109 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:58:42.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:41 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2913536253' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:58:42.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:41 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:58:42.254 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:42.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:42.356 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:42.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.388 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:42.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:42.490 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:42.529 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:42.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:42.613 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 52 16 88353 88337 106.161 74.5625 0.0121036 0.00941486 2024-09-13T14:58:42.633 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:42.800 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:42.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:42.901 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:42.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.935 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:42.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:42.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:42 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:42.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:42 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:58:42.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:42 smithi204 ceph-mon[87071]: osdmap e313: 8 total, 8 up, 8 in 2024-09-13T14:58:42.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:42 smithi204 ceph-mon[87071]: pgmap v1017: 109 pgs: 109 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:58:43.037 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:43.057 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:43.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:43.159 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:43.194 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:43.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:43.296 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:43.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:42 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:43.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:42 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:58:43.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:42 smithi096 ceph-mon[98901]: osdmap e313: 8 total, 8 up, 8 in 2024-09-13T14:58:43.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:42 smithi096 ceph-mon[98901]: pgmap v1017: 109 pgs: 109 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:58:43.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:42 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:43.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:42 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:58:43.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:42 smithi096 ceph-mon[102928]: osdmap e313: 8 total, 8 up, 8 in 2024-09-13T14:58:43.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:42 smithi096 ceph-mon[102928]: pgmap v1017: 109 pgs: 109 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:58:43.332 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:43.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:43.434 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:43.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.477 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:43.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:43.578 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:43.606 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:43.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:43.613 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 53 15 89404 89389 105.398 65.75 0.00781125 0.00948255 2024-09-13T14:58:43.708 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:43.747 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:43.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:43.849 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:43.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.890 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:43.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:43.991 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:44.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.024 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:44.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:44.111 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:44 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:58:44.124 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:44.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.164 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:44.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:44.265 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:44.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.302 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:44.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:44.403 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:44.447 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:44.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:44.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:44 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:58:44.549 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:44.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:44 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:58:44.587 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:44.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:44.614 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 54 16 90546 90530 104.767 71.3125 0.0149333 0.0095395 2024-09-13T14:58:44.689 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:44.736 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:44.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:44.841 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:44.881 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:44.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:44.983 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:45.032 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:45.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:45.136 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:45.172 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:45.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:45.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:45 smithi204 ceph-mon[87071]: pgmap v1018: 109 pgs: 109 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T14:58:45.269 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T14:58:45.269 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T14:58:45.270 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T14:58:45.270 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T14:58:45.270 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.270 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:125782 7.5 7:a83dc0eb:::benchmark_data_smithi096_196313_object5651:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2024-09-13T14:58:45.270 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:50:16.277859+0000", 2024-09-13T14:58:45.270 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 508.98452910499998, 2024-09-13T14:58:45.270 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.34631596799999997, 2024-09-13T14:58:45.270 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.270 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.270 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.270 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:45.270 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:45.270 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 125782 2024-09-13T14:58:45.271 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.271 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.271 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.271 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.271 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277859+0000", 2024-09-13T14:58:45.271 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.271 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.271 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.271 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.271 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277859+0000", 2024-09-13T14:58:45.271 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.271 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.271 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.271 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.272 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277858+0000", 2024-09-13T14:58:45.272 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:45.272 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.272 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.272 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.272 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277862+0000", 2024-09-13T14:58:45.272 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6859999999999997e-06 2024-09-13T14:58:45.272 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.272 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.272 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.272 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277872+0000", 2024-09-13T14:58:45.272 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6989999999999998e-06 2024-09-13T14:58:45.272 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.273 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.273 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.273 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.277876+0000", 2024-09-13T14:58:45.273 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.391e-06 2024-09-13T14:58:45.273 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.273 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.273 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.273 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.278833+0000", 2024-09-13T14:58:45.273 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00095691899999999998 2024-09-13T14:58:45.273 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.273 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.273 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.274 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.278950+0000", 2024-09-13T14:58:45.274 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000116446 2024-09-13T14:58:45.274 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.274 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.274 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:58:45.274 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.279046+0000", 2024-09-13T14:58:45.274 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6119999999999995e-05 2024-09-13T14:58:45.274 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.274 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.274 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.274 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.282554+0000", 2024-09-13T14:58:45.274 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035082289999999999 2024-09-13T14:58:45.275 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.275 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.275 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.275 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287518+0000", 2024-09-13T14:58:45.275 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0049643049999999996 2024-09-13T14:58:45.275 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.275 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.275 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.275 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624079+0000", 2024-09-13T14:58:45.276 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.33656060599999998 2024-09-13T14:58:45.276 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.276 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.276 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.277 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624136+0000", 2024-09-13T14:58:45.277 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6737000000000002e-05 2024-09-13T14:58:45.277 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.277 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.277 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.277 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624175+0000", 2024-09-13T14:58:45.277 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8862000000000002e-05 2024-09-13T14:58:45.277 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.277 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.277 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.277 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.277 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.277 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:125792 7.5 7:a137e5c2:::benchmark_data_smithi096_196313_object5661:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2024-09-13T14:58:45.278 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:50:16.287604+0000", 2024-09-13T14:58:45.278 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 508.97478368100002, 2024-09-13T14:58:45.278 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.33670364200000003, 2024-09-13T14:58:45.278 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.278 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.278 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.278 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:45.278 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:45.278 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 125792 2024-09-13T14:58:45.278 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.278 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.278 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.278 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.278 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287604+0000", 2024-09-13T14:58:45.279 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.279 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.279 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.279 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.279 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287604+0000", 2024-09-13T14:58:45.279 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.279 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.279 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.279 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.279 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287603+0000", 2024-09-13T14:58:45.279 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:45.279 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.279 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.279 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.280 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287609+0000", 2024-09-13T14:58:45.280 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6489999999999996e-06 2024-09-13T14:58:45.280 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.280 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.280 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.280 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287618+0000", 2024-09-13T14:58:45.280 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5820000000000008e-06 2024-09-13T14:58:45.280 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.280 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.280 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.280 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287626+0000", 2024-09-13T14:58:45.280 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1780000000000003e-06 2024-09-13T14:58:45.280 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.281 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.281 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.281 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287652+0000", 2024-09-13T14:58:45.281 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5395e-05 2024-09-13T14:58:45.281 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.281 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.281 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.281 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287735+0000", 2024-09-13T14:58:45.281 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2979000000000006e-05 2024-09-13T14:58:45.281 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.281 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.281 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:58:45.281 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.287800+0000", 2024-09-13T14:58:45.281 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5335999999999999e-05 2024-09-13T14:58:45.282 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.282 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.282 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.282 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.296078+0000", 2024-09-13T14:58:45.282 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0082774900000000002 2024-09-13T14:58:45.282 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.282 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.282 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.282 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.299594+0000", 2024-09-13T14:58:45.282 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035163949999999999 2024-09-13T14:58:45.282 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.282 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.282 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.283 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624220+0000", 2024-09-13T14:58:45.283 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.32462581499999998 2024-09-13T14:58:45.283 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.283 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.283 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.283 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624269+0000", 2024-09-13T14:58:45.283 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9626000000000002e-05 2024-09-13T14:58:45.283 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.283 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.283 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.283 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624308+0000", 2024-09-13T14:58:45.283 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8374999999999998e-05 2024-09-13T14:58:45.283 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.283 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.284 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.284 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.284 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.284 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:125795 7.5 7:acb77273:::benchmark_data_smithi096_196313_object5664:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2024-09-13T14:58:45.284 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:50:16.290753+0000", 2024-09-13T14:58:45.284 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 508.97163523099999, 2024-09-13T14:58:45.284 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.33410231899999998, 2024-09-13T14:58:45.284 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.284 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.284 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.284 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:45.284 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:45.284 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 125795 2024-09-13T14:58:45.284 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.285 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.285 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.285 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.285 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290753+0000", 2024-09-13T14:58:45.285 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.285 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.285 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.285 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.285 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290753+0000", 2024-09-13T14:58:45.285 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.285 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.285 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.285 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.285 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290751+0000", 2024-09-13T14:58:45.286 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:45.286 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.286 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.286 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.286 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290758+0000", 2024-09-13T14:58:45.286 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.238e-06 2024-09-13T14:58:45.286 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.286 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.286 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.286 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290778+0000", 2024-09-13T14:58:45.286 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9643000000000001e-05 2024-09-13T14:58:45.286 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.286 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.287 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.287 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.290786+0000", 2024-09-13T14:58:45.287 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5220000000000001e-06 2024-09-13T14:58:45.287 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.287 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.287 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.287 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.291631+0000", 2024-09-13T14:58:45.287 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000845617 2024-09-13T14:58:45.287 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.287 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.287 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.287 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.291741+0000", 2024-09-13T14:58:45.287 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000109487 2024-09-13T14:58:45.287 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.288 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.288 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:58:45.288 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.291842+0000", 2024-09-13T14:58:45.288 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000101823 2024-09-13T14:58:45.288 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.288 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.288 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.288 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.296685+0000", 2024-09-13T14:58:45.288 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0048421799999999997 2024-09-13T14:58:45.288 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.288 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.288 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.288 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.299630+0000", 2024-09-13T14:58:45.289 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002945544 2024-09-13T14:58:45.289 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.289 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.289 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.289 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624773+0000", 2024-09-13T14:58:45.289 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.32514234600000003 2024-09-13T14:58:45.289 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.289 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.289 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.289 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624815+0000", 2024-09-13T14:58:45.289 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2979999999999998e-05 2024-09-13T14:58:45.289 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.289 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.289 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.290 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:50:16.624855+0000", 2024-09-13T14:58:45.290 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9347000000000001e-05 2024-09-13T14:58:45.290 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.290 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.290 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.290 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.290 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.290 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:172721 7.5 7:a4933ecd:::benchmark_data_smithi096_196313_object52590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-09-13T14:58:45.290 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:16.905298+0000", 2024-09-13T14:58:45.290 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 448.35708941600001, 2024-09-13T14:58:45.290 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18517154299999999, 2024-09-13T14:58:45.290 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.290 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.290 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.291 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T14:58:45.291 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T14:58:45.291 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 172721 2024-09-13T14:58:45.291 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.291 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.291 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.291 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.291 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905298+0000", 2024-09-13T14:58:45.291 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.291 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.291 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.291 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.291 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905298+0000", 2024-09-13T14:58:45.292 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.292 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.292 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.292 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.292 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905297+0000", 2024-09-13T14:58:45.292 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:45.292 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.292 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.292 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.292 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905304+0000", 2024-09-13T14:58:45.292 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5290000000000003e-06 2024-09-13T14:58:45.292 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.292 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.293 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.293 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905320+0000", 2024-09-13T14:58:45.293 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5852e-05 2024-09-13T14:58:45.293 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.293 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.293 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.293 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905327+0000", 2024-09-13T14:58:45.293 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5889999999999999e-06 2024-09-13T14:58:45.293 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.293 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.293 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.293 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905375+0000", 2024-09-13T14:58:45.293 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8362000000000002e-05 2024-09-13T14:58:45.293 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.294 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.294 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.294 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905529+0000", 2024-09-13T14:58:45.294 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015435700000000001 2024-09-13T14:58:45.294 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.294 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.294 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:58:45.294 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.905659+0000", 2024-09-13T14:58:45.294 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000129231 2024-09-13T14:58:45.294 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.294 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.294 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.294 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.909873+0000", 2024-09-13T14:58:45.295 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.004214158 2024-09-13T14:58:45.295 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.295 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.295 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.295 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:16.910053+0000", 2024-09-13T14:58:45.295 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018025800000000001 2024-09-13T14:58:45.295 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.295 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.295 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.295 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:17.090381+0000", 2024-09-13T14:58:45.295 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18032772599999999 2024-09-13T14:58:45.295 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.295 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.296 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.296 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:17.090429+0000", 2024-09-13T14:58:45.296 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8335999999999999e-05 2024-09-13T14:58:45.296 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.296 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.296 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.296 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:17.090470+0000", 2024-09-13T14:58:45.296 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0713000000000001e-05 2024-09-13T14:58:45.296 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.296 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.296 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.296 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.296 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.296 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:19649 9.e 9:79df33a1:::benchmark_data_smithi096_256909_object19648:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:58:45.296 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:57:56.854522+0000", 2024-09-13T14:58:45.297 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 48.407865358999999, 2024-09-13T14:58:45.297 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.082913841000000002, 2024-09-13T14:58:45.297 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.297 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.297 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.297 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.297 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.297 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 19649 2024-09-13T14:58:45.297 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.297 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.297 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.297 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.297 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.854522+0000", 2024-09-13T14:58:45.297 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.297 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.298 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.298 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.298 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.854522+0000", 2024-09-13T14:58:45.298 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.298 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.298 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.298 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.298 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.854521+0000", 2024-09-13T14:58:45.298 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:45.298 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.298 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.298 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.298 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.856084+0000", 2024-09-13T14:58:45.299 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:45.300 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:45 smithi096 ceph-mon[98901]: pgmap v1018: 109 pgs: 109 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T14:58:45.300 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:45 smithi096 ceph-mon[102928]: pgmap v1018: 109 pgs: 109 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T14:58:45.301 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0015623589999999999 2024-09-13T14:58:45.301 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.301 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.301 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.301 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.856093+0000", 2024-09-13T14:58:45.301 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.0420000000000008e-06 2024-09-13T14:58:45.301 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.301 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.301 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.301 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.856098+0000", 2024-09-13T14:58:45.301 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9810000000000003e-06 2024-09-13T14:58:45.301 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.302 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.302 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.302 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.856155+0000", 2024-09-13T14:58:45.302 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7738999999999997e-05 2024-09-13T14:58:45.302 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.302 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.302 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.302 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.856274+0000", 2024-09-13T14:58:45.302 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011898800000000001 2024-09-13T14:58:45.302 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.302 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.302 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:58:45.302 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.856423+0000", 2024-09-13T14:58:45.303 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000149006 2024-09-13T14:58:45.303 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.303 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.303 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.303 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.858015+0000", 2024-09-13T14:58:45.303 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0015912490000000001 2024-09-13T14:58:45.303 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.303 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.303 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.303 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.860732+0000", 2024-09-13T14:58:45.303 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0027169870000000001 2024-09-13T14:58:45.303 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.303 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.303 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.304 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.937327+0000", 2024-09-13T14:58:45.304 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.076595282000000001 2024-09-13T14:58:45.304 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.304 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.304 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.304 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.937395+0000", 2024-09-13T14:58:45.304 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8269000000000007e-05 2024-09-13T14:58:45.304 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.304 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.304 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.304 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:56.937436+0000", 2024-09-13T14:58:45.304 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1007999999999998e-05 2024-09-13T14:58:45.304 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.305 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.305 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.305 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.305 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.305 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:21163 9.e 9:74b46ea3:::benchmark_data_smithi096_256909_object21162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:58:45.305 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:57:57.547606+0000", 2024-09-13T14:58:45.305 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 47.714781444000003, 2024-09-13T14:58:45.305 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.095710126000000006, 2024-09-13T14:58:45.305 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.305 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.305 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.305 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.305 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.306 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 21163 2024-09-13T14:58:45.306 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.306 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.306 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.306 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.306 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.547606+0000", 2024-09-13T14:58:45.306 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.306 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.306 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.306 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.306 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.547606+0000", 2024-09-13T14:58:45.306 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.306 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.306 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.307 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.307 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.547605+0000", 2024-09-13T14:58:45.307 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:45.307 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.307 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.307 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.307 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.547813+0000", 2024-09-13T14:58:45.307 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00020733500000000001 2024-09-13T14:58:45.307 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.307 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.307 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.307 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.547824+0000", 2024-09-13T14:58:45.307 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1783000000000001e-05 2024-09-13T14:58:45.307 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.308 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.308 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.308 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.547830+0000", 2024-09-13T14:58:45.308 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3190000000000002e-06 2024-09-13T14:58:45.308 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.308 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.308 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.308 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.547869+0000", 2024-09-13T14:58:45.308 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9582999999999997e-05 2024-09-13T14:58:45.308 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.308 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.308 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.308 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.547994+0000", 2024-09-13T14:58:45.308 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000124929 2024-09-13T14:58:45.309 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.309 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.309 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:58:45.309 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.548134+0000", 2024-09-13T14:58:45.309 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013951299999999999 2024-09-13T14:58:45.309 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.309 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.309 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.309 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.549116+0000", 2024-09-13T14:58:45.309 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00098190199999999999 2024-09-13T14:58:45.309 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.309 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.309 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.310 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.550196+0000", 2024-09-13T14:58:45.310 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001080316 2024-09-13T14:58:45.310 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.310 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.310 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.310 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.643210+0000", 2024-09-13T14:58:45.310 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.093013620000000005 2024-09-13T14:58:45.310 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.310 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.310 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.310 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.643264+0000", 2024-09-13T14:58:45.310 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4203000000000003e-05 2024-09-13T14:58:45.310 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.311 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.311 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.311 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:57:57.643316+0000", 2024-09-13T14:58:45.311 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.257e-05 2024-09-13T14:58:45.311 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.311 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.311 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.311 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.311 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.311 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:53969 9.3 9:c0acc04b:::benchmark_data_smithi096_256909_object53968:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2024-09-13T14:58:45.311 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:16.219146+0000", 2024-09-13T14:58:45.311 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 29.043241979000001, 2024-09-13T14:58:45.311 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.063123033999999995, 2024-09-13T14:58:45.311 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.312 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.312 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.312 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.312 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.312 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 53969 2024-09-13T14:58:45.312 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.312 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.312 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.312 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.313 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.219146+0000", 2024-09-13T14:58:45.313 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.313 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.313 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.313 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.313 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.219146+0000", 2024-09-13T14:58:45.313 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.313 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.313 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.313 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.313 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.219145+0000", 2024-09-13T14:58:45.313 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:45.314 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.314 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.314 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.314 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.219197+0000", 2024-09-13T14:58:45.314 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1882000000000002e-05 2024-09-13T14:58:45.314 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.314 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.314 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.314 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.219210+0000", 2024-09-13T14:58:45.314 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3488e-05 2024-09-13T14:58:45.314 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.314 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.314 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.315 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.219215+0000", 2024-09-13T14:58:45.315 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1209999999999998e-06 2024-09-13T14:58:45.315 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.315 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.315 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.315 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.219259+0000", 2024-09-13T14:58:45.315 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3475e-05 2024-09-13T14:58:45.315 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.315 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.315 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.315 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.219383+0000", 2024-09-13T14:58:45.315 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000124399 2024-09-13T14:58:45.315 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.315 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.316 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:45.316 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.220299+0000", 2024-09-13T14:58:45.316 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00091580399999999999 2024-09-13T14:58:45.316 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.316 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.316 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.316 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.222073+0000", 2024-09-13T14:58:45.316 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0017740119999999999 2024-09-13T14:58:45.316 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.316 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.316 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.316 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.225033+0000", 2024-09-13T14:58:45.316 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029604850000000001 2024-09-13T14:58:45.317 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.317 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.317 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.317 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.282169+0000", 2024-09-13T14:58:45.317 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.057135742000000003 2024-09-13T14:58:45.317 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.317 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.317 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.317 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.282223+0000", 2024-09-13T14:58:45.317 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3919999999999999e-05 2024-09-13T14:58:45.318 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.318 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.318 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.318 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.282269+0000", 2024-09-13T14:58:45.318 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5791000000000003e-05 2024-09-13T14:58:45.318 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.318 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.318 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.318 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.318 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.318 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:53980 9.3 9:ca0a68fb:::benchmark_data_smithi096_256909_object53979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2024-09-13T14:58:45.318 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:16.225320+0000", 2024-09-13T14:58:45.318 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 29.037068179999999, 2024-09-13T14:58:45.319 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.063433254999999994, 2024-09-13T14:58:45.319 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.319 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.319 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.319 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.319 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.319 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 53980 2024-09-13T14:58:45.319 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.319 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.319 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.319 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.319 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.225320+0000", 2024-09-13T14:58:45.319 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.320 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.320 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.320 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.320 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.225320+0000", 2024-09-13T14:58:45.320 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.320 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.320 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.320 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.320 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.225318+0000", 2024-09-13T14:58:45.320 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:45.320 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.320 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.320 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.320 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.225368+0000", 2024-09-13T14:58:45.321 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9747999999999999e-05 2024-09-13T14:58:45.321 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.321 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.321 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.321 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.225375+0000", 2024-09-13T14:58:45.321 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2509999999999997e-06 2024-09-13T14:58:45.321 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.321 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.321 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.321 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.225379+0000", 2024-09-13T14:58:45.321 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7809999999999999e-06 2024-09-13T14:58:45.321 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.321 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.322 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.322 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.225423+0000", 2024-09-13T14:58:45.322 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4336999999999999e-05 2024-09-13T14:58:45.322 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.322 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.322 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.322 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.225549+0000", 2024-09-13T14:58:45.322 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012613899999999999 2024-09-13T14:58:45.322 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.322 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.322 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:45.322 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.225687+0000", 2024-09-13T14:58:45.322 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013798 2024-09-13T14:58:45.322 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.323 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.323 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.323 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.227266+0000", 2024-09-13T14:58:45.323 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0015784799999999999 2024-09-13T14:58:45.323 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.323 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.323 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.323 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.228011+0000", 2024-09-13T14:58:45.323 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00074518000000000002 2024-09-13T14:58:45.323 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.323 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.323 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.323 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.288663+0000", 2024-09-13T14:58:45.324 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.060651407999999997 2024-09-13T14:58:45.324 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.324 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.324 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.324 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.288712+0000", 2024-09-13T14:58:45.324 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9370000000000003e-05 2024-09-13T14:58:45.324 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.324 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.324 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.324 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:16.288753+0000", 2024-09-13T14:58:45.324 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0917000000000001e-05 2024-09-13T14:58:45.324 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.324 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.324 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.325 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.325 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.325 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:56421 9.d 9:b08897a7:::benchmark_data_smithi096_256909_object56420:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2024-09-13T14:58:45.325 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:17.842325+0000", 2024-09-13T14:58:45.325 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 27.420062564999999, 2024-09-13T14:58:45.325 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.061758774000000002, 2024-09-13T14:58:45.325 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.325 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.325 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.325 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.325 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.325 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 56421 2024-09-13T14:58:45.325 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.326 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.326 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.326 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.326 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.842325+0000", 2024-09-13T14:58:45.326 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.326 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.326 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.326 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.326 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.842325+0000", 2024-09-13T14:58:45.326 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.326 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.326 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.326 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.326 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.842303+0000", 2024-09-13T14:58:45.327 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999781 2024-09-13T14:58:45.327 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.327 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.327 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.327 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.842365+0000", 2024-09-13T14:58:45.327 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1699999999999995e-05 2024-09-13T14:58:45.327 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.327 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.327 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.327 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.842375+0000", 2024-09-13T14:58:45.327 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9660000000000008e-06 2024-09-13T14:58:45.327 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.327 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.327 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.327 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.842380+0000", 2024-09-13T14:58:45.328 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3820000000000003e-06 2024-09-13T14:58:45.328 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.328 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.328 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.328 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.847487+0000", 2024-09-13T14:58:45.328 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.005106925 2024-09-13T14:58:45.328 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.328 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.328 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.328 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.847611+0000", 2024-09-13T14:58:45.328 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012359799999999999 2024-09-13T14:58:45.328 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.328 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.328 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:58:45.329 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.847757+0000", 2024-09-13T14:58:45.329 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014662900000000001 2024-09-13T14:58:45.329 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.329 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.329 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.329 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.852962+0000", 2024-09-13T14:58:45.329 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.005204429 2024-09-13T14:58:45.329 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.329 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.329 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.329 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.852969+0000", 2024-09-13T14:58:45.329 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7329999999999997e-06 2024-09-13T14:58:45.329 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.329 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.330 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.330 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.904033+0000", 2024-09-13T14:58:45.330 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.051064217000000002 2024-09-13T14:58:45.330 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.330 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.330 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.330 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.904063+0000", 2024-09-13T14:58:45.330 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0239999999999998e-05 2024-09-13T14:58:45.330 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.330 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.330 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.330 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.904084+0000", 2024-09-13T14:58:45.330 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0873000000000002e-05 2024-09-13T14:58:45.330 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.331 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.331 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.331 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.331 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.331 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:56426 9.d 9:b2a9213b:::benchmark_data_smithi096_256909_object56425:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2024-09-13T14:58:45.331 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:17.848236+0000", 2024-09-13T14:58:45.331 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 27.414151263000001, 2024-09-13T14:58:45.331 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.060407188000000001, 2024-09-13T14:58:45.331 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.331 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.331 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.331 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.331 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.332 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 56426 2024-09-13T14:58:45.332 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.332 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.332 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.332 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.332 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.848236+0000", 2024-09-13T14:58:45.332 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.332 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.332 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.332 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.332 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.848236+0000", 2024-09-13T14:58:45.332 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.332 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.333 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.333 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.333 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.848235+0000", 2024-09-13T14:58:45.333 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:45.333 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.333 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.333 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.333 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.848298+0000", 2024-09-13T14:58:45.333 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3299999999999994e-05 2024-09-13T14:58:45.333 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.333 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.333 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.333 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.848309+0000", 2024-09-13T14:58:45.333 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0231000000000001e-05 2024-09-13T14:58:45.334 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.334 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.334 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.334 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.848314+0000", 2024-09-13T14:58:45.334 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1959999999999999e-06 2024-09-13T14:58:45.334 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.334 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.334 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.334 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.849732+0000", 2024-09-13T14:58:45.334 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001417714 2024-09-13T14:58:45.334 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.334 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.334 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.334 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.849865+0000", 2024-09-13T14:58:45.335 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013346500000000001 2024-09-13T14:58:45.335 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.335 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.335 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:58:45.335 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.850525+0000", 2024-09-13T14:58:45.335 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00065962699999999998 2024-09-13T14:58:45.335 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.335 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.335 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.335 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.852975+0000", 2024-09-13T14:58:45.335 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0024504679999999999 2024-09-13T14:58:45.335 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.335 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.335 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.336 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.853620+0000", 2024-09-13T14:58:45.336 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00064492300000000005 2024-09-13T14:58:45.336 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.336 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.336 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.336 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.908593+0000", 2024-09-13T14:58:45.336 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.054973235000000002 2024-09-13T14:58:45.336 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.336 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.336 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.336 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.908619+0000", 2024-09-13T14:58:45.336 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5293e-05 2024-09-13T14:58:45.336 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.336 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.337 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.337 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:17.908644+0000", 2024-09-13T14:58:45.337 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5159000000000001e-05 2024-09-13T14:58:45.337 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.337 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.337 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.337 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.337 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.337 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:57210 9.3 9:cc7a0557:::benchmark_data_smithi096_256909_object57209:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2024-09-13T14:58:45.337 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:18.381926+0000", 2024-09-13T14:58:45.337 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 26.880462069, 2024-09-13T14:58:45.337 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.061792677999999997, 2024-09-13T14:58:45.337 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.337 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.338 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.338 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.338 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.338 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 57210 2024-09-13T14:58:45.338 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.338 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.338 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.338 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.338 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.381926+0000", 2024-09-13T14:58:45.338 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.338 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.338 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.338 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.338 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.381926+0000", 2024-09-13T14:58:45.339 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.339 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.339 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.339 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.339 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.381924+0000", 2024-09-13T14:58:45.339 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:45.339 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.339 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.339 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.339 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.382022+0000", 2024-09-13T14:58:45.339 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.8083000000000003e-05 2024-09-13T14:58:45.339 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.339 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.339 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.339 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.382036+0000", 2024-09-13T14:58:45.340 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.332e-05 2024-09-13T14:58:45.340 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.340 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.340 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.382041+0000", 2024-09-13T14:58:45.340 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4580000000000001e-06 2024-09-13T14:58:45.340 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.340 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.340 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.382085+0000", 2024-09-13T14:58:45.340 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3786999999999999e-05 2024-09-13T14:58:45.340 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.340 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.341 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.382213+0000", 2024-09-13T14:58:45.341 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012767900000000001 2024-09-13T14:58:45.341 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.341 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.341 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:45.341 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.382342+0000", 2024-09-13T14:58:45.341 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000129171 2024-09-13T14:58:45.341 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.341 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.341 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.341 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.384437+0000", 2024-09-13T14:58:45.341 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0020952259999999999 2024-09-13T14:58:45.341 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.341 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.342 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.342 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.387751+0000", 2024-09-13T14:58:45.342 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003314282 2024-09-13T14:58:45.342 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.342 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.342 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.342 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.443640+0000", 2024-09-13T14:58:45.342 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.055889148999999999 2024-09-13T14:58:45.342 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.342 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.342 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.342 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.443686+0000", 2024-09-13T14:58:45.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.344 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5324000000000002e-05 2024-09-13T14:58:45.344 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.345 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.345 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.345 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.443718+0000", 2024-09-13T14:58:45.345 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2570999999999997e-05 2024-09-13T14:58:45.345 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.345 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.345 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.345 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.345 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.345 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:57219 9.3 9:c1c35973:::benchmark_data_smithi096_256909_object57218:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2024-09-13T14:58:45.345 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:18.387677+0000", 2024-09-13T14:58:45.345 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 26.874711139999999, 2024-09-13T14:58:45.345 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.061440053000000001, 2024-09-13T14:58:45.345 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.346 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.346 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.346 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.346 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.346 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 57219 2024-09-13T14:58:45.346 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.346 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.346 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.346 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.346 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.387677+0000", 2024-09-13T14:58:45.346 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.346 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.346 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.346 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.347 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.387677+0000", 2024-09-13T14:58:45.347 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.347 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.347 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.347 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.347 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.387675+0000", 2024-09-13T14:58:45.347 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:45.347 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.347 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.347 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.347 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.387730+0000", 2024-09-13T14:58:45.347 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4548e-05 2024-09-13T14:58:45.347 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.348 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.348 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.348 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.387741+0000", 2024-09-13T14:58:45.348 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1121000000000001e-05 2024-09-13T14:58:45.348 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.348 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.348 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.348 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.387746+0000", 2024-09-13T14:58:45.348 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1780000000000002e-06 2024-09-13T14:58:45.348 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.348 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.348 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.348 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.387772+0000", 2024-09-13T14:58:45.348 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.6143999999999999e-05 2024-09-13T14:58:45.349 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.349 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.349 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.349 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.387905+0000", 2024-09-13T14:58:45.349 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000132306 2024-09-13T14:58:45.349 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.349 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.349 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:45.349 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.388068+0000", 2024-09-13T14:58:45.349 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016317 2024-09-13T14:58:45.349 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.349 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.349 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.349 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.392320+0000", 2024-09-13T14:58:45.350 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0042526980000000001 2024-09-13T14:58:45.350 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.350 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.350 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.350 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.443777+0000", 2024-09-13T14:58:45.350 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.051456285999999997 2024-09-13T14:58:45.350 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.350 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.350 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.350 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.448985+0000", 2024-09-13T14:58:45.350 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.005208753 2024-09-13T14:58:45.350 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.350 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.350 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.351 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.449080+0000", 2024-09-13T14:58:45.351 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5055000000000004e-05 2024-09-13T14:58:45.351 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.351 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.351 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.351 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:18.449117+0000", 2024-09-13T14:58:45.351 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6173000000000001e-05 2024-09-13T14:58:45.351 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.351 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.351 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.351 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.351 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.351 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:78832 9.3 9:c17636d1:::benchmark_data_smithi096_256909_object78831:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T14:58:45.351 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:34.319093+0000", 2024-09-13T14:58:45.352 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 10.943295096, 2024-09-13T14:58:45.352 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.087992743999999998, 2024-09-13T14:58:45.352 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.352 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.352 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.352 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.352 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.352 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 78832 2024-09-13T14:58:45.352 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.352 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.353 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.353 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.353 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.319093+0000", 2024-09-13T14:58:45.353 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.353 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.353 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.353 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.353 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.319093+0000", 2024-09-13T14:58:45.353 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.353 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.353 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.353 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.353 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.319092+0000", 2024-09-13T14:58:45.353 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:45.354 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.354 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.354 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.354 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.319150+0000", 2024-09-13T14:58:45.354 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8749999999999998e-05 2024-09-13T14:58:45.354 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.354 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.354 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.354 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.319158+0000", 2024-09-13T14:58:45.354 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9230000000000002e-06 2024-09-13T14:58:45.354 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.354 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.354 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.354 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.319162+0000", 2024-09-13T14:58:45.355 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9349999999999996e-06 2024-09-13T14:58:45.355 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.355 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.355 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.355 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.319198+0000", 2024-09-13T14:58:45.355 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5781000000000002e-05 2024-09-13T14:58:45.355 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.355 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.355 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.355 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.319282+0000", 2024-09-13T14:58:45.355 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4222999999999996e-05 2024-09-13T14:58:45.355 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.355 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.355 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:45.355 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.319366+0000", 2024-09-13T14:58:45.356 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3700000000000002e-05 2024-09-13T14:58:45.356 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.356 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.356 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.356 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.321520+0000", 2024-09-13T14:58:45.356 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0021542839999999998 2024-09-13T14:58:45.356 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.356 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.356 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.356 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.379328+0000", 2024-09-13T14:58:45.356 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.057807771000000001 2024-09-13T14:58:45.356 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.356 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.356 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.357 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.407026+0000", 2024-09-13T14:58:45.357 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.027697902 2024-09-13T14:58:45.357 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.357 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.357 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.357 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.407061+0000", 2024-09-13T14:58:45.357 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4814000000000002e-05 2024-09-13T14:58:45.357 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.357 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.357 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.357 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.407085+0000", 2024-09-13T14:58:45.357 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4627999999999999e-05 2024-09-13T14:58:45.357 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.357 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.358 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.358 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.358 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.358 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:79405 9.d 9:b149e16c:::benchmark_data_smithi096_256909_object79404:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T14:58:45.358 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:34.851021+0000", 2024-09-13T14:58:45.358 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 10.411366787, 2024-09-13T14:58:45.358 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.10046609099999999, 2024-09-13T14:58:45.358 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.358 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.358 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.358 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.358 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.358 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 79405 2024-09-13T14:58:45.358 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.359 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.359 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.359 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.359 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.851021+0000", 2024-09-13T14:58:45.359 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.359 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.359 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.359 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.359 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.851021+0000", 2024-09-13T14:58:45.359 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.359 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.359 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.359 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.359 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.851019+0000", 2024-09-13T14:58:45.360 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T14:58:45.360 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.360 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.360 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.360 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.851056+0000", 2024-09-13T14:58:45.360 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6922000000000003e-05 2024-09-13T14:58:45.360 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.360 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.360 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.360 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.851072+0000", 2024-09-13T14:58:45.360 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5464e-05 2024-09-13T14:58:45.360 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.360 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.360 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.361 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.851078+0000", 2024-09-13T14:58:45.361 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4030000000000004e-06 2024-09-13T14:58:45.361 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.361 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.361 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.361 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.852248+0000", 2024-09-13T14:58:45.361 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0011695130000000001 2024-09-13T14:58:45.361 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.361 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.361 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.361 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.852356+0000", 2024-09-13T14:58:45.361 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010843 2024-09-13T14:58:45.361 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.362 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.362 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:58:45.362 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.852481+0000", 2024-09-13T14:58:45.362 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012536099999999999 2024-09-13T14:58:45.362 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.362 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.362 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.362 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.861952+0000", 2024-09-13T14:58:45.362 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0094702269999999995 2024-09-13T14:58:45.362 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.362 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.362 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.362 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.862453+0000", 2024-09-13T14:58:45.362 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000501211 2024-09-13T14:58:45.363 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.363 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.363 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.363 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.951341+0000", 2024-09-13T14:58:45.363 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.088887775000000002 2024-09-13T14:58:45.363 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.363 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.363 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.363 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.951421+0000", 2024-09-13T14:58:45.363 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0278999999999995e-05 2024-09-13T14:58:45.363 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.363 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.363 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.363 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.951487+0000", 2024-09-13T14:58:45.364 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6205999999999996e-05 2024-09-13T14:58:45.364 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.364 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.364 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.364 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.364 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.364 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:79410 9.d 9:be9e7622:::benchmark_data_smithi096_256909_object79409:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T14:58:45.364 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:34.852084+0000", 2024-09-13T14:58:45.364 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 10.410304133, 2024-09-13T14:58:45.364 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.104851152, 2024-09-13T14:58:45.364 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.364 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.364 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.364 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.365 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.365 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 79410 2024-09-13T14:58:45.365 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.365 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.365 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.365 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.365 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.852084+0000", 2024-09-13T14:58:45.365 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.365 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.365 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.365 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.365 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.852084+0000", 2024-09-13T14:58:45.365 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.365 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.366 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.366 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.366 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.852083+0000", 2024-09-13T14:58:45.366 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:45.366 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.366 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.366 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.366 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.852138+0000", 2024-09-13T14:58:45.366 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5164000000000002e-05 2024-09-13T14:58:45.366 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.366 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.366 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.366 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.852148+0000", 2024-09-13T14:58:45.366 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0035e-05 2024-09-13T14:58:45.366 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.367 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.367 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.367 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.852154+0000", 2024-09-13T14:58:45.367 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.648e-06 2024-09-13T14:58:45.367 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.367 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.367 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.367 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.857272+0000", 2024-09-13T14:58:45.367 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0051185520000000002 2024-09-13T14:58:45.367 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.367 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.367 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.367 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.857400+0000", 2024-09-13T14:58:45.367 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012751200000000001 2024-09-13T14:58:45.368 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.368 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.368 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:58:45.368 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.857535+0000", 2024-09-13T14:58:45.368 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000134924 2024-09-13T14:58:45.368 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.368 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.368 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.368 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.862229+0000", 2024-09-13T14:58:45.368 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0046942090000000004 2024-09-13T14:58:45.368 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.368 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.368 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.368 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.862590+0000", 2024-09-13T14:58:45.369 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00036082099999999999 2024-09-13T14:58:45.369 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.369 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.369 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.369 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.956776+0000", 2024-09-13T14:58:45.369 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.094186078000000006 2024-09-13T14:58:45.369 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.369 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.369 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.369 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.956891+0000", 2024-09-13T14:58:45.369 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000115413 2024-09-13T14:58:45.369 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.369 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.369 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.370 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.956935+0000", 2024-09-13T14:58:45.370 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3689999999999997e-05 2024-09-13T14:58:45.370 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.370 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.370 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.370 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.370 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.370 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:79411 9.3 9:c4ddf934:::benchmark_data_smithi096_256909_object79410:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T14:58:45.370 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:34.855353+0000", 2024-09-13T14:58:45.370 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 10.407034592, 2024-09-13T14:58:45.370 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.099934133999999994, 2024-09-13T14:58:45.370 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.370 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.371 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.371 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.371 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.371 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 79411 2024-09-13T14:58:45.371 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.371 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.371 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.371 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.371 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.855353+0000", 2024-09-13T14:58:45.371 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.371 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.371 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.371 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.371 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.855353+0000", 2024-09-13T14:58:45.372 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.372 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.372 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.372 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.372 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.855352+0000", 2024-09-13T14:58:45.372 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:45.372 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.372 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.372 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.372 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.855425+0000", 2024-09-13T14:58:45.372 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.271e-05 2024-09-13T14:58:45.372 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.372 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.373 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.373 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.855434+0000", 2024-09-13T14:58:45.373 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.4390000000000006e-06 2024-09-13T14:58:45.373 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.373 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.373 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.373 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.855439+0000", 2024-09-13T14:58:45.373 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1499999999999998e-06 2024-09-13T14:58:45.373 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.373 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.373 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.373 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.855483+0000", 2024-09-13T14:58:45.373 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3921000000000002e-05 2024-09-13T14:58:45.374 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.374 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.374 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.374 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.855590+0000", 2024-09-13T14:58:45.374 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000106717 2024-09-13T14:58:45.374 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.374 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.374 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:45.374 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.855714+0000", 2024-09-13T14:58:45.374 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012450699999999999 2024-09-13T14:58:45.374 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.374 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.374 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.374 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.858968+0000", 2024-09-13T14:58:45.375 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0032540529999999998 2024-09-13T14:58:45.375 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.375 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.375 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.375 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.858976+0000", 2024-09-13T14:58:45.375 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2239999999999998e-06 2024-09-13T14:58:45.375 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.375 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.375 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.376 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.955197+0000", 2024-09-13T14:58:45.376 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.096221511999999995 2024-09-13T14:58:45.376 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.376 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.376 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.376 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.955250+0000", 2024-09-13T14:58:45.376 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2661999999999998e-05 2024-09-13T14:58:45.376 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.376 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.376 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.376 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.955287+0000", 2024-09-13T14:58:45.376 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7497999999999999e-05 2024-09-13T14:58:45.376 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.377 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.377 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.377 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.377 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.377 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:79414 9.e 9:72c42466:::benchmark_data_smithi096_256909_object79413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T14:58:45.377 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:34.858700+0000", 2024-09-13T14:58:45.377 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 10.403687865, 2024-09-13T14:58:45.377 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.105671327, 2024-09-13T14:58:45.377 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.377 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.377 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.377 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.377 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.378 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 79414 2024-09-13T14:58:45.378 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.378 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.378 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.378 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.378 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.858700+0000", 2024-09-13T14:58:45.378 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.378 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.378 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.378 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.378 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.858700+0000", 2024-09-13T14:58:45.378 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.378 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.378 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.379 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.379 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.858699+0000", 2024-09-13T14:58:45.379 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:45.379 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.379 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.379 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.379 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.858739+0000", 2024-09-13T14:58:45.379 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0500999999999997e-05 2024-09-13T14:58:45.379 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.379 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.379 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.379 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.858750+0000", 2024-09-13T14:58:45.379 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0892e-05 2024-09-13T14:58:45.380 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.380 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.380 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.380 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.858756+0000", 2024-09-13T14:58:45.380 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8660000000000002e-06 2024-09-13T14:58:45.380 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.380 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.380 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.380 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.858809+0000", 2024-09-13T14:58:45.380 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3483999999999998e-05 2024-09-13T14:58:45.380 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.380 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.380 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.381 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.859057+0000", 2024-09-13T14:58:45.381 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000247824 2024-09-13T14:58:45.381 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.381 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.381 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:58:45.381 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.859184+0000", 2024-09-13T14:58:45.381 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012650899999999999 2024-09-13T14:58:45.381 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.381 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.381 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.381 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.865833+0000", 2024-09-13T14:58:45.381 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.006648892 2024-09-13T14:58:45.381 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.381 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.382 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.382 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.865839+0000", 2024-09-13T14:58:45.382 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.409e-06 2024-09-13T14:58:45.382 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.382 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.382 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.382 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.964286+0000", 2024-09-13T14:58:45.382 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.098447419999999994 2024-09-13T14:58:45.382 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.382 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.382 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.382 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.964335+0000", 2024-09-13T14:58:45.382 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8715999999999999e-05 2024-09-13T14:58:45.382 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.383 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.383 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.383 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.964371+0000", 2024-09-13T14:58:45.383 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6041000000000003e-05 2024-09-13T14:58:45.383 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.383 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.383 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.383 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.383 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.383 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:79415 9.e 9:7667cbcf:::benchmark_data_smithi096_256909_object79414:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T14:58:45.383 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:34.859963+0000", 2024-09-13T14:58:45.383 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 10.402424511, 2024-09-13T14:58:45.383 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.104510877, 2024-09-13T14:58:45.384 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.384 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.384 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.384 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.384 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.384 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 79415 2024-09-13T14:58:45.384 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.384 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.384 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.384 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.384 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.859963+0000", 2024-09-13T14:58:45.384 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.384 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.384 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.385 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.385 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.859963+0000", 2024-09-13T14:58:45.385 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.385 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.385 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.385 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.385 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.859962+0000", 2024-09-13T14:58:45.385 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T14:58:45.385 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.385 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.385 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.385 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.859998+0000", 2024-09-13T14:58:45.385 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6118000000000002e-05 2024-09-13T14:58:45.385 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.386 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.386 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.386 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.860011+0000", 2024-09-13T14:58:45.386 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.271e-05 2024-09-13T14:58:45.386 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.386 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.386 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.386 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.860017+0000", 2024-09-13T14:58:45.386 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9209999999999997e-06 2024-09-13T14:58:45.386 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.386 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.386 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.387 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.862977+0000", 2024-09-13T14:58:45.387 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002960322 2024-09-13T14:58:45.387 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.387 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.387 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.387 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.863113+0000", 2024-09-13T14:58:45.387 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013584900000000001 2024-09-13T14:58:45.387 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.387 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.387 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:58:45.387 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.863254+0000", 2024-09-13T14:58:45.387 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000141261 2024-09-13T14:58:45.387 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.388 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.388 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.388 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.868517+0000", 2024-09-13T14:58:45.388 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0052622909999999997 2024-09-13T14:58:45.388 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.388 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.388 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.388 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.868590+0000", 2024-09-13T14:58:45.388 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2915000000000002e-05 2024-09-13T14:58:45.388 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.388 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.388 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.388 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.964405+0000", 2024-09-13T14:58:45.389 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.095815314999999998 2024-09-13T14:58:45.389 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.389 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.389 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.389 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.964444+0000", 2024-09-13T14:58:45.389 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8649999999999998e-05 2024-09-13T14:58:45.389 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.389 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.389 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.389 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.964474+0000", 2024-09-13T14:58:45.389 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0614000000000002e-05 2024-09-13T14:58:45.389 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.389 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.389 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.390 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.390 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.390 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:79416 9.e 9:74fa13f1:::benchmark_data_smithi096_256909_object79415:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T14:58:45.390 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:34.860023+0000", 2024-09-13T14:58:45.390 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 10.402364804999999, 2024-09-13T14:58:45.390 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.10598315, 2024-09-13T14:58:45.390 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.390 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.390 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.390 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.390 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.390 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 79416 2024-09-13T14:58:45.390 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.391 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.391 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.391 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.391 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.860023+0000", 2024-09-13T14:58:45.391 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.391 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.391 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.391 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.391 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.860023+0000", 2024-09-13T14:58:45.391 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.391 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.391 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.391 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.391 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.860022+0000", 2024-09-13T14:58:45.392 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:45.392 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.392 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.392 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.392 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.860049+0000", 2024-09-13T14:58:45.392 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.6557e-05 2024-09-13T14:58:45.392 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.392 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.392 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.392 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.860056+0000", 2024-09-13T14:58:45.392 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7089999999999996e-06 2024-09-13T14:58:45.392 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.392 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.393 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.393 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.860060+0000", 2024-09-13T14:58:45.393 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0199999999999996e-06 2024-09-13T14:58:45.393 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.393 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.393 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.393 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.865907+0000", 2024-09-13T14:58:45.393 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.005846792 2024-09-13T14:58:45.393 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.393 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.393 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.393 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.866003+0000", 2024-09-13T14:58:45.396 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6340000000000003e-05 2024-09-13T14:58:45.396 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.396 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.396 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:58:45.396 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.866094+0000", 2024-09-13T14:58:45.396 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1448999999999998e-05 2024-09-13T14:58:45.396 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.396 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.396 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.397 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.868744+0000", 2024-09-13T14:58:45.397 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002649836 2024-09-13T14:58:45.397 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.397 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.397 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.397 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.868943+0000", 2024-09-13T14:58:45.397 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00019908600000000001 2024-09-13T14:58:45.397 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.397 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.397 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.397 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.965802+0000", 2024-09-13T14:58:45.397 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.096858894000000001 2024-09-13T14:58:45.397 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.398 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.398 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.398 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.965974+0000", 2024-09-13T14:58:45.398 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00017178300000000001 2024-09-13T14:58:45.398 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.398 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.398 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.398 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:34.966006+0000", 2024-09-13T14:58:45.398 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2181999999999998e-05 2024-09-13T14:58:45.398 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.398 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.398 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.398 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.399 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.399 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:79739 9.3 9:c071d724:::benchmark_data_smithi096_256909_object79738:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T14:58:45.399 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:35.199528+0000", 2024-09-13T14:58:45.399 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 10.062859766000001, 2024-09-13T14:58:45.399 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.092732284999999998, 2024-09-13T14:58:45.399 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T14:58:45.399 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:45.399 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T14:58:45.399 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T14:58:45.399 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:58:45.399 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 79739 2024-09-13T14:58:45.399 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.399 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T14:58:45.399 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.400 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T14:58:45.400 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:35.199528+0000", 2024-09-13T14:58:45.400 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.400 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.400 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.400 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T14:58:45.400 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:35.199528+0000", 2024-09-13T14:58:45.400 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T14:58:45.400 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.400 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.400 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T14:58:45.400 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:35.199527+0000", 2024-09-13T14:58:45.400 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:45.401 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.401 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.401 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T14:58:45.401 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:35.199882+0000", 2024-09-13T14:58:45.401 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00035511800000000002 2024-09-13T14:58:45.401 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.401 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.401 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T14:58:45.401 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:35.199923+0000", 2024-09-13T14:58:45.401 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1473000000000001e-05 2024-09-13T14:58:45.401 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.401 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.401 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T14:58:45.401 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:35.199937+0000", 2024-09-13T14:58:45.402 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4335000000000001e-05 2024-09-13T14:58:45.402 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.402 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.402 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T14:58:45.402 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:35.199993+0000", 2024-09-13T14:58:45.402 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5238999999999997e-05 2024-09-13T14:58:45.402 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.402 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.402 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T14:58:45.402 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:35.200128+0000", 2024-09-13T14:58:45.402 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000134981 2024-09-13T14:58:45.402 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.402 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.402 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:45.403 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:35.200262+0000", 2024-09-13T14:58:45.403 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000134034 2024-09-13T14:58:45.403 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.403 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.403 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.403 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:35.203707+0000", 2024-09-13T14:58:45.403 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0034452189999999998 2024-09-13T14:58:45.403 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.403 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.403 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T14:58:45.403 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:35.204242+0000", 2024-09-13T14:58:45.403 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00053499400000000001 2024-09-13T14:58:45.403 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.403 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.404 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:45.404 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:35.292174+0000", 2024-09-13T14:58:45.404 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.087932262999999997 2024-09-13T14:58:45.404 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.404 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.404 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T14:58:45.404 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:35.292222+0000", 2024-09-13T14:58:45.404 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8204000000000001e-05 2024-09-13T14:58:45.404 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T14:58:45.404 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T14:58:45.404 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T14:58:45.404 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:35.292260+0000", 2024-09-13T14:58:45.404 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7873999999999997e-05 2024-09-13T14:58:45.404 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.405 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.405 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.405 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T14:58:45.405 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T14:58:45.405 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T14:58:45.405 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:45.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:45.506 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:45.547 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:45.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:45.614 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 55 16 91549 91533 104 62.6875 0.00982608 0.00960726 2024-09-13T14:58:45.649 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:45.678 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:45.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:45.780 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:45.806 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:45.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:45.811 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:58:45.816 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:58:45.909 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:45.921 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:58:45.921 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:58:45.948 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:45.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:45.951 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:58:45.952 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:58:45.991 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:58:45.992 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:58:46.023 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:58:46.023 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:58:46.050 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:46.055 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:58:46.055 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:58:46.090 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:58:46.090 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:58:46.092 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:46.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:46.123 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:58:46.123 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:58:46.153 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:58:46.154 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:58:46.194 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:46.233 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:46.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:46.335 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:46.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.373 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:46.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:46.460 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:46 smithi204 ceph-mon[87071]: from='client.36258 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:58:46.460 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:46 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2158420545' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:46.474 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:46.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.502 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:46.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:46 smithi096 ceph-mon[98901]: from='client.36258 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:58:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:46 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2158420545' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:46.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:46 smithi096 ceph-mon[102928]: from='client.36258 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:58:46.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:46 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2158420545' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:46.603 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:46.615 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 56 16 92572 92556 103.284 63.9375 0.0292017 0.00967691 2024-09-13T14:58:46.648 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:46.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:46.750 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:46.769 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:58:46.783 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:46.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:46.884 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:46.922 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:46.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:47.024 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:47.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.077 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:47.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:47.178 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:47.222 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:47.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:47.324 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:47.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.365 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:47.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:47.466 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:47.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:47 smithi204 ceph-mon[87071]: pgmap v1019: 109 pgs: 109 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T14:58:47.501 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:47.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:47.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:47 smithi096 ceph-mon[98901]: pgmap v1019: 109 pgs: 109 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T14:58:47.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:47 smithi096 ceph-mon[102928]: pgmap v1019: 109 pgs: 109 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T14:58:47.602 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:47.615 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 57 16 93587 93571 102.585 63.4375 0.0158436 0.00974316 2024-09-13T14:58:47.632 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:47.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:47.735 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:47.765 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:47.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:47.868 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:47.893 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:47.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:47.995 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:48.016 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:48.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:48.082 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:58:48.119 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:48.151 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:48.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:48.253 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:48.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.298 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:48.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:48.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:58:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:58:47] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.51.0" 2024-09-13T14:58:48.399 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:48.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.429 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:48.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:48.530 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:48.570 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:48.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:48.615 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 58 16 94575 94559 101.881 61.75 0.0110822 0.00981024 2024-09-13T14:58:48.672 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:48.714 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:48.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:48.816 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:48.850 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:48.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:48.934 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":1018,"stamp":"2024-09-13T14:58:44.078535+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5831137229,"num_objects":88931,"num_object_clones":0,"num_object_copies":266793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":88931,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":89810,"num_write_kb":5702415,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":8404818,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92035,"ondisk_log_size":92035,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22931964,"kb_used_data":16985256,"kb_used_omap":35,"kb_used_meta":5946588,"kb_avail":727029252,"statfs":{"total":767960285184,"available":744477954048,"internally_reserved":0,"allocated":17392902144,"data_stored":17375798648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36830,"internal_metadata":6089306146},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,7,0,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":759693312,"num_objects":11592,"num_object_clones":0,"num_object_copies":34776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11592,"num_write_kb":741888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004370"},"pg_stats":[{"pgid":"1.11","version":"82'100","reported_seq":217,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.903322+0000","last_change":"2024-09-13T14:56:01.088057+0000","last_active":"2024-09-13T14:58:41.903322+0000","last_peered":"2024-09-13T14:58:41.903322+0000","last_clean":"2024-09-13T14:58:41.903322+0000","last_became_active":"2024-09-13T14:55:55.203197+0000","last_became_peered":"2024-09-13T14:55:55.203197+0000","last_unstale":"2024-09-13T14:58:41.903322+0000","last_undegraded":"2024-09-13T14:58:41.903322+0000","last_fullsized":"2024-09-13T14:58:41.903322+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.087994+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.087994+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:55.677897+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":497,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.894383+0000","last_change":"2024-09-13T14:38:13.982573+0000","last_active":"2024-09-13T14:58:41.894383+0000","last_peered":"2024-09-13T14:58:41.894383+0000","last_clean":"2024-09-13T14:58:41.894383+0000","last_became_active":"2024-09-13T14:38:13.981984+0000","last_became_peered":"2024-09-13T14:38:13.981984+0000","last_unstale":"2024-09-13T14:58:41.894383+0000","last_undegraded":"2024-09-13T14:58:41.894383+0000","last_fullsized":"2024-09-13T14:58:41.894383+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:17.024846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":279,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.366280+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:58:29.366280+0000","last_peered":"2024-09-13T14:58:29.366280+0000","last_clean":"2024-09-13T14:58:29.366280+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:58:29.366280+0000","last_undegraded":"2024-09-13T14:58:29.366280+0000","last_fullsized":"2024-09-13T14:58:29.366280+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":43,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.359339+0000","last_change":"2024-09-13T14:56:04.643672+0000","last_active":"2024-09-13T14:58:29.359339+0000","last_peered":"2024-09-13T14:58:29.359339+0000","last_clean":"2024-09-13T14:58:29.359339+0000","last_became_active":"2024-09-13T14:55:55.185995+0000","last_became_peered":"2024-09-13T14:55:55.185995+0000","last_unstale":"2024-09-13T14:58:29.359339+0000","last_undegraded":"2024-09-13T14:58:29.359339+0000","last_fullsized":"2024-09-13T14:58:29.359339+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.643614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.643614+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:36.495546+0000","scrub_duration":0.001441681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":276,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.359924+0000","last_change":"2024-09-13T14:38:13.979669+0000","last_active":"2024-09-13T14:58:29.359924+0000","last_peered":"2024-09-13T14:58:29.359924+0000","last_clean":"2024-09-13T14:58:29.359924+0000","last_became_active":"2024-09-13T14:38:13.979356+0000","last_became_peered":"2024-09-13T14:38:13.979356+0000","last_unstale":"2024-09-13T14:58:29.359924+0000","last_undegraded":"2024-09-13T14:58:29.359924+0000","last_fullsized":"2024-09-13T14:58:29.359924+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:31.284056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":511,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.898246+0000","last_change":"2024-09-13T14:51:44.196384+0000","last_active":"2024-09-13T14:58:41.898246+0000","last_peered":"2024-09-13T14:58:41.898246+0000","last_clean":"2024-09-13T14:58:41.898246+0000","last_became_active":"2024-09-13T14:51:44.196082+0000","last_became_peered":"2024-09-13T14:51:44.196082+0000","last_unstale":"2024-09-13T14:58:41.898246+0000","last_undegraded":"2024-09-13T14:58:41.898246+0000","last_fullsized":"2024-09-13T14:58:41.898246+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:11:03.757803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":648,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.903671+0000","last_change":"2024-09-13T14:56:03.240962+0000","last_active":"2024-09-13T14:58:41.903671+0000","last_peered":"2024-09-13T14:58:41.903671+0000","last_clean":"2024-09-13T14:58:41.903671+0000","last_became_active":"2024-09-13T14:55:55.229584+0000","last_became_peered":"2024-09-13T14:55:55.229584+0000","last_unstale":"2024-09-13T14:58:41.903671+0000","last_undegraded":"2024-09-13T14:58:41.903671+0000","last_fullsized":"2024-09-13T14:58:41.903671+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.240882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.240882+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:23.556276+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":505,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.898521+0000","last_change":"2024-09-13T14:51:44.199134+0000","last_active":"2024-09-13T14:58:41.898521+0000","last_peered":"2024-09-13T14:58:41.898521+0000","last_clean":"2024-09-13T14:58:41.898521+0000","last_became_active":"2024-09-13T14:51:44.198525+0000","last_became_peered":"2024-09-13T14:51:44.198525+0000","last_unstale":"2024-09-13T14:58:41.898521+0000","last_undegraded":"2024-09-13T14:58:41.898521+0000","last_fullsized":"2024-09-13T14:58:41.898521+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:44.009127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":280,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.366345+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T14:58:29.366345+0000","last_peered":"2024-09-13T14:58:29.366345+0000","last_clean":"2024-09-13T14:58:29.366345+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T14:58:29.366345+0000","last_undegraded":"2024-09-13T14:58:29.366345+0000","last_fullsized":"2024-09-13T14:58:29.366345+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":43,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.359985+0000","last_change":"2024-09-13T14:56:03.605447+0000","last_active":"2024-09-13T14:58:29.359985+0000","last_peered":"2024-09-13T14:58:29.359985+0000","last_clean":"2024-09-13T14:58:29.359985+0000","last_became_active":"2024-09-13T14:55:55.218939+0000","last_became_peered":"2024-09-13T14:55:55.218939+0000","last_unstale":"2024-09-13T14:58:29.359985+0000","last_undegraded":"2024-09-13T14:58:29.359985+0000","last_fullsized":"2024-09-13T14:58:29.359985+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.605356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.605356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:12.467371+0000","scrub_duration":0.0020269620000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":158,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.897408+0000","last_change":"2024-09-13T14:51:44.207246+0000","last_active":"2024-09-13T14:58:41.897408+0000","last_peered":"2024-09-13T14:58:41.897408+0000","last_clean":"2024-09-13T14:58:41.897408+0000","last_became_active":"2024-09-13T14:51:44.206637+0000","last_became_peered":"2024-09-13T14:51:44.206637+0000","last_unstale":"2024-09-13T14:58:41.897408+0000","last_undegraded":"2024-09-13T14:58:41.897408+0000","last_fullsized":"2024-09-13T14:58:41.897408+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:42.020301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":107,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.371854+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T14:58:29.371854+0000","last_peered":"2024-09-13T14:58:29.371854+0000","last_clean":"2024-09-13T14:58:29.371854+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T14:58:29.371854+0000","last_undegraded":"2024-09-13T14:58:29.371854+0000","last_fullsized":"2024-09-13T14:58:29.371854+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"313'5642","reported_seq":5675,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:43.943224+0000","last_change":"2024-09-13T14:57:34.564761+0000","last_active":"2024-09-13T14:58:43.943224+0000","last_peered":"2024-09-13T14:58:43.943224+0000","last_clean":"2024-09-13T14:58:43.943224+0000","last_became_active":"2024-09-13T14:57:34.564455+0000","last_became_peered":"2024-09-13T14:57:34.564455+0000","last_unstale":"2024-09-13T14:58:43.943224+0000","last_undegraded":"2024-09-13T14:58:43.943224+0000","last_fullsized":"2024-09-13T14:58:43.943224+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5642,"log_dups_size":0,"ondisk_log_size":5642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:06.596934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369754112,"num_objects":5642,"num_object_clones":0,"num_object_copies":16926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5642,"num_write_kb":361088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":43,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.366498+0000","last_change":"2024-09-13T14:56:01.654803+0000","last_active":"2024-09-13T14:58:29.366498+0000","last_peered":"2024-09-13T14:58:29.366498+0000","last_clean":"2024-09-13T14:58:29.366498+0000","last_became_active":"2024-09-13T14:55:55.188788+0000","last_became_peered":"2024-09-13T14:55:55.188788+0000","last_unstale":"2024-09-13T14:58:29.366498+0000","last_undegraded":"2024-09-13T14:58:29.366498+0000","last_fullsized":"2024-09-13T14:58:29.366498+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.654717+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.654717+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:38.649187+0000","scrub_duration":0.0060902070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":482,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.901828+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:58:41.901828+0000","last_peered":"2024-09-13T14:58:41.901828+0000","last_clean":"2024-09-13T14:58:41.901828+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:58:41.901828+0000","last_undegraded":"2024-09-13T14:58:41.901828+0000","last_fullsized":"2024-09-13T14:58:41.901828+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":299,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.359577+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T14:58:29.359577+0000","last_peered":"2024-09-13T14:58:29.359577+0000","last_clean":"2024-09-13T14:58:29.359577+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T14:58:29.359577+0000","last_undegraded":"2024-09-13T14:58:29.359577+0000","last_fullsized":"2024-09-13T14:58:29.359577+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"313'5378","reported_seq":5411,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:43.563382+0000","last_change":"2024-09-13T14:57:34.842329+0000","last_active":"2024-09-13T14:58:43.563382+0000","last_peered":"2024-09-13T14:58:43.563382+0000","last_clean":"2024-09-13T14:58:43.563382+0000","last_became_active":"2024-09-13T14:57:34.842023+0000","last_became_peered":"2024-09-13T14:57:34.842023+0000","last_unstale":"2024-09-13T14:58:43.563382+0000","last_undegraded":"2024-09-13T14:58:43.563382+0000","last_fullsized":"2024-09-13T14:58:43.563382+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5378,"log_dups_size":0,"ondisk_log_size":5378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:43:26.434623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352452608,"num_objects":5378,"num_object_clones":0,"num_object_copies":16134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5378,"num_write_kb":344192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":461,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.893849+0000","last_change":"2024-09-13T14:56:01.843753+0000","last_active":"2024-09-13T14:58:41.893849+0000","last_peered":"2024-09-13T14:58:41.893849+0000","last_clean":"2024-09-13T14:58:41.893849+0000","last_became_active":"2024-09-13T14:55:55.188079+0000","last_became_peered":"2024-09-13T14:55:55.188079+0000","last_unstale":"2024-09-13T14:58:41.893849+0000","last_undegraded":"2024-09-13T14:58:41.893849+0000","last_fullsized":"2024-09-13T14:58:41.893849+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.843498+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.843498+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:54.013271+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":279,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.365354+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:58:29.365354+0000","last_peered":"2024-09-13T14:58:29.365354+0000","last_clean":"2024-09-13T14:58:29.365354+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:58:29.365354+0000","last_undegraded":"2024-09-13T14:58:29.365354+0000","last_fullsized":"2024-09-13T14:58:29.365354+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":334,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.354452+0000","last_change":"2024-09-13T14:51:11.059483+0000","last_active":"2024-09-13T14:58:29.354452+0000","last_peered":"2024-09-13T14:58:29.354452+0000","last_clean":"2024-09-13T14:58:29.354452+0000","last_became_active":"2024-09-13T14:51:11.059262+0000","last_became_peered":"2024-09-13T14:51:11.059262+0000","last_unstale":"2024-09-13T14:58:29.354452+0000","last_undegraded":"2024-09-13T14:58:29.354452+0000","last_fullsized":"2024-09-13T14:58:29.354452+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:45:35.152631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"313'5438","reported_seq":5457,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:42.287682+0000","last_change":"2024-09-13T14:57:34.560948+0000","last_active":"2024-09-13T14:58:42.287682+0000","last_peered":"2024-09-13T14:58:42.287682+0000","last_clean":"2024-09-13T14:58:42.287682+0000","last_became_active":"2024-09-13T14:57:34.560657+0000","last_became_peered":"2024-09-13T14:57:34.560657+0000","last_unstale":"2024-09-13T14:58:42.287682+0000","last_undegraded":"2024-09-13T14:58:42.287682+0000","last_fullsized":"2024-09-13T14:58:42.287682+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5438,"log_dups_size":0,"ondisk_log_size":5438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:21:56.540221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356384768,"num_objects":5438,"num_object_clones":0,"num_object_copies":16314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5438,"num_write_kb":348032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":146,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.371903+0000","last_change":"2024-09-13T14:56:08.889974+0000","last_active":"2024-09-13T14:58:29.371903+0000","last_peered":"2024-09-13T14:58:29.371903+0000","last_clean":"2024-09-13T14:58:29.371903+0000","last_became_active":"2024-09-13T14:55:55.185755+0000","last_became_peered":"2024-09-13T14:55:55.185755+0000","last_unstale":"2024-09-13T14:58:29.371903+0000","last_undegraded":"2024-09-13T14:58:29.371903+0000","last_fullsized":"2024-09-13T14:58:29.371903+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:08.889814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:08.889814+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:02.911564+0000","scrub_duration":0.0053731569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":289,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.366661+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T14:58:29.366661+0000","last_peered":"2024-09-13T14:58:29.366661+0000","last_clean":"2024-09-13T14:58:29.366661+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T14:58:29.366661+0000","last_undegraded":"2024-09-13T14:58:29.366661+0000","last_fullsized":"2024-09-13T14:58:29.366661+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":107,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.372378+0000","last_change":"2024-09-13T14:51:44.190879+0000","last_active":"2024-09-13T14:58:29.372378+0000","last_peered":"2024-09-13T14:58:29.372378+0000","last_clean":"2024-09-13T14:58:29.372378+0000","last_became_active":"2024-09-13T14:51:44.190617+0000","last_became_peered":"2024-09-13T14:51:44.190617+0000","last_unstale":"2024-09-13T14:58:29.372378+0000","last_undegraded":"2024-09-13T14:58:29.372378+0000","last_fullsized":"2024-09-13T14:58:29.372378+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:19.536298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"313'5605","reported_seq":5624,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:42.285859+0000","last_change":"2024-09-13T14:57:34.556168+0000","last_active":"2024-09-13T14:58:42.285859+0000","last_peered":"2024-09-13T14:58:42.285859+0000","last_clean":"2024-09-13T14:58:42.285859+0000","last_became_active":"2024-09-13T14:57:34.555882+0000","last_became_peered":"2024-09-13T14:57:34.555882+0000","last_unstale":"2024-09-13T14:58:42.285859+0000","last_undegraded":"2024-09-13T14:58:42.285859+0000","last_fullsized":"2024-09-13T14:58:42.285859+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5605,"log_dups_size":0,"ondisk_log_size":5605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:05:49.922717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367329280,"num_objects":5605,"num_object_clones":0,"num_object_copies":16815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5605,"num_write_kb":358720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":43,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.355534+0000","last_change":"2024-09-13T14:56:02.325723+0000","last_active":"2024-09-13T14:58:29.355534+0000","last_peered":"2024-09-13T14:58:29.355534+0000","last_clean":"2024-09-13T14:58:29.355534+0000","last_became_active":"2024-09-13T14:55:55.182743+0000","last_became_peered":"2024-09-13T14:55:55.182743+0000","last_unstale":"2024-09-13T14:58:29.355534+0000","last_undegraded":"2024-09-13T14:58:29.355534+0000","last_fullsized":"2024-09-13T14:58:29.355534+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:02.325629+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:02.325629+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:47.868641+0000","scrub_duration":0.0049176200000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":297,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.358825+0000","last_change":"2024-09-13T14:48:32.206148+0000","last_active":"2024-09-13T14:58:29.358825+0000","last_peered":"2024-09-13T14:58:29.358825+0000","last_clean":"2024-09-13T14:58:29.358825+0000","last_became_active":"2024-09-13T14:48:32.205832+0000","last_became_peered":"2024-09-13T14:48:32.205832+0000","last_unstale":"2024-09-13T14:58:29.358825+0000","last_undegraded":"2024-09-13T14:58:29.358825+0000","last_fullsized":"2024-09-13T14:58:29.358825+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:55.026094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":492,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.905058+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:58:41.905058+0000","last_peered":"2024-09-13T14:58:41.905058+0000","last_clean":"2024-09-13T14:58:41.905058+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:58:41.905058+0000","last_undegraded":"2024-09-13T14:58:41.905058+0000","last_fullsized":"2024-09-13T14:58:41.905058+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"313'5594","reported_seq":5627,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:43.939940+0000","last_change":"2024-09-13T14:57:34.835792+0000","last_active":"2024-09-13T14:58:43.939940+0000","last_peered":"2024-09-13T14:58:43.939940+0000","last_clean":"2024-09-13T14:58:43.939940+0000","last_became_active":"2024-09-13T14:57:34.835446+0000","last_became_peered":"2024-09-13T14:57:34.835446+0000","last_unstale":"2024-09-13T14:58:43.939940+0000","last_undegraded":"2024-09-13T14:58:43.939940+0000","last_fullsized":"2024-09-13T14:58:43.939940+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5594,"log_dups_size":0,"ondisk_log_size":5594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:48.872041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366608384,"num_objects":5594,"num_object_clones":0,"num_object_copies":16782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5594,"num_write_kb":358016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":460,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.366179+0000","last_change":"2024-09-13T14:51:01.996634+0000","last_active":"2024-09-13T14:58:29.366179+0000","last_peered":"2024-09-13T14:58:29.366179+0000","last_clean":"2024-09-13T14:58:29.366179+0000","last_became_active":"2024-09-13T14:51:01.996264+0000","last_became_peered":"2024-09-13T14:51:01.996264+0000","last_unstale":"2024-09-13T14:58:29.366179+0000","last_undegraded":"2024-09-13T14:58:29.366179+0000","last_fullsized":"2024-09-13T14:58:29.366179+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:30.083115+0000","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":495,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.892955+0000","last_change":"2024-09-13T14:38:13.982709+0000","last_active":"2024-09-13T14:58:41.892955+0000","last_peered":"2024-09-13T14:58:41.892955+0000","last_clean":"2024-09-13T14:58:41.892955+0000","last_became_active":"2024-09-13T14:38:13.981812+0000","last_became_peered":"2024-09-13T14:38:13.981812+0000","last_unstale":"2024-09-13T14:58:41.892955+0000","last_undegraded":"2024-09-13T14:58:41.892955+0000","last_fullsized":"2024-09-13T14:58:41.892955+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:45.029718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":297,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.356260+0000","last_change":"2024-09-13T14:48:32.196993+0000","last_active":"2024-09-13T14:58:29.356260+0000","last_peered":"2024-09-13T14:58:29.356260+0000","last_clean":"2024-09-13T14:58:29.356260+0000","last_became_active":"2024-09-13T14:48:32.196681+0000","last_became_peered":"2024-09-13T14:48:32.196681+0000","last_unstale":"2024-09-13T14:58:29.356260+0000","last_undegraded":"2024-09-13T14:58:29.356260+0000","last_fullsized":"2024-09-13T14:58:29.356260+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:56.040494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"313'5682","reported_seq":5715,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:43.573294+0000","last_change":"2024-09-13T14:57:34.835630+0000","last_active":"2024-09-13T14:58:43.573294+0000","last_peered":"2024-09-13T14:58:43.573294+0000","last_clean":"2024-09-13T14:58:43.573294+0000","last_became_active":"2024-09-13T14:57:34.835209+0000","last_became_peered":"2024-09-13T14:57:34.835209+0000","last_unstale":"2024-09-13T14:58:43.573294+0000","last_undegraded":"2024-09-13T14:58:43.573294+0000","last_fullsized":"2024-09-13T14:58:43.573294+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5682,"log_dups_size":0,"ondisk_log_size":5682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:50.159454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372375552,"num_objects":5682,"num_object_clones":0,"num_object_copies":17046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5682,"num_write_kb":363648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":220,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.359410+0000","last_change":"2024-09-13T14:40:27.868402+0000","last_active":"2024-09-13T14:58:29.359410+0000","last_peered":"2024-09-13T14:58:29.359410+0000","last_clean":"2024-09-13T14:58:29.359410+0000","last_became_active":"2024-09-13T14:40:22.966301+0000","last_became_peered":"2024-09-13T14:40:22.966301+0000","last_unstale":"2024-09-13T14:58:29.359410+0000","last_undegraded":"2024-09-13T14:58:29.359410+0000","last_fullsized":"2024-09-13T14:58:29.359410+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:18.660186+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":97,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.372512+0000","last_change":"2024-09-13T14:48:32.200521+0000","last_active":"2024-09-13T14:58:29.372512+0000","last_peered":"2024-09-13T14:58:29.372512+0000","last_clean":"2024-09-13T14:58:29.372512+0000","last_became_active":"2024-09-13T14:48:32.200256+0000","last_became_peered":"2024-09-13T14:48:32.200256+0000","last_unstale":"2024-09-13T14:58:29.372512+0000","last_undegraded":"2024-09-13T14:58:29.372512+0000","last_fullsized":"2024-09-13T14:58:29.372512+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:38.899629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":294,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.355467+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:58:29.355467+0000","last_peered":"2024-09-13T14:58:29.355467+0000","last_clean":"2024-09-13T14:58:29.355467+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:58:29.355467+0000","last_undegraded":"2024-09-13T14:58:29.355467+0000","last_fullsized":"2024-09-13T14:58:29.355467+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"313'5486","reported_seq":5505,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:42.043229+0000","last_change":"2024-09-13T14:57:34.563262+0000","last_active":"2024-09-13T14:58:42.043229+0000","last_peered":"2024-09-13T14:58:42.043229+0000","last_clean":"2024-09-13T14:58:42.043229+0000","last_became_active":"2024-09-13T14:57:34.563081+0000","last_became_peered":"2024-09-13T14:57:34.563081+0000","last_unstale":"2024-09-13T14:58:42.043229+0000","last_undegraded":"2024-09-13T14:58:42.043229+0000","last_fullsized":"2024-09-13T14:58:42.043229+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5486,"log_dups_size":0,"ondisk_log_size":5486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:40.861229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359530496,"num_objects":5486,"num_object_clones":0,"num_object_copies":16458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5486,"num_write_kb":351104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":648,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.903883+0000","last_change":"2024-09-13T14:56:01.254863+0000","last_active":"2024-09-13T14:58:41.903883+0000","last_peered":"2024-09-13T14:58:41.903883+0000","last_clean":"2024-09-13T14:58:41.903883+0000","last_became_active":"2024-09-13T14:55:55.214258+0000","last_became_peered":"2024-09-13T14:55:55.214258+0000","last_unstale":"2024-09-13T14:58:41.903883+0000","last_undegraded":"2024-09-13T14:58:41.903883+0000","last_fullsized":"2024-09-13T14:58:41.903883+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.254778+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.254778+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:26.654454+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":478,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.903135+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:58:41.903135+0000","last_peered":"2024-09-13T14:58:41.903135+0000","last_clean":"2024-09-13T14:58:41.903135+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:58:41.903135+0000","last_undegraded":"2024-09-13T14:58:41.903135+0000","last_fullsized":"2024-09-13T14:58:41.903135+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":297,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.355606+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T14:58:29.355606+0000","last_peered":"2024-09-13T14:58:29.355606+0000","last_clean":"2024-09-13T14:58:29.355606+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T14:58:29.355606+0000","last_undegraded":"2024-09-13T14:58:29.355606+0000","last_fullsized":"2024-09-13T14:58:29.355606+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"313'5605","reported_seq":5638,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:43.927057+0000","last_change":"2024-09-13T14:57:34.562198+0000","last_active":"2024-09-13T14:58:43.927057+0000","last_peered":"2024-09-13T14:58:43.927057+0000","last_clean":"2024-09-13T14:58:43.927057+0000","last_became_active":"2024-09-13T14:57:34.561893+0000","last_became_peered":"2024-09-13T14:57:34.561893+0000","last_unstale":"2024-09-13T14:58:43.927057+0000","last_undegraded":"2024-09-13T14:58:43.927057+0000","last_fullsized":"2024-09-13T14:58:43.927057+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5605,"log_dups_size":0,"ondisk_log_size":5605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:31:18.516029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367329280,"num_objects":5605,"num_object_clones":0,"num_object_copies":16815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5605,"num_write_kb":358720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":223,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.359777+0000","last_change":"2024-09-13T14:48:32.205288+0000","last_active":"2024-09-13T14:58:29.359777+0000","last_peered":"2024-09-13T14:58:29.359777+0000","last_clean":"2024-09-13T14:58:29.359777+0000","last_became_active":"2024-09-13T14:48:32.204657+0000","last_became_peered":"2024-09-13T14:48:32.204657+0000","last_unstale":"2024-09-13T14:58:29.359777+0000","last_undegraded":"2024-09-13T14:58:29.359777+0000","last_fullsized":"2024-09-13T14:58:29.359777+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:52.513329+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":151,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.894516+0000","last_change":"2024-09-13T14:51:11.059644+0000","last_active":"2024-09-13T14:58:41.894516+0000","last_peered":"2024-09-13T14:58:41.894516+0000","last_clean":"2024-09-13T14:58:41.894516+0000","last_became_active":"2024-09-13T14:51:11.058874+0000","last_became_peered":"2024-09-13T14:51:11.058874+0000","last_unstale":"2024-09-13T14:58:41.894516+0000","last_undegraded":"2024-09-13T14:58:41.894516+0000","last_fullsized":"2024-09-13T14:58:41.894516+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:15:14.736476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":479,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.904038+0000","last_change":"2024-09-13T14:38:13.979302+0000","last_active":"2024-09-13T14:58:41.904038+0000","last_peered":"2024-09-13T14:58:41.904038+0000","last_clean":"2024-09-13T14:58:41.904038+0000","last_became_active":"2024-09-13T14:38:13.979012+0000","last_became_peered":"2024-09-13T14:38:13.979012+0000","last_unstale":"2024-09-13T14:58:41.904038+0000","last_undegraded":"2024-09-13T14:58:41.904038+0000","last_fullsized":"2024-09-13T14:58:41.904038+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.570789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"313'5702","reported_seq":5735,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:43.943487+0000","last_change":"2024-09-13T14:57:34.562334+0000","last_active":"2024-09-13T14:58:43.943487+0000","last_peered":"2024-09-13T14:58:43.943487+0000","last_clean":"2024-09-13T14:58:43.943487+0000","last_became_active":"2024-09-13T14:57:34.562006+0000","last_became_peered":"2024-09-13T14:57:34.562006+0000","last_unstale":"2024-09-13T14:58:43.943487+0000","last_undegraded":"2024-09-13T14:58:43.943487+0000","last_fullsized":"2024-09-13T14:58:43.943487+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5702,"log_dups_size":0,"ondisk_log_size":5702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:48.077892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373686272,"num_objects":5702,"num_object_clones":0,"num_object_copies":17106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5702,"num_write_kb":364928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":461,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.365579+0000","last_change":"2024-09-13T14:55:58.568852+0000","last_active":"2024-09-13T14:58:29.365579+0000","last_peered":"2024-09-13T14:58:29.365579+0000","last_clean":"2024-09-13T14:58:29.365579+0000","last_became_active":"2024-09-13T14:55:51.169987+0000","last_became_peered":"2024-09-13T14:55:51.169987+0000","last_unstale":"2024-09-13T14:58:29.365579+0000","last_undegraded":"2024-09-13T14:58:29.365579+0000","last_fullsized":"2024-09-13T14:58:29.365579+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.568750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.568750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:56.273429+0000","scrub_duration":0.0074886340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":839,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.360790+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:58:29.360790+0000","last_peered":"2024-09-13T14:58:29.360790+0000","last_clean":"2024-09-13T14:58:29.360790+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:58:29.360790+0000","last_undegraded":"2024-09-13T14:58:29.360790+0000","last_fullsized":"2024-09-13T14:58:29.360790+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":151,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.893486+0000","last_change":"2024-09-13T14:51:11.059953+0000","last_active":"2024-09-13T14:58:41.893486+0000","last_peered":"2024-09-13T14:58:41.893486+0000","last_clean":"2024-09-13T14:58:41.893486+0000","last_became_active":"2024-09-13T14:51:11.059396+0000","last_became_peered":"2024-09-13T14:51:11.059396+0000","last_unstale":"2024-09-13T14:58:41.893486+0000","last_undegraded":"2024-09-13T14:58:41.893486+0000","last_fullsized":"2024-09-13T14:58:41.893486+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:11:51.119142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":97,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.372581+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T14:58:29.372581+0000","last_peered":"2024-09-13T14:58:29.372581+0000","last_clean":"2024-09-13T14:58:29.372581+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T14:58:29.372581+0000","last_undegraded":"2024-09-13T14:58:29.372581+0000","last_fullsized":"2024-09-13T14:58:29.372581+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"313'5575","reported_seq":5608,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:43.949650+0000","last_change":"2024-09-13T14:57:34.836468+0000","last_active":"2024-09-13T14:58:43.949650+0000","last_peered":"2024-09-13T14:58:43.949650+0000","last_clean":"2024-09-13T14:58:43.949650+0000","last_became_active":"2024-09-13T14:57:34.836144+0000","last_became_peered":"2024-09-13T14:57:34.836144+0000","last_unstale":"2024-09-13T14:58:43.949650+0000","last_undegraded":"2024-09-13T14:58:43.949650+0000","last_fullsized":"2024-09-13T14:58:43.949650+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5575,"log_dups_size":0,"ondisk_log_size":5575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:40:48.019738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365363200,"num_objects":5575,"num_object_clones":0,"num_object_copies":16725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5575,"num_write_kb":356800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":642,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.903514+0000","last_change":"2024-09-13T14:55:59.245841+0000","last_active":"2024-09-13T14:58:41.903514+0000","last_peered":"2024-09-13T14:58:41.903514+0000","last_clean":"2024-09-13T14:58:41.903514+0000","last_became_active":"2024-09-13T14:55:51.159487+0000","last_became_peered":"2024-09-13T14:55:51.159487+0000","last_unstale":"2024-09-13T14:58:41.903514+0000","last_undegraded":"2024-09-13T14:58:41.903514+0000","last_fullsized":"2024-09-13T14:58:41.903514+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.245213+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.245213+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:58:06.592954+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":524,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.365221+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:58:29.365221+0000","last_peered":"2024-09-13T14:58:29.365221+0000","last_clean":"2024-09-13T14:58:29.365221+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:58:29.365221+0000","last_undegraded":"2024-09-13T14:58:29.365221+0000","last_fullsized":"2024-09-13T14:58:29.365221+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":514,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.903273+0000","last_change":"2024-09-13T14:51:11.061677+0000","last_active":"2024-09-13T14:58:41.903273+0000","last_peered":"2024-09-13T14:58:41.903273+0000","last_clean":"2024-09-13T14:58:41.903273+0000","last_became_active":"2024-09-13T14:51:11.061205+0000","last_became_peered":"2024-09-13T14:51:11.061205+0000","last_unstale":"2024-09-13T14:58:41.903273+0000","last_undegraded":"2024-09-13T14:58:41.903273+0000","last_fullsized":"2024-09-13T14:58:41.903273+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:38.521784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":517,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.899207+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T14:58:41.899207+0000","last_peered":"2024-09-13T14:58:41.899207+0000","last_clean":"2024-09-13T14:58:41.899207+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T14:58:41.899207+0000","last_undegraded":"2024-09-13T14:58:41.899207+0000","last_fullsized":"2024-09-13T14:58:41.899207+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"313'5533","reported_seq":5552,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:42.263964+0000","last_change":"2024-09-13T14:57:34.559226+0000","last_active":"2024-09-13T14:58:42.263964+0000","last_peered":"2024-09-13T14:58:42.263964+0000","last_clean":"2024-09-13T14:58:42.263964+0000","last_became_active":"2024-09-13T14:57:34.558860+0000","last_became_peered":"2024-09-13T14:57:34.558860+0000","last_unstale":"2024-09-13T14:58:42.263964+0000","last_undegraded":"2024-09-13T14:58:42.263964+0000","last_fullsized":"2024-09-13T14:58:42.263964+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5533,"log_dups_size":0,"ondisk_log_size":5533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:56:32.571916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362610688,"num_objects":5533,"num_object_clones":0,"num_object_copies":16599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5533,"num_write_kb":354112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":90,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.367229+0000","last_change":"2024-09-13T14:55:59.574827+0000","last_active":"2024-09-13T14:58:29.367229+0000","last_peered":"2024-09-13T14:58:29.367229+0000","last_clean":"2024-09-13T14:58:29.367229+0000","last_became_active":"2024-09-13T14:55:51.162903+0000","last_became_peered":"2024-09-13T14:55:51.162903+0000","last_unstale":"2024-09-13T14:58:29.367229+0000","last_undegraded":"2024-09-13T14:58:29.367229+0000","last_fullsized":"2024-09-13T14:58:29.367229+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.574773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.574773+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:39.074072+0000","scrub_duration":0.001553708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":841,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.892704+0000","last_change":"2024-09-13T14:56:41.406888+0000","last_active":"2024-09-13T14:58:41.892704+0000","last_peered":"2024-09-13T14:58:41.892704+0000","last_clean":"2024-09-13T14:58:41.892704+0000","last_became_active":"2024-09-13T14:56:41.406299+0000","last_became_peered":"2024-09-13T14:56:41.406299+0000","last_unstale":"2024-09-13T14:58:41.892704+0000","last_undegraded":"2024-09-13T14:58:41.892704+0000","last_fullsized":"2024-09-13T14:58:41.892704+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:53:00.535322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":494,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.905497+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T14:58:41.905497+0000","last_peered":"2024-09-13T14:58:41.905497+0000","last_clean":"2024-09-13T14:58:41.905497+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T14:58:41.905497+0000","last_undegraded":"2024-09-13T14:58:41.905497+0000","last_fullsized":"2024-09-13T14:58:41.905497+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":512,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.903719+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:58:41.903719+0000","last_peered":"2024-09-13T14:58:41.903719+0000","last_clean":"2024-09-13T14:58:41.903719+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:58:41.903719+0000","last_undegraded":"2024-09-13T14:58:41.903719+0000","last_fullsized":"2024-09-13T14:58:41.903719+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"313'5473","reported_seq":5506,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:43.970986+0000","last_change":"2024-09-13T14:57:34.564910+0000","last_active":"2024-09-13T14:58:43.970986+0000","last_peered":"2024-09-13T14:58:43.970986+0000","last_clean":"2024-09-13T14:58:43.970986+0000","last_became_active":"2024-09-13T14:57:34.564633+0000","last_became_peered":"2024-09-13T14:57:34.564633+0000","last_unstale":"2024-09-13T14:58:43.970986+0000","last_undegraded":"2024-09-13T14:58:43.970986+0000","last_fullsized":"2024-09-13T14:58:43.970986+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5473,"log_dups_size":0,"ondisk_log_size":5473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:53:38.983345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358678528,"num_objects":5473,"num_object_clones":0,"num_object_copies":16419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5473,"num_write_kb":350272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":233,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.359057+0000","last_change":"2024-09-13T14:56:01.658240+0000","last_active":"2024-09-13T14:58:29.359057+0000","last_peered":"2024-09-13T14:58:29.359057+0000","last_clean":"2024-09-13T14:58:29.359057+0000","last_became_active":"2024-09-13T14:55:51.160480+0000","last_became_peered":"2024-09-13T14:55:51.160480+0000","last_unstale":"2024-09-13T14:58:29.359057+0000","last_undegraded":"2024-09-13T14:58:29.359057+0000","last_fullsized":"2024-09-13T14:58:29.359057+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.658143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.658143+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:19.262955+0000","scrub_duration":0.0038903079999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":939,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.360134+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T14:58:29.360134+0000","last_peered":"2024-09-13T14:58:29.360134+0000","last_clean":"2024-09-13T14:58:29.360134+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T14:58:29.360134+0000","last_undegraded":"2024-09-13T14:58:29.360134+0000","last_fullsized":"2024-09-13T14:58:29.360134+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":298,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.355079+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T14:58:29.355079+0000","last_peered":"2024-09-13T14:58:29.355079+0000","last_clean":"2024-09-13T14:58:29.355079+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T14:58:29.355079+0000","last_undegraded":"2024-09-13T14:58:29.355079+0000","last_fullsized":"2024-09-13T14:58:29.355079+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":512,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.904301+0000","last_change":"2024-09-13T14:51:11.060994+0000","last_active":"2024-09-13T14:58:41.904301+0000","last_peered":"2024-09-13T14:58:41.904301+0000","last_clean":"2024-09-13T14:58:41.904301+0000","last_became_active":"2024-09-13T14:51:11.060666+0000","last_became_peered":"2024-09-13T14:51:11.060666+0000","last_unstale":"2024-09-13T14:58:41.904301+0000","last_undegraded":"2024-09-13T14:58:41.904301+0000","last_fullsized":"2024-09-13T14:58:41.904301+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:40:15.828766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"313'5465","reported_seq":5484,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:42.259956+0000","last_change":"2024-09-13T14:57:34.560777+0000","last_active":"2024-09-13T14:58:42.259956+0000","last_peered":"2024-09-13T14:58:42.259956+0000","last_clean":"2024-09-13T14:58:42.259956+0000","last_became_active":"2024-09-13T14:57:34.560516+0000","last_became_peered":"2024-09-13T14:57:34.560516+0000","last_unstale":"2024-09-13T14:58:42.259956+0000","last_undegraded":"2024-09-13T14:58:42.259956+0000","last_fullsized":"2024-09-13T14:58:42.259956+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5465,"log_dups_size":0,"ondisk_log_size":5465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:06.505880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358154240,"num_objects":5465,"num_object_clones":0,"num_object_copies":16395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5465,"num_write_kb":349760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":137,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.371830+0000","last_change":"2024-09-13T14:56:01.916851+0000","last_active":"2024-09-13T14:58:29.371830+0000","last_peered":"2024-09-13T14:58:29.371830+0000","last_clean":"2024-09-13T14:58:29.371830+0000","last_became_active":"2024-09-13T14:55:51.195404+0000","last_became_peered":"2024-09-13T14:55:51.195404+0000","last_unstale":"2024-09-13T14:58:29.371830+0000","last_undegraded":"2024-09-13T14:58:29.371830+0000","last_fullsized":"2024-09-13T14:58:29.371830+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.916764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.916764+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:47:35.114766+0000","scrub_duration":0.0035298819999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":294,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.360095+0000","last_change":"2024-09-13T14:38:13.980578+0000","last_active":"2024-09-13T14:58:29.360095+0000","last_peered":"2024-09-13T14:58:29.360095+0000","last_clean":"2024-09-13T14:58:29.360095+0000","last_became_active":"2024-09-13T14:38:13.980258+0000","last_became_peered":"2024-09-13T14:38:13.980258+0000","last_unstale":"2024-09-13T14:58:29.360095+0000","last_undegraded":"2024-09-13T14:58:29.360095+0000","last_fullsized":"2024-09-13T14:58:29.360095+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:15.259246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":997,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.904480+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:58:41.904480+0000","last_peered":"2024-09-13T14:58:41.904480+0000","last_clean":"2024-09-13T14:58:41.904480+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:58:41.904480+0000","last_undegraded":"2024-09-13T14:58:41.904480+0000","last_fullsized":"2024-09-13T14:58:41.904480+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":469,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.903764+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:58:41.903764+0000","last_peered":"2024-09-13T14:58:41.903764+0000","last_clean":"2024-09-13T14:58:41.903764+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:58:41.903764+0000","last_undegraded":"2024-09-13T14:58:41.903764+0000","last_fullsized":"2024-09-13T14:58:41.903764+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"313'5460","reported_seq":5479,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:42.066582+0000","last_change":"2024-09-13T14:57:34.841879+0000","last_active":"2024-09-13T14:58:42.066582+0000","last_peered":"2024-09-13T14:58:42.066582+0000","last_clean":"2024-09-13T14:58:42.066582+0000","last_became_active":"2024-09-13T14:57:34.841546+0000","last_became_peered":"2024-09-13T14:57:34.841546+0000","last_unstale":"2024-09-13T14:58:42.066582+0000","last_undegraded":"2024-09-13T14:58:42.066582+0000","last_fullsized":"2024-09-13T14:58:42.066582+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5460,"log_dups_size":0,"ondisk_log_size":5460,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:36.453841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357826560,"num_objects":5460,"num_object_clones":0,"num_object_copies":16380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5460,"num_write_kb":349440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":127,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.371743+0000","last_change":"2024-09-13T14:56:04.997125+0000","last_active":"2024-09-13T14:58:29.371743+0000","last_peered":"2024-09-13T14:58:29.371743+0000","last_clean":"2024-09-13T14:58:29.371743+0000","last_became_active":"2024-09-13T14:55:51.164957+0000","last_became_peered":"2024-09-13T14:55:51.164957+0000","last_unstale":"2024-09-13T14:58:29.371743+0000","last_undegraded":"2024-09-13T14:58:29.371743+0000","last_fullsized":"2024-09-13T14:58:29.371743+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.997001+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.997001+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:58:48.818249+0000","scrub_duration":0.019401547000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":494,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.904390+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T14:58:41.904390+0000","last_peered":"2024-09-13T14:58:41.904390+0000","last_clean":"2024-09-13T14:58:41.904390+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T14:58:41.904390+0000","last_undegraded":"2024-09-13T14:58:41.904390+0000","last_fullsized":"2024-09-13T14:58:41.904390+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":526,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.366710+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:58:29.366710+0000","last_peered":"2024-09-13T14:58:29.366710+0000","last_clean":"2024-09-13T14:58:29.366710+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:58:29.366710+0000","last_undegraded":"2024-09-13T14:58:29.366710+0000","last_fullsized":"2024-09-13T14:58:29.366710+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":482,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.899952+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:58:41.899952+0000","last_peered":"2024-09-13T14:58:41.899952+0000","last_clean":"2024-09-13T14:58:41.899952+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:58:41.899952+0000","last_undegraded":"2024-09-13T14:58:41.899952+0000","last_fullsized":"2024-09-13T14:58:41.899952+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"313'5491","reported_seq":5510,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:42.058450+0000","last_change":"2024-09-13T14:57:34.562351+0000","last_active":"2024-09-13T14:58:42.058450+0000","last_peered":"2024-09-13T14:58:42.058450+0000","last_clean":"2024-09-13T14:58:42.058450+0000","last_became_active":"2024-09-13T14:57:34.561412+0000","last_became_peered":"2024-09-13T14:57:34.561412+0000","last_unstale":"2024-09-13T14:58:42.058450+0000","last_undegraded":"2024-09-13T14:58:42.058450+0000","last_fullsized":"2024-09-13T14:58:42.058450+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5491,"log_dups_size":0,"ondisk_log_size":5491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:37:07.655839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359858176,"num_objects":5491,"num_object_clones":0,"num_object_copies":16473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5491,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":264,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.354889+0000","last_change":"2024-09-13T14:55:58.308418+0000","last_active":"2024-09-13T14:58:29.354889+0000","last_peered":"2024-09-13T14:58:29.354889+0000","last_clean":"2024-09-13T14:58:29.354889+0000","last_became_active":"2024-09-13T14:55:51.160771+0000","last_became_peered":"2024-09-13T14:55:51.160771+0000","last_unstale":"2024-09-13T14:58:29.354889+0000","last_undegraded":"2024-09-13T14:58:29.354889+0000","last_fullsized":"2024-09-13T14:58:29.354889+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.308240+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.308240+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:05.665088+0000","scrub_duration":0.0063105979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":495,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.899019+0000","last_change":"2024-09-13T14:38:13.982660+0000","last_active":"2024-09-13T14:58:41.899019+0000","last_peered":"2024-09-13T14:58:41.899019+0000","last_clean":"2024-09-13T14:58:41.899019+0000","last_became_active":"2024-09-13T14:38:13.982147+0000","last_became_peered":"2024-09-13T14:38:13.982147+0000","last_unstale":"2024-09-13T14:58:41.899019+0000","last_undegraded":"2024-09-13T14:58:41.899019+0000","last_fullsized":"2024-09-13T14:58:41.899019+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:01.083123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":851,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.906803+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T14:58:41.906803+0000","last_peered":"2024-09-13T14:58:41.906803+0000","last_clean":"2024-09-13T14:58:41.906803+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T14:58:41.906803+0000","last_undegraded":"2024-09-13T14:58:41.906803+0000","last_fullsized":"2024-09-13T14:58:41.906803+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":158,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.894601+0000","last_change":"2024-09-13T14:51:44.203186+0000","last_active":"2024-09-13T14:58:41.894601+0000","last_peered":"2024-09-13T14:58:41.894601+0000","last_clean":"2024-09-13T14:58:41.894601+0000","last_became_active":"2024-09-13T14:51:44.202851+0000","last_became_peered":"2024-09-13T14:51:44.202851+0000","last_unstale":"2024-09-13T14:58:41.894601+0000","last_undegraded":"2024-09-13T14:58:41.894601+0000","last_fullsized":"2024-09-13T14:58:41.894601+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:05:49.173340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"313'5774","reported_seq":5807,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:43.949627+0000","last_change":"2024-09-13T14:57:34.842873+0000","last_active":"2024-09-13T14:58:43.949627+0000","last_peered":"2024-09-13T14:58:43.949627+0000","last_clean":"2024-09-13T14:58:43.949627+0000","last_became_active":"2024-09-13T14:57:34.842571+0000","last_became_peered":"2024-09-13T14:57:34.842571+0000","last_unstale":"2024-09-13T14:58:43.949627+0000","last_undegraded":"2024-09-13T14:58:43.949627+0000","last_fullsized":"2024-09-13T14:58:43.949627+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5774,"log_dups_size":0,"ondisk_log_size":5774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:29:23.565563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378404864,"num_objects":5774,"num_object_clones":0,"num_object_copies":17322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5774,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":449,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.894067+0000","last_change":"2024-09-13T14:55:59.914869+0000","last_active":"2024-09-13T14:58:41.894067+0000","last_peered":"2024-09-13T14:58:41.894067+0000","last_clean":"2024-09-13T14:58:41.894067+0000","last_became_active":"2024-09-13T14:55:51.162703+0000","last_became_peered":"2024-09-13T14:55:51.162703+0000","last_unstale":"2024-09-13T14:58:41.894067+0000","last_undegraded":"2024-09-13T14:58:41.894067+0000","last_fullsized":"2024-09-13T14:58:41.894067+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.914801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.914801+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:17.053641+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":503,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.898243+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:58:41.898243+0000","last_peered":"2024-09-13T14:58:41.898243+0000","last_clean":"2024-09-13T14:58:41.898243+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:58:41.898243+0000","last_undegraded":"2024-09-13T14:58:41.898243+0000","last_fullsized":"2024-09-13T14:58:41.898243+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1126,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.372050+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T14:58:29.372050+0000","last_peered":"2024-09-13T14:58:29.372050+0000","last_clean":"2024-09-13T14:58:29.372050+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T14:58:29.372050+0000","last_undegraded":"2024-09-13T14:58:29.372050+0000","last_fullsized":"2024-09-13T14:58:29.372050+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":280,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.354353+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T14:58:29.354353+0000","last_peered":"2024-09-13T14:58:29.354353+0000","last_clean":"2024-09-13T14:58:29.354353+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T14:58:29.354353+0000","last_undegraded":"2024-09-13T14:58:29.354353+0000","last_fullsized":"2024-09-13T14:58:29.354353+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":297,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.355034+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T14:58:29.355034+0000","last_peered":"2024-09-13T14:58:29.355034+0000","last_clean":"2024-09-13T14:58:29.355034+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T14:58:29.355034+0000","last_undegraded":"2024-09-13T14:58:29.355034+0000","last_fullsized":"2024-09-13T14:58:29.355034+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":510,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.900122+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T14:58:41.900122+0000","last_peered":"2024-09-13T14:58:41.900122+0000","last_clean":"2024-09-13T14:58:41.900122+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T14:58:41.900122+0000","last_undegraded":"2024-09-13T14:58:41.900122+0000","last_fullsized":"2024-09-13T14:58:41.900122+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":493,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.900607+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T14:58:41.900607+0000","last_peered":"2024-09-13T14:58:41.900607+0000","last_clean":"2024-09-13T14:58:41.900607+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T14:58:41.900607+0000","last_undegraded":"2024-09-13T14:58:41.900607+0000","last_fullsized":"2024-09-13T14:58:41.900607+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":293,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.366651+0000","last_change":"2024-09-13T14:35:50.983513+0000","last_active":"2024-09-13T14:58:29.366651+0000","last_peered":"2024-09-13T14:58:29.366651+0000","last_clean":"2024-09-13T14:58:29.366651+0000","last_became_active":"2024-09-13T14:35:50.978874+0000","last_became_peered":"2024-09-13T14:35:50.978874+0000","last_unstale":"2024-09-13T14:58:29.366651+0000","last_undegraded":"2024-09-13T14:58:29.366651+0000","last_fullsized":"2024-09-13T14:58:29.366651+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:16.326928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":71,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.901395+0000","last_change":"2024-09-13T14:56:01.235655+0000","last_active":"2024-09-13T14:58:41.901395+0000","last_peered":"2024-09-13T14:58:41.901395+0000","last_clean":"2024-09-13T14:58:41.901395+0000","last_became_active":"2024-09-13T14:55:55.195703+0000","last_became_peered":"2024-09-13T14:55:55.195703+0000","last_unstale":"2024-09-13T14:58:41.901395+0000","last_undegraded":"2024-09-13T14:58:41.901395+0000","last_fullsized":"2024-09-13T14:58:41.901395+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.235497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.235497+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:55.951927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":277,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.359657+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:58:29.359657+0000","last_peered":"2024-09-13T14:58:29.359657+0000","last_clean":"2024-09-13T14:58:29.359657+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:58:29.359657+0000","last_undegraded":"2024-09-13T14:58:29.359657+0000","last_fullsized":"2024-09-13T14:58:29.359657+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":462,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.893015+0000","last_change":"2024-09-13T14:48:32.200953+0000","last_active":"2024-09-13T14:58:41.893015+0000","last_peered":"2024-09-13T14:58:41.893015+0000","last_clean":"2024-09-13T14:58:41.893015+0000","last_became_active":"2024-09-13T14:48:32.200634+0000","last_became_peered":"2024-09-13T14:48:32.200634+0000","last_unstale":"2024-09-13T14:58:41.893015+0000","last_undegraded":"2024-09-13T14:58:41.893015+0000","last_fullsized":"2024-09-13T14:58:41.893015+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:18.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":490,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.898949+0000","last_change":"2024-09-13T14:38:13.977886+0000","last_active":"2024-09-13T14:58:41.898949+0000","last_peered":"2024-09-13T14:58:41.898949+0000","last_clean":"2024-09-13T14:58:41.898949+0000","last_became_active":"2024-09-13T14:38:13.977573+0000","last_became_peered":"2024-09-13T14:38:13.977573+0000","last_unstale":"2024-09-13T14:58:41.898949+0000","last_undegraded":"2024-09-13T14:58:41.898949+0000","last_fullsized":"2024-09-13T14:58:41.898949+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:21.215001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":511,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.902379+0000","last_change":"2024-09-13T14:48:32.201812+0000","last_active":"2024-09-13T14:58:41.902379+0000","last_peered":"2024-09-13T14:58:41.902379+0000","last_clean":"2024-09-13T14:58:41.902379+0000","last_became_active":"2024-09-13T14:48:32.201486+0000","last_became_peered":"2024-09-13T14:48:32.201486+0000","last_unstale":"2024-09-13T14:58:41.902379+0000","last_undegraded":"2024-09-13T14:58:41.902379+0000","last_fullsized":"2024-09-13T14:58:41.902379+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:23:29.693906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":97,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.371981+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T14:58:29.371981+0000","last_peered":"2024-09-13T14:58:29.371981+0000","last_clean":"2024-09-13T14:58:29.371981+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T14:58:29.371981+0000","last_undegraded":"2024-09-13T14:58:29.371981+0000","last_fullsized":"2024-09-13T14:58:29.371981+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":515,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.902195+0000","last_change":"2024-09-13T14:38:13.981851+0000","last_active":"2024-09-13T14:58:41.902195+0000","last_peered":"2024-09-13T14:58:41.902195+0000","last_clean":"2024-09-13T14:58:41.902195+0000","last_became_active":"2024-09-13T14:38:13.981483+0000","last_became_peered":"2024-09-13T14:38:13.981483+0000","last_unstale":"2024-09-13T14:58:41.902195+0000","last_undegraded":"2024-09-13T14:58:41.902195+0000","last_fullsized":"2024-09-13T14:58:41.902195+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:00.887703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":514,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.898406+0000","last_change":"2024-09-13T14:51:44.198771+0000","last_active":"2024-09-13T14:58:41.898406+0000","last_peered":"2024-09-13T14:58:41.898406+0000","last_clean":"2024-09-13T14:58:41.898406+0000","last_became_active":"2024-09-13T14:51:44.198466+0000","last_became_peered":"2024-09-13T14:51:44.198466+0000","last_unstale":"2024-09-13T14:58:41.898406+0000","last_undegraded":"2024-09-13T14:58:41.898406+0000","last_fullsized":"2024-09-13T14:58:41.898406+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:41:16.099954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":478,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.900420+0000","last_change":"2024-09-13T14:38:13.979264+0000","last_active":"2024-09-13T14:58:41.900420+0000","last_peered":"2024-09-13T14:58:41.900420+0000","last_clean":"2024-09-13T14:58:41.900420+0000","last_became_active":"2024-09-13T14:38:13.978974+0000","last_became_peered":"2024-09-13T14:38:13.978974+0000","last_unstale":"2024-09-13T14:58:41.900420+0000","last_undegraded":"2024-09-13T14:58:41.900420+0000","last_fullsized":"2024-09-13T14:58:41.900420+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:20.744499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":511,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.893385+0000","last_change":"2024-09-13T14:38:13.983524+0000","last_active":"2024-09-13T14:58:41.893385+0000","last_peered":"2024-09-13T14:58:41.893385+0000","last_clean":"2024-09-13T14:58:41.893385+0000","last_became_active":"2024-09-13T14:38:13.982185+0000","last_became_peered":"2024-09-13T14:38:13.982185+0000","last_unstale":"2024-09-13T14:58:41.893385+0000","last_undegraded":"2024-09-13T14:58:41.893385+0000","last_fullsized":"2024-09-13T14:58:41.893385+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:50.449504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":296,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.354698+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:58:29.354698+0000","last_peered":"2024-09-13T14:58:29.354698+0000","last_clean":"2024-09-13T14:58:29.354698+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:58:29.354698+0000","last_undegraded":"2024-09-13T14:58:29.354698+0000","last_fullsized":"2024-09-13T14:58:29.354698+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":312,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.904621+0000","last_change":"2024-09-13T14:56:41.388016+0000","last_active":"2024-09-13T14:58:41.904621+0000","last_peered":"2024-09-13T14:58:41.904621+0000","last_clean":"2024-09-13T14:58:41.904621+0000","last_became_active":"2024-09-13T14:56:41.387745+0000","last_became_peered":"2024-09-13T14:56:41.387745+0000","last_unstale":"2024-09-13T14:58:41.904621+0000","last_undegraded":"2024-09-13T14:58:41.904621+0000","last_fullsized":"2024-09-13T14:58:41.904621+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:55.236441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":483,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.899816+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:58:41.899816+0000","last_peered":"2024-09-13T14:58:41.899816+0000","last_clean":"2024-09-13T14:58:41.899816+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:58:41.899816+0000","last_undegraded":"2024-09-13T14:58:41.899816+0000","last_fullsized":"2024-09-13T14:58:41.899816+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":294,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.355006+0000","last_change":"2024-09-13T14:38:13.980031+0000","last_active":"2024-09-13T14:58:29.355006+0000","last_peered":"2024-09-13T14:58:29.355006+0000","last_clean":"2024-09-13T14:58:29.355006+0000","last_became_active":"2024-09-13T14:38:13.979732+0000","last_became_peered":"2024-09-13T14:38:13.979732+0000","last_unstale":"2024-09-13T14:58:29.355006+0000","last_undegraded":"2024-09-13T14:58:29.355006+0000","last_fullsized":"2024-09-13T14:58:29.355006+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:27.943286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":279,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.358756+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:58:29.358756+0000","last_peered":"2024-09-13T14:58:29.358756+0000","last_clean":"2024-09-13T14:58:29.358756+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:58:29.358756+0000","last_undegraded":"2024-09-13T14:58:29.358756+0000","last_fullsized":"2024-09-13T14:58:29.358756+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":294,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.354626+0000","last_change":"2024-09-13T14:38:13.980507+0000","last_active":"2024-09-13T14:58:29.354626+0000","last_peered":"2024-09-13T14:58:29.354626+0000","last_clean":"2024-09-13T14:58:29.354626+0000","last_became_active":"2024-09-13T14:38:13.980209+0000","last_became_peered":"2024-09-13T14:38:13.980209+0000","last_unstale":"2024-09-13T14:58:29.354626+0000","last_undegraded":"2024-09-13T14:58:29.354626+0000","last_fullsized":"2024-09-13T14:58:29.354626+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:39.397840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":318,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.360016+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T14:58:29.360016+0000","last_peered":"2024-09-13T14:58:29.360016+0000","last_clean":"2024-09-13T14:58:29.360016+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T14:58:29.360016+0000","last_undegraded":"2024-09-13T14:58:29.360016+0000","last_fullsized":"2024-09-13T14:58:29.360016+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":268,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.359170+0000","last_change":"2024-09-13T14:37:40.925048+0000","last_active":"2024-09-13T14:58:29.359170+0000","last_peered":"2024-09-13T14:58:29.359170+0000","last_clean":"2024-09-13T14:58:29.359170+0000","last_became_active":"2024-09-13T14:37:40.924378+0000","last_became_peered":"2024-09-13T14:37:40.924378+0000","last_unstale":"2024-09-13T14:58:29.359170+0000","last_undegraded":"2024-09-13T14:58:29.359170+0000","last_fullsized":"2024-09-13T14:58:29.359170+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:52.162996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":519,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.893160+0000","last_change":"2024-09-13T14:48:32.202404+0000","last_active":"2024-09-13T14:58:41.893160+0000","last_peered":"2024-09-13T14:58:41.893160+0000","last_clean":"2024-09-13T14:58:41.893160+0000","last_became_active":"2024-09-13T14:48:32.202064+0000","last_became_peered":"2024-09-13T14:48:32.202064+0000","last_unstale":"2024-09-13T14:58:41.893160+0000","last_undegraded":"2024-09-13T14:58:41.893160+0000","last_fullsized":"2024-09-13T14:58:41.893160+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:25.674819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":300,"reported_epoch":312,"state":"active+clean","last_fresh":"2024-09-13T14:58:29.354270+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T14:58:29.354270+0000","last_peered":"2024-09-13T14:58:29.354270+0000","last_clean":"2024-09-13T14:58:29.354270+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T14:58:29.354270+0000","last_undegraded":"2024-09-13T14:58:29.354270+0000","last_fullsized":"2024-09-13T14:58:29.354270+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":535,"reported_epoch":313,"state":"active+clean","last_fresh":"2024-09-13T14:58:41.898902+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T14:58:41.898902+0000","last_peered":"2024-09-13T14:58:41.898902+0000","last_clean":"2024-09-13T14:58:41.898902+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T14:58:41.898902+0000","last_undegraded":"2024-09-13T14:58:41.898902+0000","last_fullsized":"2024-09-13T14:58:41.898902+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5826347008,"num_objects":88903,"num_object_clones":0,"num_object_copies":266709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":88903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88903,"num_write_kb":5689792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17444962304,"data_stored":17444962304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88903,"ondisk_log_size":88903,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":16172,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2100,"ondisk_log_size":2100,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397897,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4012916,"kb_used_data":3123500,"kb_used_omap":1,"kb_used_meta":889406,"kb_avail":89732236,"statfs":{"total":95995035648,"available":91885809664,"internally_reserved":0,"allocated":3198464000,"data_stored":3196323396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910752088},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594126,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1362092,"kb_used_data":1060580,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92383060,"statfs":{"total":95995035648,"available":94600253440,"internally_reserved":0,"allocated":1086033920,"data_stored":1083901080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790355,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4053500,"kb_used_data":3164784,"kb_used_omap":1,"kb_used_meta":888702,"kb_avail":89691652,"statfs":{"total":95995035648,"available":91844251648,"internally_reserved":0,"allocated":3240738816,"data_stored":3238598753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910031192},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986584,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1938300,"kb_used_data":1049392,"kb_used_omap":1,"kb_used_meta":888894,"kb_avail":91806852,"statfs":{"total":95995035648,"available":94010216448,"internally_reserved":0,"allocated":1074577408,"data_stored":1072436532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910227800},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":104,"seq":446676599032,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3646192,"kb_used_data":2866084,"kb_used_omap":6,"kb_used_meta":780089,"kb_avail":90098960,"statfs":{"total":95995035648,"available":92261335040,"internally_reserved":0,"allocated":2934870016,"data_stored":2932741610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":798812128},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762559,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1869116,"kb_used_data":1084144,"kb_used_omap":8,"kb_used_meta":784951,"kb_avail":91876036,"statfs":{"total":95995035648,"available":94081060864,"internally_reserved":0,"allocated":1110163456,"data_stored":1108023388,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":803790282},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926085,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3971888,"kb_used_data":3214568,"kb_used_omap":7,"kb_used_meta":757304,"kb_avail":89773264,"statfs":{"total":95995035648,"available":91927822336,"internally_reserved":0,"allocated":3291717632,"data_stored":3289584849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":775480009},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122325,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2077960,"kb_used_data":1422204,"kb_used_omap":7,"kb_used_meta":655736,"kb_avail":91667192,"statfs":{"total":95995035648,"available":93867204608,"internally_reserved":0,"allocated":1456336896,"data_stored":1454189040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":671474255},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1467678720,"data_stored":1467678720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3293577216,"data_stored":3293577216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1115488256,"data_stored":1115488256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2939355136,"data_stored":2939355136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1072955392,"data_stored":1072955392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3243245568,"data_stored":3243245568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1077411840,"data_stored":1077411840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3235250176,"data_stored":3235250176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:58:48.939 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T14:58:48.939 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:58:48.940 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 2, 3, 4, 5, 7, 6] 2024-09-13T14:58:48.940 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd out 3 2024-09-13T14:58:48.952 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:48.992 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:48.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:49.094 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:49.145 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:49.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:49.151 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:49 smithi096 ceph-mon[98901]: pgmap v1020: 109 pgs: 109 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T14:58:49.246 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:49.295 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:49.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:49.397 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:49.434 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:49.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:49.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:49 smithi204 ceph-mon[87071]: pgmap v1020: 109 pgs: 109 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T14:58:49.508 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:49 smithi096 ceph-mon[102928]: pgmap v1020: 109 pgs: 109 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T14:58:49.536 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:49.602 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:49.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.603 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:49.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:49.616 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 59 16 95665 95649 101.308 68.125 0.0294768 0.00986559 2024-09-13T14:58:49.643 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:49.704 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:49.763 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:49.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:49.865 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:49.904 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:49.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:50.006 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:50.039 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:50.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:50.142 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:50.181 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:50.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:50.283 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:50.320 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:50.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:50.422 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:50.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.468 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:50.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:58:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:58:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:50.569 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:50.610 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:50.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:50.616 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:58:50.615654+0000 min lat: 0.00112813 max lat: 0.278402 avg lat: 0.00991941 2024-09-13T14:58:50.616 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:58:50.616 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 60 16 96756 96740 100.756 68.1875 0.0175133 0.00991941 2024-09-13T14:58:50.711 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:50.721 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:50.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.753 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:50.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:50.854 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:50.891 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:50.993 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:51.043 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:51.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:51.145 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:51.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.183 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:51.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:51.284 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:51.311 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:51.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:51.413 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:51.447 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:51.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:51.550 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:51.571 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:58:51.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.572 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:51.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:51.616 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 61 16 97658 97642 100.028 56.375 0.00416874 0.00996787 2024-09-13T14:58:51.673 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:51.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.716 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:51.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:51.817 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:51.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.849 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:51.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:51.858 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:51 smithi096 ceph-mon[98901]: pgmap v1021: 109 pgs: 109 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:58:51.859 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:58:51.859 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:51.952 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:51.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.986 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:51.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:52.087 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:52.123 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:52.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:52.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:51 smithi204 ceph-mon[87071]: pgmap v1021: 109 pgs: 109 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:58:52.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:58:52.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:52.225 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:52.253 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:52.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:52.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:51 smithi096 ceph-mon[102928]: pgmap v1021: 109 pgs: 109 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:58:52.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:58:52.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:52.355 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:52.389 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:52.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:52.491 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:52.529 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:52.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:52.616 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 62 16 98610 98594 99.3743 59.5 0.013694 0.0100583 2024-09-13T14:58:52.632 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:52.663 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:52.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:52.765 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:52.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.801 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:52.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:52.902 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:52.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.939 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:52.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:53.040 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:53.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.078 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:53.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:53.179 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:53.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.219 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:53.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:53.321 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:53.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.361 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:53.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:53.462 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:53.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.491 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:53.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:53.592 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:53.616 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 63 16 99673 99657 98.8514 66.4375 0.0163666 0.010111 2024-09-13T14:58:53.625 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:53.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:53.727 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:53.776 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:53.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:53.878 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:53.920 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:53.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:54.022 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:54.043 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:54.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:54.145 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:54.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.188 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:54.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:54.189 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:53 smithi096 ceph-mon[98901]: pgmap v1022: 109 pgs: 109 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:58:54.189 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:53 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:58:54.190 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:53 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:58:54.190 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:53 smithi096 ceph-mon[102928]: pgmap v1022: 109 pgs: 109 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:58:54.190 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:53 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:58:54.190 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:53 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:58:54.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:53 smithi204 ceph-mon[87071]: pgmap v1022: 109 pgs: 109 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:58:54.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:53 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:58:54.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:53 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:58:54.289 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:54.326 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:54.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:54.428 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:54.457 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:54.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:54.559 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:54.593 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:54.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:54.617 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 64 16 100565 100549 98.1771 55.75 0.0102434 0.0101808 2024-09-13T14:58:54.694 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:54.731 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:54.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:54.833 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:54.870 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:54.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:54.972 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:55.002 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:55.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:55.104 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:55.140 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:55.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:55.242 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:55.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.282 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:55.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:55.383 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:55.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.421 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:55.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:55.522 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:55.559 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:55.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:55.617 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 65 16 101705 101689 97.7628 71.25 0.0150168 0.0102243 2024-09-13T14:58:55.660 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:55.687 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:55.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:55.789 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:55.828 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:55.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:55.930 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:55.964 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:55.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:55.990 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:55 smithi096 ceph-mon[98901]: pgmap v1023: 109 pgs: 109 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:58:55.990 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:55 smithi096 ceph-mon[102928]: pgmap v1023: 109 pgs: 109 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:58:56.065 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:56.106 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:56.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:56.208 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:56.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:55 smithi204 ceph-mon[87071]: pgmap v1023: 109 pgs: 109 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:58:56.240 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:56.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:56.342 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:56.405 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:56.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:56.507 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:56.541 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:56.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:56.617 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 66 16 102797 102781 97.3155 68.25 0.00773444 0.0102709 2024-09-13T14:58:56.642 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:56.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.677 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:56.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:56.778 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:56.810 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:56.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:56.912 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:56.945 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:56.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:57.047 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:57.101 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:57.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:57.203 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:57.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.254 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:57.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:57.275 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:58:57.275 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:58:57.355 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:57.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.399 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:57.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:57.500 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:57.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.552 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:57.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:57.617 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 67 16 103785 103769 96.7847 61.75 0.00999021 0.0103276 2024-09-13T14:58:57.653 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:57.690 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:57.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:57.792 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:57.821 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:57.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:57.923 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:57.959 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:57.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:58.060 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:58.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:57 smithi096 ceph-mon[98901]: pgmap v1024: 109 pgs: 109 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s 2024-09-13T14:58:58.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:57 smithi096 ceph-mon[102928]: pgmap v1024: 109 pgs: 109 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s 2024-09-13T14:58:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:58:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:58:57] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.51.0" 2024-09-13T14:58:58.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.089 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:58.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:58.190 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:58.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:57 smithi204 ceph-mon[87071]: pgmap v1024: 109 pgs: 109 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s 2024-09-13T14:58:58.220 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:58.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:58.322 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:58.354 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:58.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:58.455 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:58.479 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:58:58.503 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:58.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:58.605 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:58.618 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 68 16 104818 104802 96.3084 64.5625 0.0150152 0.0103774 2024-09-13T14:58:58.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.645 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:58.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:58.745 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:58.771 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:58.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:58.812 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:58.873 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:58.892 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:58:58.899 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:58.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:58.951 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:58 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3566543724' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:58.951 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:58 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3566543724' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:59.001 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:59.040 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:59.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:59.142 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:59.179 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:59.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:59.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:58 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3566543724' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:59.281 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:59.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.317 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:59.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:59.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:59 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a[98876]: 2024-09-13T14:58:59.123+0000 7f16aeda4640 -1 mon.a@0(leader).osd e314 definitely_dead 0 2024-09-13T14:58:59.418 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:59.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.454 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:59.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:59.555 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:59.585 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:58:59.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:59.619 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 69 16 105775 105759 95.7795 59.8125 0.0183477 0.0104358 2024-09-13T14:58:59.687 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:59.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.723 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:59.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:59.825 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:59.877 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:58:59.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:59.897 INFO:teuthology.orchestra.run.smithi096.stderr:marked out osd.3. 2024-09-13T14:58:59.927 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:59 smithi096 ceph-mon[98901]: pgmap v1025: 109 pgs: 109 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:58:59.927 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:59 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:59.928 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:59 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3566543724' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:59.928 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:59 smithi096 ceph-mon[98901]: osdmap e314: 8 total, 8 up, 8 in 2024-09-13T14:58:59.928 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:59 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1633760897' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-09-13T14:58:59.928 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:58:59 smithi096 ceph-mon[98901]: Client client.admin marked osd.3 out, while it was still marked up 2024-09-13T14:58:59.928 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:59 smithi096 ceph-mon[102928]: pgmap v1025: 109 pgs: 109 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:58:59.928 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:59 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:59.928 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:59 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3566543724' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:59.928 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:59 smithi096 ceph-mon[102928]: osdmap e314: 8 total, 8 up, 8 in 2024-09-13T14:58:59.928 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:59 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1633760897' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-09-13T14:58:59.928 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:58:59 smithi096 ceph-mon[102928]: Client client.admin marked osd.3 out, while it was still marked up 2024-09-13T14:58:59.979 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:00.000 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:00.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:00.102 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:00.123 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:00.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:00.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:59 smithi204 ceph-mon[87071]: pgmap v1025: 109 pgs: 109 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:59:00.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:59 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:00.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:59 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3566543724' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:00.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:59 smithi204 ceph-mon[87071]: osdmap e314: 8 total, 8 up, 8 in 2024-09-13T14:59:00.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:59 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1633760897' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-09-13T14:59:00.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:58:59 smithi204 ceph-mon[87071]: Client client.admin marked osd.3 out, while it was still marked up 2024-09-13T14:59:00.226 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:00.248 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:00.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:00.350 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:00.373 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:00.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:59:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:59:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:00.475 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:00.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.499 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:00.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:00.600 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:00.619 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 70 16 106082 106066 94.6853 19.1875 0.00278613 0.0104485 2024-09-13T14:59:00.622 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:00.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:00.725 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:00.747 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:00.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:00.849 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:00.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.872 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:00.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:00.973 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:00.994 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:00.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:01.096 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:01.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.119 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:01.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:01.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:00 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1633760897' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-09-13T14:59:01.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:00 smithi204 ceph-mon[87071]: osdmap e315: 8 total, 8 up, 7 in 2024-09-13T14:59:01.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:00 smithi204 ceph-mon[87071]: pgmap v1028: 109 pgs: 109 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:59:01.219 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:01.240 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:01.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:01.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:00 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1633760897' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-09-13T14:59:01.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:00 smithi096 ceph-mon[98901]: osdmap e315: 8 total, 8 up, 7 in 2024-09-13T14:59:01.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:00 smithi096 ceph-mon[98901]: pgmap v1028: 109 pgs: 109 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:59:01.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:00 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1633760897' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-09-13T14:59:01.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:00 smithi096 ceph-mon[102928]: osdmap e315: 8 total, 8 up, 7 in 2024-09-13T14:59:01.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:00 smithi096 ceph-mon[102928]: pgmap v1028: 109 pgs: 109 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:59:01.342 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:01.363 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:01.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:01.465 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:01.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.486 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:01.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:01.587 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:01.610 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:01.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:01.619 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 71 16 106082 106066 93.3518 0 - 0.0104485 2024-09-13T14:59:01.712 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:01.734 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:01.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:01.775 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:01.798 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:59:01.836 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:01.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.860 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:01.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:01.961 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:02.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.009 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:02.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:02.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:01 smithi096 ceph-mon[98901]: osdmap e316: 8 total, 8 up, 7 in 2024-09-13T14:59:02.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:01 smithi096 ceph-mon[102928]: osdmap e316: 8 total, 8 up, 7 in 2024-09-13T14:59:02.111 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:02.140 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:02.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:02.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:01 smithi204 ceph-mon[87071]: osdmap e316: 8 total, 8 up, 7 in 2024-09-13T14:59:02.242 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:02.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.270 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:02.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:02.371 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:02.399 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:02.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:02.501 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:02.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.538 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:02.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:02.619 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 72 16 106335 106319 92.275 7.90625 0.0179332 0.010831 2024-09-13T14:59:02.639 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:02.667 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:02.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:02.769 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:02.805 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:02.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:02.910 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:02.944 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:02.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:03.046 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:03.074 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:03.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:03.176 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:03.201 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:02 smithi096 ceph-mon[98901]: 1.c scrub starts 2024-09-13T14:59:03.201 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:02 smithi096 ceph-mon[98901]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 314'6614 2024-09-13T14:59:03.201 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:02 smithi096 ceph-mon[98901]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 314'6705 2024-09-13T14:59:03.202 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:02 smithi096 ceph-mon[98901]: osdmap e317: 8 total, 8 up, 7 in 2024-09-13T14:59:03.202 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:02 smithi096 ceph-mon[98901]: 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 314'6769 2024-09-13T14:59:03.202 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:02 smithi096 ceph-mon[98901]: 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 314'6639 2024-09-13T14:59:03.202 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:02 smithi096 ceph-mon[98901]: pgmap v1031: 109 pgs: 1 remapped+peering, 108 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 747 op/s 2024-09-13T14:59:03.202 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:02 smithi096 ceph-mon[102928]: 1.c scrub starts 2024-09-13T14:59:03.202 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:02 smithi096 ceph-mon[102928]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 314'6614 2024-09-13T14:59:03.202 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:02 smithi096 ceph-mon[102928]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 314'6705 2024-09-13T14:59:03.202 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:02 smithi096 ceph-mon[102928]: osdmap e317: 8 total, 8 up, 7 in 2024-09-13T14:59:03.203 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:02 smithi096 ceph-mon[102928]: 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 314'6769 2024-09-13T14:59:03.203 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:02 smithi096 ceph-mon[102928]: 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 314'6639 2024-09-13T14:59:03.203 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:02 smithi096 ceph-mon[102928]: pgmap v1031: 109 pgs: 1 remapped+peering, 108 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 747 op/s 2024-09-13T14:59:03.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:02 smithi204 ceph-mon[87071]: 1.c scrub starts 2024-09-13T14:59:03.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:02 smithi204 ceph-mon[87071]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 314'6614 2024-09-13T14:59:03.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:02 smithi204 ceph-mon[87071]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 314'6705 2024-09-13T14:59:03.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:02 smithi204 ceph-mon[87071]: osdmap e317: 8 total, 8 up, 7 in 2024-09-13T14:59:03.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:02 smithi204 ceph-mon[87071]: 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 314'6769 2024-09-13T14:59:03.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:02 smithi204 ceph-mon[87071]: 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 314'6639 2024-09-13T14:59:03.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:02 smithi204 ceph-mon[87071]: pgmap v1031: 109 pgs: 1 remapped+peering, 108 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 747 op/s 2024-09-13T14:59:03.222 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:03.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:03.325 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:03.357 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:03.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:03.459 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:03.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.493 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:03.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:03.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:03.619 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 73 16 107093 107077 91.6599 47.375 0.0105283 0.0109046 2024-09-13T14:59:03.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.638 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:03.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:03.739 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:03.781 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:03.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:03.883 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:03.920 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:03.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:04.021 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:04.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.059 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:04.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:04.115 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:59:04.115 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:59:04.146 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:59:04.146 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:59:04.161 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:04.176 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:59:04.176 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:59:04.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.197 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:04.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:04.204 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T14:59:04.204 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:59:04.228 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:59:04.229 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:59:04.256 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:59:04.256 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:59:04.284 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:59:04.284 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:59:04.298 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:04.322 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:59:04.322 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:59:04.343 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:04 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2321711419' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:04.343 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:04 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2321711419' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:04.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.344 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:04.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:04.445 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:04.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:04 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2321711419' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:04.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.489 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:04.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:04.590 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:04.619 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 74 16 107873 107857 91.08 48.75 0.0791214 0.0109726 2024-09-13T14:59:04.624 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:04.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:04.726 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:04.756 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:04.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:04.858 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:04.890 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:04.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:04.992 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:05.026 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:05.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:05.128 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:05.169 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:05.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:05.272 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:05.308 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:05.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:05.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:05 smithi096 ceph-mon[98901]: pgmap v1032: 109 pgs: 2 active+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 activating+remapped, 1 remapped+peering, 102 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 4.2 KiB/s rd, 47 MiB/s wr, 761 op/s; 19577/318171 objects degraded (6.153%); 20080/318171 objects misplaced (6.311%); 65 KiB/s, 0 objects/s recovering 2024-09-13T14:59:05.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:05 smithi096 ceph-mon[98901]: Health check failed: Degraded data redundancy: 19577/318171 objects degraded (6.153%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:05.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:05 smithi096 ceph-mon[102928]: pgmap v1032: 109 pgs: 2 active+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 activating+remapped, 1 remapped+peering, 102 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 4.2 KiB/s rd, 47 MiB/s wr, 761 op/s; 19577/318171 objects degraded (6.153%); 20080/318171 objects misplaced (6.311%); 65 KiB/s, 0 objects/s recovering 2024-09-13T14:59:05.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:05 smithi096 ceph-mon[102928]: Health check failed: Degraded data redundancy: 19577/318171 objects degraded (6.153%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:05.411 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:05.454 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:05.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:05 smithi204 ceph-mon[87071]: pgmap v1032: 109 pgs: 2 active+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 activating+remapped, 1 remapped+peering, 102 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 4.2 KiB/s rd, 47 MiB/s wr, 761 op/s; 19577/318171 objects degraded (6.153%); 20080/318171 objects misplaced (6.311%); 65 KiB/s, 0 objects/s recovering 2024-09-13T14:59:05.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:05 smithi204 ceph-mon[87071]: Health check failed: Degraded data redundancy: 19577/318171 objects degraded (6.153%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:05.556 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:05.591 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:05.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:05.620 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 75 16 108655 108639 90.5173 48.875 0.00740646 0.0110423 2024-09-13T14:59:05.692 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:05.726 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:05.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:05.828 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:05.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.861 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:05.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:05.962 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:05.998 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:05.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:06.101 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:06.132 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:06.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:06.234 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:06.274 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:06.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:06.376 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:06.417 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:06.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:06.438 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:06.453 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 6] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:59:06.453 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:59:06.453 INFO:tasks.thrashosds.thrasher:Killing osd 3, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:59:06.453 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-13T14:59:06.454 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.3 2024-09-13T14:59:06.475 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:59:06.475 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T14:59:06.476 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T14:59:06.476 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T14:59:06.476 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:59:06.519 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:06.550 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:06.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:06.620 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 76 16 109368 109352 89.9126 44.5625 0.0184198 0.0111163 2024-09-13T14:59:06.651 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:06.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.693 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:06.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:06.696 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:59:06 smithi096 systemd[1]: Stopping Ceph osd.3 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T14:59:06.794 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:06.824 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:06.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:06.926 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:06.958 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:06.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:07.060 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:07.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:07.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:07.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:07.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.091 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:07.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:07.192 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:07.227 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:07.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:07.328 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:07.370 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:07.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:07.472 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:07.513 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:07.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:07.615 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:07.619 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 77 16 110136 110120 89.3682 48 0.0124414 0.0111828 2024-09-13T14:59:07.652 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:07.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:07.754 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:07.791 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:07.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:07.893 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:07.923 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:07.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:07.935 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:59:08.025 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:08.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.053 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:08.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:08.154 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:08.199 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:08.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:08.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:07 smithi204 ceph-mon[87071]: pgmap v1033: 109 pgs: 2 active+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 activating+remapped, 1 remapped+peering, 101 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 3.6 KiB/s rd, 23 MiB/s wr, 362 op/s; 19577/319851 objects degraded (6.121%); 26827/319851 objects misplaced (8.387%); 520 KiB/s, 7 objects/s recovering 2024-09-13T14:59:08.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:07 smithi204 ceph-mon[87071]: 1.c scrub starts 2024-09-13T14:59:08.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:07 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:08.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:07 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:08.301 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:08.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:07 smithi096 ceph-mon[98901]: pgmap v1033: 109 pgs: 2 active+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 activating+remapped, 1 remapped+peering, 101 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 3.6 KiB/s rd, 23 MiB/s wr, 362 op/s; 19577/319851 objects degraded (6.121%); 26827/319851 objects misplaced (8.387%); 520 KiB/s, 7 objects/s recovering 2024-09-13T14:59:08.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:07 smithi096 ceph-mon[98901]: 1.c scrub starts 2024-09-13T14:59:08.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:07 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:08.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:07 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:08.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:07 smithi096 ceph-mon[102928]: pgmap v1033: 109 pgs: 2 active+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 activating+remapped, 1 remapped+peering, 101 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 3.6 KiB/s rd, 23 MiB/s wr, 362 op/s; 19577/319851 objects degraded (6.121%); 26827/319851 objects misplaced (8.387%); 520 KiB/s, 7 objects/s recovering 2024-09-13T14:59:08.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:07 smithi096 ceph-mon[102928]: 1.c scrub starts 2024-09-13T14:59:08.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:07 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:08.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:07 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:08.324 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:59:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:59:07] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.51.0" 2024-09-13T14:59:08.336 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:08.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:08.438 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:08.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.478 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:08.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:08.579 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:08.610 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:08.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:08.619 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 78 16 110861 110845 88.8034 45.3125 0.0163171 0.0112549 2024-09-13T14:59:08.712 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:08.741 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:08.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:08.843 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:08.873 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:08.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:08.975 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:09.006 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:09.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:09.108 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:09.150 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:09.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:09.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:08 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:59:09.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:08 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:59:09.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:08 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:09.251 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:09.284 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:09.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:09.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:08 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:59:09.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:08 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:59:09.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:08 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:09.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:08 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:59:09.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:08 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:59:09.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:08 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:09.386 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:09.417 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:09.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:09.519 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:09.555 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:09.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:09.620 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 79 16 111591 111575 88.2569 45.625 0.0666744 0.0113244 2024-09-13T14:59:09.657 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:09.691 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:09.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:09.696 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:59:09.793 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:09.831 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:09.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:09.933 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:09.974 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:09.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:10.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:09 smithi096 ceph-mon[98901]: pgmap v1034: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 101 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 2.7 KiB/s rd, 34 MiB/s wr, 550 op/s; 19577/326346 objects degraded (5.999%); 32960/326346 objects misplaced (10.100%); 7.7 MiB/s, 123 objects/s recovering 2024-09-13T14:59:10.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:09 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 19577/326346 objects degraded (5.999%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:10.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:09 smithi096 ceph-mon[102928]: pgmap v1034: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 101 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 2.7 KiB/s rd, 34 MiB/s wr, 550 op/s; 19577/326346 objects degraded (5.999%); 32960/326346 objects misplaced (10.100%); 7.7 MiB/s, 123 objects/s recovering 2024-09-13T14:59:10.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:09 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 19577/326346 objects degraded (5.999%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:10.075 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:10.114 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:10.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:10.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:09 smithi204 ceph-mon[87071]: pgmap v1034: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 101 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 2.7 KiB/s rd, 34 MiB/s wr, 550 op/s; 19577/326346 objects degraded (5.999%); 32960/326346 objects misplaced (10.100%); 7.7 MiB/s, 123 objects/s recovering 2024-09-13T14:59:10.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:09 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 19577/326346 objects degraded (5.999%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:10.216 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:10.244 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:10.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:10.346 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:10.378 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:10.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:59:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:59:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:10.479 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:10.519 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:10.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:10.620 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T14:59:10.619640+0000 min lat: 0.00112813 max lat: 2.44725 avg lat: 0.0113917 2024-09-13T14:59:10.620 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:59:10.620 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 80 16 112331 112315 87.7318 46.25 0.0372847 0.0113917 2024-09-13T14:59:10.620 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:10.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.662 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:10.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:10.763 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:10.806 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:10.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:10.907 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:10.944 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:10.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:11.045 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:11.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.087 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:11.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:11.188 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:11.235 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:11.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:11.337 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:11.375 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:11.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:11.478 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:11.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.516 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:11.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:11.617 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:11.621 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 81 16 113108 113092 87.2482 48.5625 0.0152177 0.0114563 2024-09-13T14:59:11.657 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:11.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:11.665 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:11.690 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:11.760 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:11.793 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:11.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:11.886 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:59:11 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[126267]: 2024-09-13T14:59:11.810+0000 7fa501991640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:59:11.886 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:59:11 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[126267]: 2024-09-13T14:59:11.810+0000 7fa501991640 -1 osd.3 317 *** Got signal Terminated *** 2024-09-13T14:59:11.886 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:59:11 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[126267]: 2024-09-13T14:59:11.810+0000 7fa501991640 -1 osd.3 317 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:59:11.894 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:11.920 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:11.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:12.022 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:12.044 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:12.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:12.145 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:12.167 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:12.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:12.177 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:11 smithi096 ceph-mon[98901]: pgmap v1035: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 100 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 2.4 KiB/s rd, 41 MiB/s wr, 656 op/s; 19577/331200 objects degraded (5.911%); 33409/331200 objects misplaced (10.087%); 6.7 MiB/s, 107 objects/s recovering 2024-09-13T14:59:12.177 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:11 smithi096 ceph-mon[98901]: osd.3 marked itself down and dead 2024-09-13T14:59:12.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:11 smithi096 ceph-mon[102928]: pgmap v1035: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 100 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 2.4 KiB/s rd, 41 MiB/s wr, 656 op/s; 19577/331200 objects degraded (5.911%); 33409/331200 objects misplaced (10.087%); 6.7 MiB/s, 107 objects/s recovering 2024-09-13T14:59:12.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:11 smithi096 ceph-mon[102928]: osd.3 marked itself down and dead 2024-09-13T14:59:12.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:11 smithi204 ceph-mon[87071]: pgmap v1035: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 100 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 2.4 KiB/s rd, 41 MiB/s wr, 656 op/s; 19577/331200 objects degraded (5.911%); 33409/331200 objects misplaced (10.087%); 6.7 MiB/s, 107 objects/s recovering 2024-09-13T14:59:12.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:11 smithi204 ceph-mon[87071]: osd.3 marked itself down and dead 2024-09-13T14:59:12.268 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:12.289 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:12.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:12.391 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:12.412 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:12.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:12.513 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:12.536 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:12.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:12.620 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 82 16 113329 113313 86.3526 13.8125 0.00681781 0.0114789 2024-09-13T14:59:12.639 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:12.660 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:12.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:12.762 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:12.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.784 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:12.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:12.822 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:59:12 smithi096 podman[265030]: 2024-09-13 14:59:12.558531978 +0000 UTC m=+0.884340895 container died 23f73e3db413d5be7981f58e0a82848e994159130080718b7a2e98d36eb9897d (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-13T14:59:12.885 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:12.925 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:12.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:13.027 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:13.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.064 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:13.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:13.165 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:13.201 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:13.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:13.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:12 smithi204 ceph-mon[87071]: osdmap e318: 8 total, 7 up, 7 in 2024-09-13T14:59:13.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:12 smithi204 ceph-mon[87071]: pgmap v1037: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 100 active+clean; 6.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 2.2 KiB/s rd, 39 MiB/s wr, 617 op/s; 25988/331896 objects degraded (7.830%); 26901/331896 objects misplaced (8.105%); 6.8 MiB/s, 108 objects/s recovering 2024-09-13T14:59:13.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:12 smithi204 ceph-mon[87071]: osdmap e319: 8 total, 7 up, 7 in 2024-09-13T14:59:13.304 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:13.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:12 smithi096 ceph-mon[98901]: osdmap e318: 8 total, 7 up, 7 in 2024-09-13T14:59:13.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:12 smithi096 ceph-mon[98901]: pgmap v1037: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 100 active+clean; 6.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 2.2 KiB/s rd, 39 MiB/s wr, 617 op/s; 25988/331896 objects degraded (7.830%); 26901/331896 objects misplaced (8.105%); 6.8 MiB/s, 108 objects/s recovering 2024-09-13T14:59:13.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:12 smithi096 ceph-mon[98901]: osdmap e319: 8 total, 7 up, 7 in 2024-09-13T14:59:13.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:12 smithi096 ceph-mon[102928]: osdmap e318: 8 total, 7 up, 7 in 2024-09-13T14:59:13.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:12 smithi096 ceph-mon[102928]: pgmap v1037: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 100 active+clean; 6.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 2.2 KiB/s rd, 39 MiB/s wr, 617 op/s; 25988/331896 objects degraded (7.830%); 26901/331896 objects misplaced (8.105%); 6.8 MiB/s, 108 objects/s recovering 2024-09-13T14:59:13.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:12 smithi096 ceph-mon[102928]: osdmap e319: 8 total, 7 up, 7 in 2024-09-13T14:59:13.338 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:13.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:13.440 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:13.480 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:13.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:13.582 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:13.607 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:13.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:13.620 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 83 16 113902 113886 85.7437 35.8125 0.0367362 0.0116571 2024-09-13T14:59:13.709 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:13.738 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:13.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:13.840 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:13.877 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:13.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:13.979 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:14.016 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:14.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:14.044 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:13 smithi096 ceph-mon[98901]: 1.c scrub starts 2024-09-13T14:59:14.044 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:13 smithi096 ceph-mon[98901]: 9.2 continuing backfill to osd.7 from (307'1484,317'7082] MIN to 317'7082 2024-09-13T14:59:14.044 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:13 smithi096 ceph-mon[98901]: 9.3 continuing backfill to osd.5 from (307'1014,317'7073] 9:c3d7b3d1:::benchmark_data_smithi096_256909_object12733:head to 317'7073 2024-09-13T14:59:14.044 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:13 smithi096 ceph-mon[98901]: 9.c continuing backfill to osd.0 from (306'500,317'7234] MIN to 317'7234 2024-09-13T14:59:14.044 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:13 smithi096 ceph-mon[98901]: 9.8 continuing backfill to osd.7 from (306'300,317'7126] 9:101bedb3:::benchmark_data_smithi096_256909_object15566:head to 317'7126 2024-09-13T14:59:14.044 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:13 smithi096 ceph-mon[102928]: 1.c scrub starts 2024-09-13T14:59:14.044 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:13 smithi096 ceph-mon[102928]: 9.2 continuing backfill to osd.7 from (307'1484,317'7082] MIN to 317'7082 2024-09-13T14:59:14.044 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:13 smithi096 ceph-mon[102928]: 9.3 continuing backfill to osd.5 from (307'1014,317'7073] 9:c3d7b3d1:::benchmark_data_smithi096_256909_object12733:head to 317'7073 2024-09-13T14:59:14.044 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:13 smithi096 ceph-mon[102928]: 9.c continuing backfill to osd.0 from (306'500,317'7234] MIN to 317'7234 2024-09-13T14:59:14.045 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:13 smithi096 ceph-mon[102928]: 9.8 continuing backfill to osd.7 from (306'300,317'7126] 9:101bedb3:::benchmark_data_smithi096_256909_object15566:head to 317'7126 2024-09-13T14:59:14.117 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:14.158 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:14.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:14.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:13 smithi204 ceph-mon[87071]: 1.c scrub starts 2024-09-13T14:59:14.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:13 smithi204 ceph-mon[87071]: 9.2 continuing backfill to osd.7 from (307'1484,317'7082] MIN to 317'7082 2024-09-13T14:59:14.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:13 smithi204 ceph-mon[87071]: 9.3 continuing backfill to osd.5 from (307'1014,317'7073] 9:c3d7b3d1:::benchmark_data_smithi096_256909_object12733:head to 317'7073 2024-09-13T14:59:14.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:13 smithi204 ceph-mon[87071]: 9.c continuing backfill to osd.0 from (306'500,317'7234] MIN to 317'7234 2024-09-13T14:59:14.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:13 smithi204 ceph-mon[87071]: 9.8 continuing backfill to osd.7 from (306'300,317'7126] 9:101bedb3:::benchmark_data_smithi096_256909_object15566:head to 317'7126 2024-09-13T14:59:14.260 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:14.296 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:14.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:14.398 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:14.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.438 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:14.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:14.539 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:14.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.579 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:14.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:14.620 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 84 16 114697 114681 85.3145 49.6875 0.0124201 0.0117157 2024-09-13T14:59:14.680 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:14.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.722 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:14.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:14.808 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:59:14.808 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:59:14.808 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:59:14.808 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:59:14.808 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:59:14.808 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:59:14.809 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:59:14.809 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:59:14.809 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:59:14.809 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:59:14.809 INFO:teuthology.orchestra.run.smithi096.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T14:59:14.809 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:59:14.809 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:59:14.809 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:59:14.809 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:59:14.823 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:14.856 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:14.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:14.958 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:14.992 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:14.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:15.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:14 smithi096 ceph-mon[98901]: pgmap v1039: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 remapped+peering, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 100 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 50 MiB/s wr, 792 op/s; 33118/338874 objects degraded (9.773%); 6962/338874 objects misplaced (2.054%); 14 MiB/s, 227 objects/s recovering 2024-09-13T14:59:15.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:14 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 25988/331896 objects degraded (7.830%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:15.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:14 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1514188122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:15.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:14 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3463397797' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:15.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:14 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:15.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:14 smithi096 ceph-mon[102928]: pgmap v1039: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 remapped+peering, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 100 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 50 MiB/s wr, 792 op/s; 33118/338874 objects degraded (9.773%); 6962/338874 objects misplaced (2.054%); 14 MiB/s, 227 objects/s recovering 2024-09-13T14:59:15.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:14 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 25988/331896 objects degraded (7.830%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:15.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:14 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1514188122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:15.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:14 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3463397797' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:15.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:14 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:15.095 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:15.127 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T14:59:15.139 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:15.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:15.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:14 smithi204 ceph-mon[87071]: pgmap v1039: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 remapped+peering, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 100 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 50 MiB/s wr, 792 op/s; 33118/338874 objects degraded (9.773%); 6962/338874 objects misplaced (2.054%); 14 MiB/s, 227 objects/s recovering 2024-09-13T14:59:15.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:14 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 25988/331896 objects degraded (7.830%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:15.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:14 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1514188122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:15.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:14 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3463397797' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:15.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:14 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:15.241 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:15.276 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:15.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:15.378 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:15.411 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:15.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:15.513 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:15.554 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:15.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:15.620 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 85 15 115513 115498 84.9115 51.0625 0.0131388 0.0117704 2024-09-13T14:59:15.656 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:15.688 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:15.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:15.790 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:15.817 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:15.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:15.918 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:15.955 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:15.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:16.057 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:16.089 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:16.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:16.191 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:16.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:15 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:16.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:15 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:59:16.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:15 smithi204 ceph-mon[87071]: osdmap e320: 8 total, 7 up, 7 in 2024-09-13T14:59:16.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.232 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:16.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:16.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:15 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:16.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:15 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:59:16.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:15 smithi096 ceph-mon[98901]: osdmap e320: 8 total, 7 up, 7 in 2024-09-13T14:59:16.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:15 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:16.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:15 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:59:16.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:15 smithi096 ceph-mon[102928]: osdmap e320: 8 total, 7 up, 7 in 2024-09-13T14:59:16.333 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:16.378 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:16.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:16.479 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:16.520 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:16.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:16.620 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 86 16 116338 116322 84.523 51.5 0.00805606 0.0118244 2024-09-13T14:59:16.625 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:16.661 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:16.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:16.757 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:59:16 smithi096 podman[265030]: 2024-09-13 14:59:16.528858829 +0000 UTC m=+4.854667747 container cleanup 23f73e3db413d5be7981f58e0a82848e994159130080718b7a2e98d36eb9897d (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-13T14:59:16.757 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:59:16 smithi096 bash[265030]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3 2024-09-13T14:59:16.763 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:16.802 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:16.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:16.904 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:16.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.956 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:16.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:17.057 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:17.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:16 smithi096 ceph-mon[98901]: pgmap v1041: 109 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 2 remapped+peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 34 MiB/s wr, 550 op/s; 40251/341112 objects degraded (11.800%); 9.2 MiB/s, 146 objects/s recovering 2024-09-13T14:59:17.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:16 smithi096 ceph-mon[102928]: pgmap v1041: 109 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 2 remapped+peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 34 MiB/s wr, 550 op/s; 40251/341112 objects degraded (11.800%); 9.2 MiB/s, 146 objects/s recovering 2024-09-13T14:59:17.073 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:59:16 smithi096 podman[265200]: 2024-09-13 14:59:16.755729719 +0000 UTC m=+4.231685831 container remove 23f73e3db413d5be7981f58e0a82848e994159130080718b7a2e98d36eb9897d (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-13T14:59:17.086 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:17.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:17.189 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:17.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:16 smithi204 ceph-mon[87071]: pgmap v1041: 109 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 2 remapped+peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 34 MiB/s wr, 550 op/s; 40251/341112 objects degraded (11.800%); 9.2 MiB/s, 146 objects/s recovering 2024-09-13T14:59:17.223 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:59:17.224 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:59:17.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.231 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:17.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:17.332 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:17.359 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:17.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:17.461 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:17.488 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:17.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:17.590 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:17.620 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 87 16 117139 117123 84.1268 50.0625 0.0211557 0.0118807 2024-09-13T14:59:17.626 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:17.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:17.728 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:17.756 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:17.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:17.856 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:59:17.856 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:59:17.859 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:17.889 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:17.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:17.899 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:59:17.992 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:18.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.026 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:18.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:18.127 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:18.166 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:18.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:18.183 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:59:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:59:17] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.51.0" 2024-09-13T14:59:18.268 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:18.310 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:18.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:18.411 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:18.442 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:18.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:18.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:18 smithi204 ceph-mon[87071]: 1.c scrub starts 2024-09-13T14:59:18.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:18 smithi204 ceph-mon[87071]: 1.c scrub ok 2024-09-13T14:59:18.545 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:18.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:18 smithi096 ceph-mon[98901]: 1.c scrub starts 2024-09-13T14:59:18.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:18 smithi096 ceph-mon[98901]: 1.c scrub ok 2024-09-13T14:59:18.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:18 smithi096 ceph-mon[102928]: 1.c scrub starts 2024-09-13T14:59:18.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:18 smithi096 ceph-mon[102928]: 1.c scrub ok 2024-09-13T14:59:18.578 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:18.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:18.621 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 88 16 117910 117894 83.7184 48.1875 0.012557 0.0119399 2024-09-13T14:59:18.680 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:18.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.715 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:18.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:18.817 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:18.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.876 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:18.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:18.878 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:59:18.892 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:18.978 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:19.012 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:19.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:19.114 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:19.151 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:19.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:19.224 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:59:19.252 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:19.290 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:19.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:19.392 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:19.428 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:19.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:19.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:19 smithi204 ceph-mon[87071]: pgmap v1042: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 53 MiB/s wr, 841 op/s; 52633/346884 objects degraded (15.173%); 15 MiB/s, 241 objects/s recovering 2024-09-13T14:59:19.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:19 smithi096 ceph-mon[98901]: pgmap v1042: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 53 MiB/s wr, 841 op/s; 52633/346884 objects degraded (15.173%); 15 MiB/s, 241 objects/s recovering 2024-09-13T14:59:19.477 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:19 smithi096 ceph-mon[102928]: pgmap v1042: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 53 MiB/s wr, 841 op/s; 52633/346884 objects degraded (15.173%); 15 MiB/s, 241 objects/s recovering 2024-09-13T14:59:19.530 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:19.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.555 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:19.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:19.621 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 89 16 118620 118604 83.2763 44.375 0.0083588 0.0120009 2024-09-13T14:59:19.657 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:19.692 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:19.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:19.794 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:19.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.826 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:19.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:19.927 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:19.954 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:19.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:20.056 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:20.090 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:20.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:20.192 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:20.222 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:20.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.228 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:20.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:20.329 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:20.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.372 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:20.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:20.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:20 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 52633/346884 objects degraded (15.173%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:59:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:59:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:20.474 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:20.485 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:20 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 52633/346884 objects degraded (15.173%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:20.486 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:20 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 52633/346884 objects degraded (15.173%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:20.523 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:20.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:20.621 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 90 1 119405 119404 82.9066 50 0.0266091 0.012057 2024-09-13T14:59:20.624 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:20.647 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:20.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:20.748 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:20.772 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:20.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:20.874 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:20.898 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:20.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:21.000 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:21.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.023 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:21.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:21.124 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:21.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.146 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:21.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:21.248 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:21.269 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:21.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:21.371 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:21.392 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:21.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:21.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:21 smithi204 ceph-mon[87071]: pgmap v1043: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s; 53162/352269 objects degraded (15.091%); 11 MiB/s, 175 objects/s recovering 2024-09-13T14:59:21.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:21.495 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:21.516 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:21.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:21.573 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:59:21 smithi096 podman[266186]: 2024-09-13 14:59:21.226782081 +0000 UTC m=+0.019640518 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:59:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:21 smithi096 ceph-mon[102928]: pgmap v1043: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s; 53162/352269 objects degraded (15.091%); 11 MiB/s, 175 objects/s recovering 2024-09-13T14:59:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:21.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:21 smithi096 ceph-mon[98901]: pgmap v1043: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s; 53162/352269 objects degraded (15.091%); 11 MiB/s, 175 objects/s recovering 2024-09-13T14:59:21.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:21.618 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:21.621 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Total time run: 90.0151 2024-09-13T14:59:21.621 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Total writes made: 119405 2024-09-13T14:59:21.621 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Write size: 65536 2024-09-13T14:59:21.621 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Object size: 65536 2024-09-13T14:59:21.621 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Bandwidth (MB/sec): 82.9063 2024-09-13T14:59:21.621 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev Bandwidth: 42.0091 2024-09-13T14:59:21.621 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max bandwidth (MB/sec): 228.5 2024-09-13T14:59:21.621 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min bandwidth (MB/sec): 0 2024-09-13T14:59:21.621 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Average IOPS: 1326 2024-09-13T14:59:21.621 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev IOPS: 672.156 2024-09-13T14:59:21.622 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max IOPS: 3656 2024-09-13T14:59:21.622 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min IOPS: 0 2024-09-13T14:59:21.622 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Average Latency(s): 0.0120571 2024-09-13T14:59:21.622 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev Latency(s): 0.0311829 2024-09-13T14:59:21.622 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max latency(s): 2.44725 2024-09-13T14:59:21.622 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min latency(s): 0.00112813 2024-09-13T14:59:21.626 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:59:21.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.653 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:21.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:21.754 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:21.782 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:21.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:21.884 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:21.925 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:21.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:21.985 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:59:21 smithi096 podman[266186]: 2024-09-13 14:59:21.836696347 +0000 UTC m=+0.629554770 container create 841f41428dd8148eaf067d2807b69e08175d90fabbfad205d0dacfc391f54c05 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:59:22.029 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:22.064 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:22.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:22.166 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:22.203 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:22.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:22.305 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:22.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.340 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:22.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:22.441 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:22.473 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:22.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:22.550 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:59:22 smithi096 podman[266186]: 2024-09-13 14:59:22.382714388 +0000 UTC m=+1.175572818 container init 841f41428dd8148eaf067d2807b69e08175d90fabbfad205d0dacfc391f54c05 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:59:22.550 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:59:22 smithi096 podman[266186]: 2024-09-13 14:59:22.388688212 +0000 UTC m=+1.181546642 container start 841f41428dd8148eaf067d2807b69e08175d90fabbfad205d0dacfc391f54c05 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731) 2024-09-13T14:59:22.550 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:59:22 smithi096 podman[266186]: 2024-09-13 14:59:22.549073824 +0000 UTC m=+1.341932251 container attach 841f41428dd8148eaf067d2807b69e08175d90fabbfad205d0dacfc391f54c05 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-13T14:59:22.575 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:22.611 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:22.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:22.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:22.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.746 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:22.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:22.847 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:22.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.885 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:22.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:22.930 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:59:22.931 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:59:22.960 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:59:22.960 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:59:22.981 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:59:22.982 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:59:22.982 INFO:teuthology.orchestra.run.smithi096.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T14:59:22.986 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:23.013 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:59:23.013 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:59:23.019 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:23.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:23.042 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:59:23.043 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:59:23.083 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:59:23.083 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:59:23.118 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:59:23.118 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:59:23.121 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:23.155 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:23.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:23.256 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:23.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.289 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:23.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:23.390 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:23.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.423 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:23.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:23.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:23 smithi204 ceph-mon[87071]: pgmap v1044: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 35 MiB/s wr, 565 op/s; 53022/352743 objects degraded (15.031%); 5.1 MiB/s, 82 objects/s recovering 2024-09-13T14:59:23.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:23 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1859290935' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:23.524 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:23.557 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:23.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:23.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:23 smithi096 ceph-mon[98901]: pgmap v1044: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 35 MiB/s wr, 565 op/s; 53022/352743 objects degraded (15.031%); 5.1 MiB/s, 82 objects/s recovering 2024-09-13T14:59:23.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:23 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1859290935' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:23.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:23 smithi096 ceph-mon[102928]: pgmap v1044: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 35 MiB/s wr, 565 op/s; 53022/352743 objects degraded (15.031%); 5.1 MiB/s, 82 objects/s recovering 2024-09-13T14:59:23.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:23 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1859290935' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:23.659 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:23.694 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:23.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:23.796 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:23.828 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:23.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:23.930 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:23.959 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:23.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:24.061 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:24.098 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:24.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:24.122 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:24.142 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:59:24.142 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:59:24.200 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:24.219 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T14:59:24.233 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:24.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:24.335 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:24.363 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:24.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:24.453 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:24 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/500740317' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:24.453 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:24 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:24.453 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:24 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/500740317' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:24.453 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:24 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:24.465 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:24.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:24 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/500740317' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:24.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:24 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:24.497 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:24.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:24.599 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:24.630 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:24.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:24.732 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:24.773 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:24.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:24.876 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:24.916 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:24.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:25.019 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:25.052 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:25.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:25.154 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:25.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.195 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:25.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:25.296 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:25.331 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:25.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:25.434 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:25.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.465 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:25.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:25.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:25 smithi204 ceph-mon[87071]: pgmap v1045: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 39 MiB/s wr, 676 op/s; 52042/355824 objects degraded (14.626%); 12 MiB/s, 198 objects/s recovering 2024-09-13T14:59:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:25 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:59:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:25 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:25 smithi204 ceph-mon[87071]: osdmap e321: 8 total, 7 up, 7 in 2024-09-13T14:59:25.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:25 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 52042/355824 objects degraded (14.626%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:25.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:25 smithi096 ceph-mon[98901]: pgmap v1045: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 39 MiB/s wr, 676 op/s; 52042/355824 objects degraded (14.626%); 12 MiB/s, 198 objects/s recovering 2024-09-13T14:59:25.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:25 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:59:25.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:25 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:25.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:25 smithi096 ceph-mon[98901]: osdmap e321: 8 total, 7 up, 7 in 2024-09-13T14:59:25.472 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:25 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 52042/355824 objects degraded (14.626%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:25.472 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:25 smithi096 ceph-mon[102928]: pgmap v1045: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 39 MiB/s wr, 676 op/s; 52042/355824 objects degraded (14.626%); 12 MiB/s, 198 objects/s recovering 2024-09-13T14:59:25.472 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:25 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:59:25.472 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:25 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:25.472 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:25 smithi096 ceph-mon[102928]: osdmap e321: 8 total, 7 up, 7 in 2024-09-13T14:59:25.473 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:25 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 52042/355824 objects degraded (14.626%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:25.473 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 14:59:25 smithi096 podman[266186]: 2024-09-13 14:59:25.250772217 +0000 UTC m=+4.043630660 container died 841f41428dd8148eaf067d2807b69e08175d90fabbfad205d0dacfc391f54c05 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True) 2024-09-13T14:59:25.566 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:25.589 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:25.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:25.690 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:25.728 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:25.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:25.830 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:25.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.862 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:25.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:25.963 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:25.993 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:25.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:26.095 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:26.136 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:26.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:26.143 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:59:26.238 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:26.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.287 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:26.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:26.388 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:26.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.429 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:26.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:26.530 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:26.562 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:26.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:26.664 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:26.704 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:26.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:26.806 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:26.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.837 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:26.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:26.938 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:26.977 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:26.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:27.079 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:27.111 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:27.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:27.213 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:27.240 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:27.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:27.342 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:27.369 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:27.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:27.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:27 smithi204 ceph-mon[87071]: pgmap v1047: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 36 MiB/s wr, 681 op/s; 51993/355029 objects degraded (14.645%); 12 MiB/s, 198 objects/s recovering 2024-09-13T14:59:27.471 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:27.496 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:27.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:27.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:27 smithi096 ceph-mon[98901]: pgmap v1047: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 36 MiB/s wr, 681 op/s; 51993/355029 objects degraded (14.645%); 12 MiB/s, 198 objects/s recovering 2024-09-13T14:59:27.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:27 smithi096 ceph-mon[102928]: pgmap v1047: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 36 MiB/s wr, 681 op/s; 51993/355029 objects degraded (14.645%); 12 MiB/s, 198 objects/s recovering 2024-09-13T14:59:27.598 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:27.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.639 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:27.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:27.740 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:27.766 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:27.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:27.868 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:27.903 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:27.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:28.005 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:28.031 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:28.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:28.133 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:28.159 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:28.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:28.262 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:28.299 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:28.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:28.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:59:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:59:27] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.51.0" 2024-09-13T14:59:28.401 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:28.434 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:28.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:28.536 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:28.567 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:28.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:28.669 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:28.695 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:28.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:28.797 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:28.820 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:28.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:28.922 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:28.959 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:28.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:29.061 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:29.101 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:29.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:29.203 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:29.240 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:29.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:29.342 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:29.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.380 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:29.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:29.445 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:29 smithi096 ceph-mon[98901]: pgmap v1048: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 24 MiB/s wr, 717 op/s; 50859/348180 objects degraded (14.607%); 13 MiB/s, 211 objects/s recovering 2024-09-13T14:59:29.445 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:29 smithi096 ceph-mon[102928]: pgmap v1048: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 24 MiB/s wr, 717 op/s; 50859/348180 objects degraded (14.607%); 13 MiB/s, 211 objects/s recovering 2024-09-13T14:59:29.482 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:29.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.513 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:29.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:29.614 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:29.647 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:29.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:29.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:29 smithi204 ceph-mon[87071]: pgmap v1048: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 24 MiB/s wr, 717 op/s; 50859/348180 objects degraded (14.607%); 13 MiB/s, 211 objects/s recovering 2024-09-13T14:59:29.750 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:29.787 DEBUG:teuthology.orchestra.run.smithi096:> sudo pkill -f 'journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.3.service' 2024-09-13T14:59:29.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.789 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:29.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:29.809 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:29.890 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:59:29.890 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-13T14:59:29.890 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:29.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.929 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:29.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:30.030 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:30.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.066 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:30.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:30.167 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:30.200 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:30.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:30.302 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:30.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:30 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 50859/348180 objects degraded (14.607%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:30.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:30 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 50859/348180 objects degraded (14.607%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:30.337 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:30.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:30.439 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:30.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:30 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 50859/348180 objects degraded (14.607%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:59:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:59:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:30.470 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:30.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:30.572 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:30.595 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:59:30.605 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:30.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:30.708 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:30.739 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:30.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:30.841 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:30.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.872 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:30.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:30.973 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:31.008 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:31.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:31.110 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:31.148 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:31.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:31.250 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:31.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.290 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:31.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:31.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:31 smithi096 ceph-mon[98901]: pgmap v1049: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 13 MiB/s wr, 715 op/s; 50343/342852 objects degraded (14.684%); 13 MiB/s, 211 objects/s recovering 2024-09-13T14:59:31.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:31 smithi096 ceph-mon[102928]: pgmap v1049: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 13 MiB/s wr, 715 op/s; 50343/342852 objects degraded (14.684%); 13 MiB/s, 211 objects/s recovering 2024-09-13T14:59:31.390 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:31.427 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:31.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:31.489 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:31 smithi204 ceph-mon[87071]: pgmap v1049: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 13 MiB/s wr, 715 op/s; 50343/342852 objects degraded (14.684%); 13 MiB/s, 211 objects/s recovering 2024-09-13T14:59:31.529 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:31.561 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:31.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:31.663 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:31.694 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:31.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:31.796 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:31.831 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:31.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:31.875 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:59:31.875 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:59:31.875 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.875 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.46040.0:127800 9.2 9:40f7b512:::benchmark_data_smithi096_256909_object8392:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-13T14:59:31.875 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:31.845101+0000", 2024-09-13T14:59:31.875 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.026853215999999999, 2024-09-13T14:59:31.875 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.026880746000000001, 2024-09-13T14:59:31.875 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:31.876 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:31.876 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:59:31.876 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:31.876 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.46040", 2024-09-13T14:59:31.876 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:59:31.876 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 127800 2024-09-13T14:59:31.876 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.876 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:31.876 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.877 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:31.877 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.845101+0000", 2024-09-13T14:59:31.877 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.877 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.877 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.877 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:31.877 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.845101+0000", 2024-09-13T14:59:31.877 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.877 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.877 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.877 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:31.877 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.845102+0000", 2024-09-13T14:59:31.878 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.559e-06 2024-09-13T14:59:31.878 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.878 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.878 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:31.878 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.845108+0000", 2024-09-13T14:59:31.879 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4319999999999998e-06 2024-09-13T14:59:31.879 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.879 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.879 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:31.880 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.845126+0000", 2024-09-13T14:59:31.880 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8318999999999999e-05 2024-09-13T14:59:31.880 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.880 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.880 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:31.880 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.845134+0000", 2024-09-13T14:59:31.880 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.2239999999999995e-06 2024-09-13T14:59:31.880 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.880 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.881 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:31.881 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.845216+0000", 2024-09-13T14:59:31.881 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1798000000000005e-05 2024-09-13T14:59:31.881 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.881 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.881 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:31.881 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.845349+0000", 2024-09-13T14:59:31.881 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013352399999999999 2024-09-13T14:59:31.882 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.882 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.882 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:59:31.882 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.845485+0000", 2024-09-13T14:59:31.882 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000135418 2024-09-13T14:59:31.882 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.882 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.882 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:31.882 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.852456+0000", 2024-09-13T14:59:31.882 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0069713479999999996 2024-09-13T14:59:31.882 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.882 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.882 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:31.883 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.852498+0000", 2024-09-13T14:59:31.883 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2243000000000002e-05 2024-09-13T14:59:31.883 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.883 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:31.883 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.883 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.883 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.883 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.46040.0:127801 9.2 9:4f2ffc17:::benchmark_data_smithi096_256909_object8393:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-13T14:59:31.883 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:31.845326+0000", 2024-09-13T14:59:31.883 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.026628217999999999, 2024-09-13T14:59:31.884 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.026705077000000001, 2024-09-13T14:59:31.884 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:31.884 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:31.884 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:59:31.884 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:31.884 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.46040", 2024-09-13T14:59:31.884 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:59:31.884 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 127801 2024-09-13T14:59:31.884 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.884 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:31.884 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.884 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:31.884 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.845326+0000", 2024-09-13T14:59:31.885 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.885 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.885 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.885 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:31.885 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.845326+0000", 2024-09-13T14:59:31.885 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.885 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.885 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.885 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:31.885 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.845326+0000", 2024-09-13T14:59:31.885 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5799999999999998e-07 2024-09-13T14:59:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.845331+0000", 2024-09-13T14:59:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3329999999999998e-06 2024-09-13T14:59:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.845341+0000", 2024-09-13T14:59:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0448999999999999e-05 2024-09-13T14:59:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:31.886 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.845349+0000", 2024-09-13T14:59:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2579999999999998e-06 2024-09-13T14:59:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.848853+0000", 2024-09-13T14:59:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0035042659999999998 2024-09-13T14:59:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.849009+0000", 2024-09-13T14:59:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00015584299999999999 2024-09-13T14:59:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.887 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:59:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.849138+0000", 2024-09-13T14:59:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012918000000000001 2024-09-13T14:59:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.852833+0000", 2024-09-13T14:59:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.003695154 2024-09-13T14:59:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.888 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.853120+0000", 2024-09-13T14:59:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00028742200000000001 2024-09-13T14:59:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.46040.0:127817 9.4 9:2c940d80:::benchmark_data_smithi096_256909_object8409:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-13T14:59:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:31.866734+0000", 2024-09-13T14:59:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0052200459999999999, 2024-09-13T14:59:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0053371800000000004, 2024-09-13T14:59:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:31.889 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:59:31.890 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:31.890 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.46040", 2024-09-13T14:59:31.890 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:59:31.890 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 127817 2024-09-13T14:59:31.890 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.890 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:31.890 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.890 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:31.890 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866734+0000", 2024-09-13T14:59:31.890 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.890 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.890 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.890 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:31.891 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866734+0000", 2024-09-13T14:59:31.891 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.891 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.891 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.891 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:31.891 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866735+0000", 2024-09-13T14:59:31.891 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.561e-06 2024-09-13T14:59:31.891 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.891 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.891 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:31.891 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866740+0000", 2024-09-13T14:59:31.891 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0239999999999997e-06 2024-09-13T14:59:31.891 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.891 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.892 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:31.892 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866752+0000", 2024-09-13T14:59:31.892 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1335000000000001e-05 2024-09-13T14:59:31.892 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.892 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.892 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:31.892 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866757+0000", 2024-09-13T14:59:31.892 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6949999999999997e-06 2024-09-13T14:59:31.892 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.892 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.892 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:31.892 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866814+0000", 2024-09-13T14:59:31.892 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6149000000000001e-05 2024-09-13T14:59:31.893 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.893 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.893 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:31.893 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866947+0000", 2024-09-13T14:59:31.893 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013388399999999999 2024-09-13T14:59:31.893 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.893 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.893 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,4", 2024-09-13T14:59:31.893 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.867050+0000", 2024-09-13T14:59:31.893 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000103047 2024-09-13T14:59:31.893 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.893 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.893 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:31.894 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.869911+0000", 2024-09-13T14:59:31.894 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0028604429999999998 2024-09-13T14:59:31.894 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.894 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:31.894 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.894 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.894 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.894 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.46040.0:127818 9.2 9:42f1ed88:::benchmark_data_smithi096_256909_object8410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-13T14:59:31.894 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:31.866785+0000", 2024-09-13T14:59:31.894 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0051691860000000001, 2024-09-13T14:59:31.894 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0053221900000000001, 2024-09-13T14:59:31.894 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:31.894 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:31.894 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:59:31.895 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:31.895 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.46040", 2024-09-13T14:59:31.895 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:59:31.895 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 127818 2024-09-13T14:59:31.895 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.895 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:31.895 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.895 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:31.895 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866785+0000", 2024-09-13T14:59:31.895 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.895 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.895 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.895 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:31.895 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866785+0000", 2024-09-13T14:59:31.896 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.896 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.896 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.897 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:31.897 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866785+0000", 2024-09-13T14:59:31.897 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.1399999999999997e-07 2024-09-13T14:59:31.897 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.897 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.897 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:31.897 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866788+0000", 2024-09-13T14:59:31.897 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.712e-06 2024-09-13T14:59:31.897 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.897 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.897 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:31.897 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866797+0000", 2024-09-13T14:59:31.897 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.5650000000000007e-06 2024-09-13T14:59:31.898 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.898 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.898 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:31.898 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866802+0000", 2024-09-13T14:59:31.898 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8999999999999997e-06 2024-09-13T14:59:31.898 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.898 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.898 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:31.898 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866848+0000", 2024-09-13T14:59:31.898 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5772000000000002e-05 2024-09-13T14:59:31.898 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.898 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.898 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:31.898 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.866960+0000", 2024-09-13T14:59:31.899 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000111438 2024-09-13T14:59:31.899 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.899 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.899 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:59:31.899 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.867041+0000", 2024-09-13T14:59:31.899 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1423999999999998e-05 2024-09-13T14:59:31.899 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.899 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.899 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:31.899 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.869543+0000", 2024-09-13T14:59:31.899 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0025016069999999999 2024-09-13T14:59:31.899 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.899 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.900 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:31.900 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.870363+0000", 2024-09-13T14:59:31.900 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00082056200000000005 2024-09-13T14:59:31.900 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.900 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:31.900 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.900 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.900 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.900 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:127819 9.a e321/302 9:5c32e7f5:::benchmark_data_smithi096_256909_object8411:head v 321'7976, mlcod=321'7974)", 2024-09-13T14:59:31.900 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:31.871834+0000", 2024-09-13T14:59:31.900 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.000120312, 2024-09-13T14:59:31.901 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00032362899999999997, 2024-09-13T14:59:31.901 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:31.901 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:31.901 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:59:31.901 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:31.902 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.902 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:31.902 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871834+0000", 2024-09-13T14:59:31.902 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.902 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.902 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.903 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:31.903 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871834+0000", 2024-09-13T14:59:31.903 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.903 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.903 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.904 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:31.904 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871834+0000", 2024-09-13T14:59:31.904 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.8599999999999996e-07 2024-09-13T14:59:31.904 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.904 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.904 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:31.904 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871845+0000", 2024-09-13T14:59:31.904 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0484e-05 2024-09-13T14:59:31.904 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.904 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.904 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:31.904 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871857+0000", 2024-09-13T14:59:31.904 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1888e-05 2024-09-13T14:59:31.905 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.905 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.905 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:31.905 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871864+0000", 2024-09-13T14:59:31.905 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1749999999999999e-06 2024-09-13T14:59:31.905 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.905 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.906 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:31.906 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871917+0000", 2024-09-13T14:59:31.906 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2769999999999998e-05 2024-09-13T14:59:31.906 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.906 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.906 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:31.907 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871947+0000", 2024-09-13T14:59:31.907 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.9788e-05 2024-09-13T14:59:31.907 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.907 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:31.907 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.907 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.908 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.908 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:127813 9.f e321/316 9:f0c74945:::benchmark_data_smithi096_256909_object8405:head v 321'7936, mlcod=0'0)", 2024-09-13T14:59:31.908 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:31.871881+0000", 2024-09-13T14:59:31.908 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 7.2459999999999994e-05, 2024-09-13T14:59:31.908 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00032574600000000002, 2024-09-13T14:59:31.908 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:31.908 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:31.908 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:59:31.908 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:31.908 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.908 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:31.908 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871881+0000", 2024-09-13T14:59:31.909 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.909 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.909 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.909 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:31.909 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871881+0000", 2024-09-13T14:59:31.909 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.909 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.909 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.909 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:31.909 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871882+0000", 2024-09-13T14:59:31.909 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.2800000000000003e-07 2024-09-13T14:59:31.909 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.910 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.910 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:31.910 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871886+0000", 2024-09-13T14:59:31.910 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0080000000000004e-06 2024-09-13T14:59:31.910 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.910 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.910 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:31.910 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871893+0000", 2024-09-13T14:59:31.910 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.503e-06 2024-09-13T14:59:31.910 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.910 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.910 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:31.910 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871899+0000", 2024-09-13T14:59:31.910 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6080000000000003e-06 2024-09-13T14:59:31.910 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.911 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.911 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:31.911 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871963+0000", 2024-09-13T14:59:31.911 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4401999999999998e-05 2024-09-13T14:59:31.911 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.911 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.911 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:31.912 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.871978+0000", 2024-09-13T14:59:31.912 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5106000000000001e-05 2024-09-13T14:59:31.913 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.913 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:31.913 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.913 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.913 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.913 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.46040.0:127800 9.2 e321/318 ondisk, result = 0)", 2024-09-13T14:59:31.914 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:31.872140+0000", 2024-09-13T14:59:31.914 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 4294967295.999814, 2024-09-13T14:59:31.914 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000119762, 2024-09-13T14:59:31.914 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:31.914 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:31.915 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T14:59:31.915 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:31.915 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.915 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:31.915 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872140+0000", 2024-09-13T14:59:31.915 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.915 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.915 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.915 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:31.915 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872140+0000", 2024-09-13T14:59:31.916 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.916 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.916 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.916 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:31.916 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872140+0000", 2024-09-13T14:59:31.916 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1999999999999998e-07 2024-09-13T14:59:31.917 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.917 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.917 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:31.917 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872145+0000", 2024-09-13T14:59:31.917 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3239999999999996e-06 2024-09-13T14:59:31.917 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.917 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.917 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:31.917 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872157+0000", 2024-09-13T14:59:31.917 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.269e-05 2024-09-13T14:59:31.918 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.918 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.918 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:31.918 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872164+0000", 2024-09-13T14:59:31.918 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7320000000000001e-06 2024-09-13T14:59:31.918 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.918 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.918 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:31.919 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872214+0000", 2024-09-13T14:59:31.919 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9954000000000003e-05 2024-09-13T14:59:31.919 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.919 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.920 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:31.920 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872222+0000", 2024-09-13T14:59:31.920 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.4329999999999998e-06 2024-09-13T14:59:31.920 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.920 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:31.920 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.920 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.920 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.920 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.46040.0:127801 9.2 e321/318)", 2024-09-13T14:59:31.920 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:31.872179+0000", 2024-09-13T14:59:31.921 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 4294967295.9997749, 2024-09-13T14:59:31.921 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000107623, 2024-09-13T14:59:31.921 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:31.921 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:31.921 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "queued for pg", 2024-09-13T14:59:31.921 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:31.921 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.921 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:31.921 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872179+0000", 2024-09-13T14:59:31.921 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.921 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.921 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.922 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:31.922 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872179+0000", 2024-09-13T14:59:31.922 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.922 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.922 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.922 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:31.922 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872179+0000", 2024-09-13T14:59:31.922 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.2500000000000001e-07 2024-09-13T14:59:31.922 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.922 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.922 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:31.922 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872181+0000", 2024-09-13T14:59:31.922 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0439999999999998e-06 2024-09-13T14:59:31.923 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.923 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.923 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:31.923 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872188+0000", 2024-09-13T14:59:31.923 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5050000000000004e-06 2024-09-13T14:59:31.923 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.924 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.924 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:31.924 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872192+0000", 2024-09-13T14:59:31.924 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6439999999999999e-06 2024-09-13T14:59:31.924 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.924 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:31.924 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.925 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.925 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.925 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.46040.0:127818 9.2 e321/318)", 2024-09-13T14:59:31.925 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:31.872199+0000", 2024-09-13T14:59:31.925 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 4294967295.9997549, 2024-09-13T14:59:31.927 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010695499999999999, 2024-09-13T14:59:31.927 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:31.927 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:31.927 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "queued for pg", 2024-09-13T14:59:31.927 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:31.927 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.927 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:31.927 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872199+0000", 2024-09-13T14:59:31.928 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.928 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.928 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.929 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:31.929 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872199+0000", 2024-09-13T14:59:31.929 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:31.929 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.929 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.929 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:31.929 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872199+0000", 2024-09-13T14:59:31.929 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3799999999999999e-07 2024-09-13T14:59:31.929 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.929 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.929 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:31.929 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872201+0000", 2024-09-13T14:59:31.930 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.668e-06 2024-09-13T14:59:31.930 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.930 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.930 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:31.930 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872219+0000", 2024-09-13T14:59:31.930 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7853e-05 2024-09-13T14:59:31.930 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:31.930 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:31.930 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:31.930 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:31.872224+0000", 2024-09-13T14:59:31.930 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9989999999999999e-06 2024-09-13T14:59:31.930 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.930 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:31.930 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.931 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:31.931 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2024-09-13T14:59:31.931 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 9 2024-09-13T14:59:31.931 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:59:31.932 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:31.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.966 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:31.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:32.067 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:32.110 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:32.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:32.211 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:32.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.239 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:32.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:32.340 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:32.385 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:32.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:32.487 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:32.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.531 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:32.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:32.632 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:32.674 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:32.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:32.706 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:59:32.775 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:32.809 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:32.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:32.910 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:32.955 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:32.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:32.998 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:59:32.998 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:59:33.030 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:59:33.030 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:59:33.057 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:33.066 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:59:33.066 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:59:33.067 INFO:teuthology.orchestra.run.smithi096.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T14:59:33.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.091 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:33.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:33.097 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:59:33.097 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:59:33.098 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:33.123 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:33.133 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:59:33.133 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:59:33.169 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:59:33.169 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:59:33.193 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:33.212 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:59:33.213 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:59:33.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.227 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:33.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:33.328 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:33.358 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:33.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:33.460 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:33.493 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:33.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:33.530 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:33 smithi096 ceph-mon[98901]: pgmap v1050: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 12 MiB/s wr, 723 op/s; 50240/342102 objects degraded (14.686%); 13 MiB/s, 207 objects/s recovering 2024-09-13T14:59:33.530 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:33 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3687252590' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:33.531 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:33 smithi096 ceph-mon[102928]: pgmap v1050: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 12 MiB/s wr, 723 op/s; 50240/342102 objects degraded (14.686%); 13 MiB/s, 207 objects/s recovering 2024-09-13T14:59:33.531 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:33 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3687252590' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:33.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:33.622 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:33.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:33.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:33 smithi204 ceph-mon[87071]: pgmap v1050: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 12 MiB/s wr, 723 op/s; 50240/342102 objects degraded (14.686%); 13 MiB/s, 207 objects/s recovering 2024-09-13T14:59:33.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:33 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3687252590' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:33.724 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:33.754 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:33.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:33.857 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:33.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.884 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:33.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:33.985 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:34.017 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:34.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:34.119 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:34.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.154 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:34.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:34.255 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:34.299 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:34.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:34.401 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:34.423 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:34.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:34.525 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:34.553 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:34.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:34.655 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:34.680 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:34.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:34.718 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:59:34.718 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:59:34.782 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:34.824 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:34.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:34.891 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 6] out_osds: [3] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2024-09-13T14:59:34.891 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:59:34.892 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-13T14:59:34.892 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:59:34.926 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:34.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.980 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:34.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:35.081 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:35.118 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:35.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:35.220 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:35.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.253 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:35.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:35.354 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:35.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.406 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:35.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:35.507 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:35.523 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:35 smithi096 ceph-mon[102928]: pgmap v1051: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.8 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 1.7 MiB/s wr, 802 op/s; 48967/333366 objects degraded (14.689%); 9.5 MiB/s, 152 objects/s recovering 2024-09-13T14:59:35.523 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:35 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 48967/333366 objects degraded (14.689%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:35.523 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:35 smithi096 ceph-mon[98901]: pgmap v1051: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.8 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 1.7 MiB/s wr, 802 op/s; 48967/333366 objects degraded (14.689%); 9.5 MiB/s, 152 objects/s recovering 2024-09-13T14:59:35.523 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:35 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 48967/333366 objects degraded (14.689%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:35.537 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:35.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:35.639 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:35.680 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:35.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:35.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:35 smithi204 ceph-mon[87071]: pgmap v1051: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.8 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 1.7 MiB/s wr, 802 op/s; 48967/333366 objects degraded (14.689%); 9.5 MiB/s, 152 objects/s recovering 2024-09-13T14:59:35.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:35 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 48967/333366 objects degraded (14.689%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:35.782 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:35.822 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:35.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:35.924 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:35.962 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:35.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:36.064 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:36.109 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:36.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:36.211 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:36.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.252 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:36.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:36.353 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:36.391 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:36.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:36.403 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:36.439 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T14:59:36.439 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:59:36.493 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:36.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.538 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:36.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:36.639 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:36.669 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:36.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:36.720 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:59:36.771 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:36.803 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:36.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:36.905 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:36.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.937 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:36.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:37.038 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:37.052 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:37.052 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:37.053 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:37.053 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:37.076 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:37.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:37.178 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:37.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.218 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:37.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:37.320 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:37.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.365 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:37.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:37.466 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:37.502 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:37.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:37.604 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:37.639 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:37.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:37.741 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:37.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.779 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:37.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:37.881 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:37.919 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:37.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:38.021 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:38.053 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:38.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:38.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:37 smithi096 ceph-mon[98901]: pgmap v1052: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 48726/330345 objects degraded (14.750%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:59:38.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:59:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:59:37] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.51.0" 2024-09-13T14:59:38.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:37 smithi096 ceph-mon[102928]: pgmap v1052: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 48726/330345 objects degraded (14.750%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:59:38.156 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:38.190 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:38.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:37 smithi204 ceph-mon[87071]: pgmap v1052: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 48726/330345 objects degraded (14.750%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:59:38.292 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:38.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.329 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:38.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:38.420 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:59:38.430 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:38.488 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:38.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:38.590 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:38.620 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:38.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:38.722 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:38.754 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:38.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:38.856 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:38.909 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:38.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:38.960 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:39.015 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:39.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.057 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:39.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:39.158 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:39.200 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:39.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:39.302 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:39.337 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:39.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:39.439 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:39.466 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:39.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:39.555 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:39.567 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:39.606 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:39.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:39.708 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:39.741 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:39.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:39.842 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:39.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.880 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:39.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:39.881 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T14:59:39.981 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:40.018 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:40.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:40.120 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:40.156 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:40.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:40.198 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:39 smithi096 ceph-mon[98901]: pgmap v1053: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 47665/323244 objects degraded (14.746%); 12 MiB/s, 185 objects/s recovering 2024-09-13T14:59:40.198 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:39 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/845144439' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:59:40.198 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:39 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 47665/323244 objects degraded (14.746%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:40.198 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:39 smithi096 ceph-mon[102928]: pgmap v1053: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 47665/323244 objects degraded (14.746%); 12 MiB/s, 185 objects/s recovering 2024-09-13T14:59:40.198 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:39 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/845144439' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:59:40.198 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:39 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 47665/323244 objects degraded (14.746%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:40.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:39 smithi204 ceph-mon[87071]: pgmap v1053: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 47665/323244 objects degraded (14.746%); 12 MiB/s, 185 objects/s recovering 2024-09-13T14:59:40.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:39 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/845144439' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:59:40.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:39 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 47665/323244 objects degraded (14.746%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:40.258 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:40.288 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:40.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:40.390 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:40.426 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:40.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:59:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:59:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:40.528 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:40.571 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:40.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:40.673 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:40.704 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:40.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:40.806 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:40.838 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:40.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:40.940 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:40.971 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:40.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:41.073 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:41.106 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:40 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:41.106 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:40 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/845144439' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:59:41.106 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:40 smithi096 ceph-mon[98901]: osdmap e322: 8 total, 7 up, 7 in 2024-09-13T14:59:41.108 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:41.108 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:40 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:41.108 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:40 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/845144439' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:59:41.109 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:40 smithi096 ceph-mon[102928]: osdmap e322: 8 total, 7 up, 7 in 2024-09-13T14:59:41.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:41.210 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:41.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:40 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:41.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:40 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/845144439' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:59:41.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:40 smithi204 ceph-mon[87071]: osdmap e322: 8 total, 7 up, 7 in 2024-09-13T14:59:41.257 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:41.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:41.359 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:41.390 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:41.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:41.492 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:41.528 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T14:59:41.529 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T14:59:41.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.533 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:41.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:41.634 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:41.675 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:41.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:41.776 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:41.813 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:41.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:41.915 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:41.953 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:41.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:42.055 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:42.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:41 smithi096 ceph-mon[98901]: pgmap v1055: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 47185/318021 objects degraded (14.837%); 9.2 MiB/s, 147 objects/s recovering 2024-09-13T14:59:42.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:41 smithi096 ceph-mon[98901]: from='client.46166 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:59:42.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:41 smithi096 ceph-mon[102928]: pgmap v1055: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 47185/318021 objects degraded (14.837%); 9.2 MiB/s, 147 objects/s recovering 2024-09-13T14:59:42.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:41 smithi096 ceph-mon[102928]: from='client.46166 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:59:42.091 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:42.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:42.193 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:42.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:41 smithi204 ceph-mon[87071]: pgmap v1055: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 47185/318021 objects degraded (14.837%); 9.2 MiB/s, 147 objects/s recovering 2024-09-13T14:59:42.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:41 smithi204 ceph-mon[87071]: from='client.46166 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:59:42.222 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:42.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:42.324 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:42.350 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:42.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:42.452 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:42.484 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:42.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:42.586 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:42.612 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:42.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:42.714 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:42.742 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:42.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:42.844 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:42.873 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:42.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:42.976 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:43.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.019 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:43.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:43.120 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:43.155 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:43.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:43.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:42 smithi204 ceph-mon[87071]: pgmap v1056: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 826 op/s; 47073/317313 objects degraded (14.835%); 9.3 MiB/s, 148 objects/s recovering 2024-09-13T14:59:43.256 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:43.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.293 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:43.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:43.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:42 smithi096 ceph-mon[98901]: pgmap v1056: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 826 op/s; 47073/317313 objects degraded (14.835%); 9.3 MiB/s, 148 objects/s recovering 2024-09-13T14:59:43.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:42 smithi096 ceph-mon[102928]: pgmap v1056: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 826 op/s; 47073/317313 objects degraded (14.835%); 9.3 MiB/s, 148 objects/s recovering 2024-09-13T14:59:43.394 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:43.428 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:43.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:43.443 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":1055,"stamp":"2024-09-13T14:59:40.096735+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6950182343,"num_objects":106007,"num_object_clones":0,"num_object_copies":318021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47185,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":106007,"num_whiteouts":0,"num_read":6008,"num_read_kb":13034,"num_write":133744,"num_write_kb":7654593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6181,"num_bytes_recovered":406554452,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104927,"ondisk_log_size":104927,"up":327,"acting":319,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":22869420,"kb_used_data":17609716,"kb_used_omap":29,"kb_used_meta":5259490,"kb_avail":633346644,"statfs":{"total":671965249536,"available":648546963456,"internally_reserved":0,"allocated":18032349184,"data_stored":18016927482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30654,"internal_metadata":5385717826},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-542441472,"num_objects":-8277,"num_object_clones":0,"num_object_copies":-24831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3158,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8277,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1472,"num_bytes_recovered":96468992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002801"},"pg_stats":[{"pgid":"1.11","version":"82'100","reported_seq":240,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.233685+0000","last_change":"2024-09-13T14:59:00.929470+0000","last_active":"2024-09-13T14:59:24.233685+0000","last_peered":"2024-09-13T14:59:24.233685+0000","last_clean":"2024-09-13T14:59:24.233685+0000","last_became_active":"2024-09-13T14:59:00.928726+0000","last_became_peered":"2024-09-13T14:59:00.928726+0000","last_unstale":"2024-09-13T14:59:24.233685+0000","last_undegraded":"2024-09-13T14:59:24.233685+0000","last_fullsized":"2024-09-13T14:59:24.233685+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.087994+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.087994+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:58:12.866190+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":86,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.234015+0000","last_change":"2024-09-13T14:59:00.935629+0000","last_active":"2024-09-13T14:59:24.234015+0000","last_peered":"2024-09-13T14:59:24.234015+0000","last_clean":"2024-09-13T14:59:24.234015+0000","last_became_active":"2024-09-13T14:59:00.935106+0000","last_became_peered":"2024-09-13T14:59:00.935106+0000","last_unstale":"2024-09-13T14:59:24.234015+0000","last_undegraded":"2024-09-13T14:59:24.234015+0000","last_fullsized":"2024-09-13T14:59:24.234015+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:12.481049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":287,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.908055+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T14:59:14.908055+0000","last_peered":"2024-09-13T14:59:14.908055+0000","last_clean":"2024-09-13T14:59:14.908055+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T14:59:14.908055+0000","last_undegraded":"2024-09-13T14:59:14.908055+0000","last_fullsized":"2024-09-13T14:59:14.908055+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":51,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.911154+0000","last_change":"2024-09-13T14:56:04.643672+0000","last_active":"2024-09-13T14:59:14.911154+0000","last_peered":"2024-09-13T14:59:14.911154+0000","last_clean":"2024-09-13T14:59:14.911154+0000","last_became_active":"2024-09-13T14:55:55.185995+0000","last_became_peered":"2024-09-13T14:55:55.185995+0000","last_unstale":"2024-09-13T14:59:14.911154+0000","last_undegraded":"2024-09-13T14:59:14.911154+0000","last_fullsized":"2024-09-13T14:59:14.911154+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.643614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.643614+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:36.495546+0000","scrub_duration":0.001441681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":294,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.912445+0000","last_change":"2024-09-13T14:59:00.926231+0000","last_active":"2024-09-13T14:59:14.912445+0000","last_peered":"2024-09-13T14:59:14.912445+0000","last_clean":"2024-09-13T14:59:14.912445+0000","last_became_active":"2024-09-13T14:59:00.925808+0000","last_became_peered":"2024-09-13T14:59:00.925808+0000","last_unstale":"2024-09-13T14:59:14.912445+0000","last_undegraded":"2024-09-13T14:59:14.912445+0000","last_fullsized":"2024-09-13T14:59:14.912445+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:39.672143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":535,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.235918+0000","last_change":"2024-09-13T14:59:00.923848+0000","last_active":"2024-09-13T14:59:24.235918+0000","last_peered":"2024-09-13T14:59:24.235918+0000","last_clean":"2024-09-13T14:59:24.235918+0000","last_became_active":"2024-09-13T14:59:00.923522+0000","last_became_peered":"2024-09-13T14:59:00.923522+0000","last_unstale":"2024-09-13T14:59:24.235918+0000","last_undegraded":"2024-09-13T14:59:24.235918+0000","last_fullsized":"2024-09-13T14:59:24.235918+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:35:35.267527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":671,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.230802+0000","last_change":"2024-09-13T14:59:00.930294+0000","last_active":"2024-09-13T14:59:24.230802+0000","last_peered":"2024-09-13T14:59:24.230802+0000","last_clean":"2024-09-13T14:59:24.230802+0000","last_became_active":"2024-09-13T14:59:00.929860+0000","last_became_peered":"2024-09-13T14:59:00.929860+0000","last_unstale":"2024-09-13T14:59:24.230802+0000","last_undegraded":"2024-09-13T14:59:24.230802+0000","last_fullsized":"2024-09-13T14:59:24.230802+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.240882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.240882+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:35.209670+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":528,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.236862+0000","last_change":"2024-09-13T14:59:00.925448+0000","last_active":"2024-09-13T14:59:24.236862+0000","last_peered":"2024-09-13T14:59:24.236862+0000","last_clean":"2024-09-13T14:59:24.236862+0000","last_became_active":"2024-09-13T14:59:00.925152+0000","last_became_peered":"2024-09-13T14:59:00.925152+0000","last_unstale":"2024-09-13T14:59:24.236862+0000","last_undegraded":"2024-09-13T14:59:24.236862+0000","last_fullsized":"2024-09-13T14:59:24.236862+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:06.703642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":288,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.908861+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T14:59:14.908861+0000","last_peered":"2024-09-13T14:59:14.908861+0000","last_clean":"2024-09-13T14:59:14.908861+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T14:59:14.908861+0000","last_undegraded":"2024-09-13T14:59:14.908861+0000","last_fullsized":"2024-09-13T14:59:14.908861+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":51,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.911665+0000","last_change":"2024-09-13T14:56:03.605447+0000","last_active":"2024-09-13T14:59:14.911665+0000","last_peered":"2024-09-13T14:59:14.911665+0000","last_clean":"2024-09-13T14:59:14.911665+0000","last_became_active":"2024-09-13T14:55:55.218939+0000","last_became_peered":"2024-09-13T14:55:55.218939+0000","last_unstale":"2024-09-13T14:59:14.911665+0000","last_undegraded":"2024-09-13T14:59:14.911665+0000","last_fullsized":"2024-09-13T14:59:14.911665+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.605356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.605356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:12.467371+0000","scrub_duration":0.0020269620000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":43,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.906107+0000","last_change":"2024-09-13T14:59:00.925287+0000","last_active":"2024-09-13T14:59:14.906107+0000","last_peered":"2024-09-13T14:59:14.906107+0000","last_clean":"2024-09-13T14:59:14.906107+0000","last_became_active":"2024-09-13T14:59:00.923868+0000","last_became_peered":"2024-09-13T14:59:00.923868+0000","last_unstale":"2024-09-13T14:59:14.906107+0000","last_undegraded":"2024-09-13T14:59:14.906107+0000","last_fullsized":"2024-09-13T14:59:14.906107+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:28.106847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":115,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.907946+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T14:59:14.907946+0000","last_peered":"2024-09-13T14:59:14.907946+0000","last_clean":"2024-09-13T14:59:14.907946+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T14:59:14.907946+0000","last_undegraded":"2024-09-13T14:59:14.907946+0000","last_fullsized":"2024-09-13T14:59:14.907946+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"321'8441","reported_seq":8491,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:39.182408+0000","last_change":"2024-09-13T14:57:34.564761+0000","last_active":"2024-09-13T14:59:39.182408+0000","last_peered":"2024-09-13T14:59:39.182408+0000","last_clean":"2024-09-13T14:59:39.182408+0000","last_became_active":"2024-09-13T14:57:34.564455+0000","last_became_peered":"2024-09-13T14:57:34.564455+0000","last_unstale":"2024-09-13T14:59:39.182408+0000","last_undegraded":"2024-09-13T14:59:39.182408+0000","last_fullsized":"2024-09-13T14:59:39.182408+0000","mapping_epoch":302,"log_start":"307'1600","ondisk_log_start":"307'1600","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6841,"log_dups_size":0,"ondisk_log_size":6841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:06.596934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435879936,"num_objects":6651,"num_object_clones":0,"num_object_copies":19953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8441,"num_write_kb":482944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":51,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.908769+0000","last_change":"2024-09-13T14:56:01.654803+0000","last_active":"2024-09-13T14:59:14.908769+0000","last_peered":"2024-09-13T14:59:14.908769+0000","last_clean":"2024-09-13T14:59:14.908769+0000","last_became_active":"2024-09-13T14:55:55.188788+0000","last_became_peered":"2024-09-13T14:55:55.188788+0000","last_unstale":"2024-09-13T14:59:14.908769+0000","last_undegraded":"2024-09-13T14:59:14.908769+0000","last_fullsized":"2024-09-13T14:59:14.908769+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.654717+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.654717+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:38.649187+0000","scrub_duration":0.0060902070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":498,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.237168+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T14:59:24.237168+0000","last_peered":"2024-09-13T14:59:24.237168+0000","last_clean":"2024-09-13T14:59:24.237168+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T14:59:24.237168+0000","last_undegraded":"2024-09-13T14:59:24.237168+0000","last_fullsized":"2024-09-13T14:59:24.237168+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":307,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.911378+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T14:59:14.911378+0000","last_peered":"2024-09-13T14:59:14.911378+0000","last_clean":"2024-09-13T14:59:14.911378+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T14:59:14.911378+0000","last_undegraded":"2024-09-13T14:59:14.911378+0000","last_fullsized":"2024-09-13T14:59:14.911378+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"321'8031","reported_seq":8085,"reported_epoch":321,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:59:38.654776+0000","last_change":"2024-09-13T14:59:02.220708+0000","last_active":"2024-09-13T14:59:38.654776+0000","last_peered":"2024-09-13T14:59:38.654776+0000","last_clean":"2024-09-13T14:58:59.891928+0000","last_became_active":"2024-09-13T14:59:02.208127+0000","last_became_peered":"2024-09-13T14:59:02.208127+0000","last_unstale":"2024-09-13T14:59:38.654776+0000","last_undegraded":"2024-09-13T14:59:02.073241+0000","last_fullsized":"2024-09-13T14:59:01.920942+0000","mapping_epoch":316,"log_start":"308'2448","ondisk_log_start":"308'2448","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5583,"log_dups_size":0,"ondisk_log_size":5583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421068800,"num_objects":6425,"num_object_clones":0,"num_object_copies":19275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6402,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8031,"num_write_kb":462592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":23}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":431,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.237694+0000","last_change":"2024-09-13T14:59:17.222646+0000","last_active":"2024-09-13T14:59:24.237694+0000","last_peered":"2024-09-13T14:59:24.237694+0000","last_clean":"2024-09-13T14:59:24.237694+0000","last_became_active":"2024-09-13T14:59:00.927379+0000","last_became_peered":"2024-09-13T14:59:00.927379+0000","last_unstale":"2024-09-13T14:59:24.237694+0000","last_undegraded":"2024-09-13T14:59:24.237694+0000","last_fullsized":"2024-09-13T14:59:24.237694+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:59:17.222405+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:59:17.222405+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.099900+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":287,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.907652+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T14:59:14.907652+0000","last_peered":"2024-09-13T14:59:14.907652+0000","last_clean":"2024-09-13T14:59:14.907652+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T14:59:14.907652+0000","last_undegraded":"2024-09-13T14:59:14.907652+0000","last_fullsized":"2024-09-13T14:59:14.907652+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":352,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.911781+0000","last_change":"2024-09-13T14:59:00.929566+0000","last_active":"2024-09-13T14:59:14.911781+0000","last_peered":"2024-09-13T14:59:14.911781+0000","last_clean":"2024-09-13T14:59:14.911781+0000","last_became_active":"2024-09-13T14:59:00.929271+0000","last_became_peered":"2024-09-13T14:59:00.929271+0000","last_unstale":"2024-09-13T14:59:14.911781+0000","last_undegraded":"2024-09-13T14:59:14.911781+0000","last_fullsized":"2024-09-13T14:59:14.911781+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:25.655536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"321'8239","reported_seq":8276,"reported_epoch":321,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:59:37.405100+0000","last_change":"2024-09-13T14:59:02.370697+0000","last_active":"2024-09-13T14:59:37.405100+0000","last_peered":"2024-09-13T14:59:37.405100+0000","last_clean":"2024-09-13T14:58:59.856641+0000","last_became_active":"2024-09-13T14:59:02.311296+0000","last_became_peered":"2024-09-13T14:59:02.311296+0000","last_unstale":"2024-09-13T14:59:37.405100+0000","last_undegraded":"2024-09-13T14:59:01.964581+0000","last_fullsized":"2024-09-13T14:59:01.916393+0000","mapping_epoch":316,"log_start":"308'1700","ondisk_log_start":"308'1700","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6539,"log_dups_size":0,"ondisk_log_size":6539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:14:42.957367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435486720,"num_objects":6645,"num_object_clones":0,"num_object_copies":19935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8239,"num_write_kb":476288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":67}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":154,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.909160+0000","last_change":"2024-09-13T14:56:08.889974+0000","last_active":"2024-09-13T14:59:14.909160+0000","last_peered":"2024-09-13T14:59:14.909160+0000","last_clean":"2024-09-13T14:59:14.909160+0000","last_became_active":"2024-09-13T14:55:55.185755+0000","last_became_peered":"2024-09-13T14:55:55.185755+0000","last_unstale":"2024-09-13T14:59:14.909160+0000","last_undegraded":"2024-09-13T14:59:14.909160+0000","last_fullsized":"2024-09-13T14:59:14.909160+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:08.889814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:08.889814+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:02.911564+0000","scrub_duration":0.0053731569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":297,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.909266+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T14:59:14.909266+0000","last_peered":"2024-09-13T14:59:14.909266+0000","last_clean":"2024-09-13T14:59:14.909266+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T14:59:14.909266+0000","last_undegraded":"2024-09-13T14:59:14.909266+0000","last_fullsized":"2024-09-13T14:59:14.909266+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":115,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.907251+0000","last_change":"2024-09-13T14:51:44.190879+0000","last_active":"2024-09-13T14:59:14.907251+0000","last_peered":"2024-09-13T14:59:14.907251+0000","last_clean":"2024-09-13T14:59:14.907251+0000","last_became_active":"2024-09-13T14:51:44.190617+0000","last_became_peered":"2024-09-13T14:51:44.190617+0000","last_unstale":"2024-09-13T14:59:14.907251+0000","last_undegraded":"2024-09-13T14:59:14.907251+0000","last_fullsized":"2024-09-13T14:59:14.907251+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:19.536298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"321'8460","reported_seq":8487,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:37.375598+0000","last_change":"2024-09-13T14:57:34.556168+0000","last_active":"2024-09-13T14:59:37.375598+0000","last_peered":"2024-09-13T14:59:37.375598+0000","last_clean":"2024-09-13T14:59:37.375598+0000","last_became_active":"2024-09-13T14:57:34.555882+0000","last_became_peered":"2024-09-13T14:57:34.555882+0000","last_unstale":"2024-09-13T14:59:37.375598+0000","last_undegraded":"2024-09-13T14:59:37.375598+0000","last_fullsized":"2024-09-13T14:59:37.375598+0000","mapping_epoch":302,"log_start":"308'1912","ondisk_log_start":"308'1912","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6548,"log_dups_size":0,"ondisk_log_size":6548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:05:49.922717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":444071936,"num_objects":6776,"num_object_clones":0,"num_object_copies":20328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8460,"num_write_kb":487552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":61,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.913559+0000","last_change":"2024-09-13T14:59:00.925894+0000","last_active":"2024-09-13T14:59:14.913559+0000","last_peered":"2024-09-13T14:59:14.913559+0000","last_clean":"2024-09-13T14:59:14.913559+0000","last_became_active":"2024-09-13T14:59:00.924919+0000","last_became_peered":"2024-09-13T14:59:00.924919+0000","last_unstale":"2024-09-13T14:59:14.913559+0000","last_undegraded":"2024-09-13T14:59:14.913559+0000","last_fullsized":"2024-09-13T14:59:14.913559+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:02.325629+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:02.325629+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:18.872411+0000","scrub_duration":0.0049176200000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":315,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.912217+0000","last_change":"2024-09-13T14:59:00.930126+0000","last_active":"2024-09-13T14:59:14.912217+0000","last_peered":"2024-09-13T14:59:14.912217+0000","last_clean":"2024-09-13T14:59:14.912217+0000","last_became_active":"2024-09-13T14:59:00.929803+0000","last_became_peered":"2024-09-13T14:59:00.929803+0000","last_unstale":"2024-09-13T14:59:14.912217+0000","last_undegraded":"2024-09-13T14:59:14.912217+0000","last_fullsized":"2024-09-13T14:59:14.912217+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:44.456770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":508,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.237567+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T14:59:24.237567+0000","last_peered":"2024-09-13T14:59:24.237567+0000","last_clean":"2024-09-13T14:59:24.237567+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T14:59:24.237567+0000","last_undegraded":"2024-09-13T14:59:24.237567+0000","last_fullsized":"2024-09-13T14:59:24.237567+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"321'8182","reported_seq":9797,"reported_epoch":321,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:59:36.968556+0000","last_change":"2024-09-13T14:59:11.915459+0000","last_active":"2024-09-13T14:59:36.968556+0000","last_peered":"2024-09-13T14:59:36.968556+0000","last_clean":"2024-09-13T14:58:59.874928+0000","last_became_active":"2024-09-13T14:59:02.174454+0000","last_became_peered":"2024-09-13T14:59:02.174454+0000","last_unstale":"2024-09-13T14:59:36.968556+0000","last_undegraded":"2024-09-13T14:59:11.911350+0000","last_fullsized":"2024-09-13T14:59:01.917676+0000","mapping_epoch":316,"log_start":"308'1849","ondisk_log_start":"308'1849","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6333,"log_dups_size":0,"ondisk_log_size":6333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:39.210607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435421184,"num_objects":6644,"num_object_clones":0,"num_object_copies":19932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5817,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8182,"num_write_kb":474432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":782,"num_bytes_recovered":51249152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"6,7","objects":827}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":478,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.908600+0000","last_change":"2024-09-13T14:59:00.932316+0000","last_active":"2024-09-13T14:59:14.908600+0000","last_peered":"2024-09-13T14:59:14.908600+0000","last_clean":"2024-09-13T14:59:14.908600+0000","last_became_active":"2024-09-13T14:59:00.931497+0000","last_became_peered":"2024-09-13T14:59:00.931497+0000","last_unstale":"2024-09-13T14:59:14.908600+0000","last_undegraded":"2024-09-13T14:59:14.908600+0000","last_fullsized":"2024-09-13T14:59:14.908600+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:35.403609+0000","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":48,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.237247+0000","last_change":"2024-09-13T14:59:00.936078+0000","last_active":"2024-09-13T14:59:24.237247+0000","last_peered":"2024-09-13T14:59:24.237247+0000","last_clean":"2024-09-13T14:59:24.237247+0000","last_became_active":"2024-09-13T14:59:00.925997+0000","last_became_peered":"2024-09-13T14:59:00.925997+0000","last_unstale":"2024-09-13T14:59:24.237247+0000","last_undegraded":"2024-09-13T14:59:24.237247+0000","last_fullsized":"2024-09-13T14:59:24.237247+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:43.716136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":315,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.912595+0000","last_change":"2024-09-13T14:59:00.934706+0000","last_active":"2024-09-13T14:59:14.912595+0000","last_peered":"2024-09-13T14:59:14.912595+0000","last_clean":"2024-09-13T14:59:14.912595+0000","last_became_active":"2024-09-13T14:59:00.934367+0000","last_became_peered":"2024-09-13T14:59:00.934367+0000","last_unstale":"2024-09-13T14:59:14.912595+0000","last_undegraded":"2024-09-13T14:59:14.912595+0000","last_fullsized":"2024-09-13T14:59:14.912595+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:37:52.860543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"321'8524","reported_seq":8574,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:38.701098+0000","last_change":"2024-09-13T14:57:34.835630+0000","last_active":"2024-09-13T14:59:38.701098+0000","last_peered":"2024-09-13T14:59:38.701098+0000","last_clean":"2024-09-13T14:59:38.701098+0000","last_became_active":"2024-09-13T14:57:34.835209+0000","last_became_peered":"2024-09-13T14:57:34.835209+0000","last_unstale":"2024-09-13T14:59:38.701098+0000","last_undegraded":"2024-09-13T14:59:38.701098+0000","last_fullsized":"2024-09-13T14:59:38.701098+0000","mapping_epoch":302,"log_start":"308'2922","ondisk_log_start":"308'2922","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5602,"log_dups_size":0,"ondisk_log_size":5602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:50.159454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436338688,"num_objects":6658,"num_object_clones":0,"num_object_copies":19974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8524,"num_write_kb":485824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":242,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.911222+0000","last_change":"2024-09-13T14:59:00.934514+0000","last_active":"2024-09-13T14:59:14.911222+0000","last_peered":"2024-09-13T14:59:14.911222+0000","last_clean":"2024-09-13T14:59:14.911222+0000","last_became_active":"2024-09-13T14:59:00.928674+0000","last_became_peered":"2024-09-13T14:59:00.928674+0000","last_unstale":"2024-09-13T14:59:14.911222+0000","last_undegraded":"2024-09-13T14:59:14.911222+0000","last_fullsized":"2024-09-13T14:59:14.911222+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:58:13.987558+0000","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":105,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.907354+0000","last_change":"2024-09-13T14:48:32.200521+0000","last_active":"2024-09-13T14:59:14.907354+0000","last_peered":"2024-09-13T14:59:14.907354+0000","last_clean":"2024-09-13T14:59:14.907354+0000","last_became_active":"2024-09-13T14:48:32.200256+0000","last_became_peered":"2024-09-13T14:48:32.200256+0000","last_unstale":"2024-09-13T14:59:14.907354+0000","last_undegraded":"2024-09-13T14:59:14.907354+0000","last_fullsized":"2024-09-13T14:59:14.907354+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:38.899629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":302,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.912417+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T14:59:14.912417+0000","last_peered":"2024-09-13T14:59:14.912417+0000","last_clean":"2024-09-13T14:59:14.912417+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T14:59:14.912417+0000","last_undegraded":"2024-09-13T14:59:14.912417+0000","last_fullsized":"2024-09-13T14:59:14.912417+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"321'8280","reported_seq":18824,"reported_epoch":321,"state":"active+undersized+degraded+remapped+backfilling","last_fresh":"2024-09-13T14:59:37.144749+0000","last_change":"2024-09-13T14:59:12.902518+0000","last_active":"2024-09-13T14:59:37.144749+0000","last_peered":"2024-09-13T14:59:37.144749+0000","last_clean":"2024-09-13T14:58:59.853307+0000","last_became_active":"2024-09-13T14:59:12.882540+0000","last_became_peered":"2024-09-13T14:59:12.882540+0000","last_unstale":"2024-09-13T14:59:37.144749+0000","last_undegraded":"2024-09-13T14:59:12.878960+0000","last_fullsized":"2024-09-13T14:59:12.878769+0000","mapping_epoch":318,"log_start":"308'2214","ondisk_log_start":"308'2214","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6066,"log_dups_size":0,"ondisk_log_size":6066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:07:52.521219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436994048,"num_objects":6668,"num_object_clones":0,"num_object_copies":20004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1723,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8280,"num_write_kb":478336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5248,"num_bytes_recovered":343932928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":6668}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":664,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.230998+0000","last_change":"2024-09-13T14:56:01.254863+0000","last_active":"2024-09-13T14:59:24.230998+0000","last_peered":"2024-09-13T14:59:24.230998+0000","last_clean":"2024-09-13T14:59:24.230998+0000","last_became_active":"2024-09-13T14:55:55.214258+0000","last_became_peered":"2024-09-13T14:55:55.214258+0000","last_unstale":"2024-09-13T14:59:24.230998+0000","last_undegraded":"2024-09-13T14:59:24.230998+0000","last_fullsized":"2024-09-13T14:59:24.230998+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.254778+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.254778+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:26.654454+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":494,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.233493+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T14:59:24.233493+0000","last_peered":"2024-09-13T14:59:24.233493+0000","last_clean":"2024-09-13T14:59:24.233493+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T14:59:24.233493+0000","last_undegraded":"2024-09-13T14:59:24.233493+0000","last_fullsized":"2024-09-13T14:59:24.233493+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":305,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.913698+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T14:59:14.913698+0000","last_peered":"2024-09-13T14:59:14.913698+0000","last_clean":"2024-09-13T14:59:14.913698+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T14:59:14.913698+0000","last_undegraded":"2024-09-13T14:59:14.913698+0000","last_fullsized":"2024-09-13T14:59:14.913698+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"321'8333","reported_seq":8390,"reported_epoch":321,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-09-13T14:59:38.685905+0000","last_change":"2024-09-13T14:59:12.888109+0000","last_active":"2024-09-13T14:59:38.685905+0000","last_peered":"2024-09-13T14:59:38.685905+0000","last_clean":"2024-09-13T14:58:59.863442+0000","last_became_active":"2024-09-13T14:59:12.882847+0000","last_became_peered":"2024-09-13T14:59:12.882847+0000","last_unstale":"2024-09-13T14:59:38.685905+0000","last_undegraded":"2024-09-13T14:59:12.878900+0000","last_fullsized":"2024-09-13T14:59:12.878712+0000","mapping_epoch":318,"log_start":"308'2684","ondisk_log_start":"308'2684","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5649,"log_dups_size":0,"ondisk_log_size":5649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435224576,"num_objects":6641,"num_object_clones":0,"num_object_copies":19923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8333,"num_write_kb":479168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":6641}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":231,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.911536+0000","last_change":"2024-09-13T14:48:32.205288+0000","last_active":"2024-09-13T14:59:14.911536+0000","last_peered":"2024-09-13T14:59:14.911536+0000","last_clean":"2024-09-13T14:59:14.911536+0000","last_became_active":"2024-09-13T14:48:32.204657+0000","last_became_peered":"2024-09-13T14:48:32.204657+0000","last_unstale":"2024-09-13T14:59:14.911536+0000","last_undegraded":"2024-09-13T14:59:14.911536+0000","last_fullsized":"2024-09-13T14:59:14.911536+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:52.513329+0000","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":249,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.230992+0000","last_change":"2024-09-13T14:59:00.931433+0000","last_active":"2024-09-13T14:59:24.230992+0000","last_peered":"2024-09-13T14:59:24.230992+0000","last_clean":"2024-09-13T14:59:24.230992+0000","last_became_active":"2024-09-13T14:59:00.928906+0000","last_became_peered":"2024-09-13T14:59:00.928906+0000","last_unstale":"2024-09-13T14:59:24.230992+0000","last_undegraded":"2024-09-13T14:59:24.230992+0000","last_fullsized":"2024-09-13T14:59:24.230992+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:12.786568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":503,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.231593+0000","last_change":"2024-09-13T14:59:00.925277+0000","last_active":"2024-09-13T14:59:24.231593+0000","last_peered":"2024-09-13T14:59:24.231593+0000","last_clean":"2024-09-13T14:59:24.231593+0000","last_became_active":"2024-09-13T14:59:00.924952+0000","last_became_peered":"2024-09-13T14:59:00.924952+0000","last_unstale":"2024-09-13T14:59:24.231593+0000","last_undegraded":"2024-09-13T14:59:24.231593+0000","last_fullsized":"2024-09-13T14:59:24.231593+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:08.888187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"321'8382","reported_seq":8524,"reported_epoch":321,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-09-13T14:59:39.127363+0000","last_change":"2024-09-13T14:59:12.893143+0000","last_active":"2024-09-13T14:59:39.127363+0000","last_peered":"2024-09-13T14:59:39.127363+0000","last_clean":"2024-09-13T14:58:59.874256+0000","last_became_active":"2024-09-13T14:59:12.882701+0000","last_became_peered":"2024-09-13T14:59:12.882701+0000","last_unstale":"2024-09-13T14:59:39.127363+0000","last_undegraded":"2024-09-13T14:59:12.880389+0000","last_fullsized":"2024-09-13T14:59:12.880226+0000","mapping_epoch":318,"log_start":"307'1500","ondisk_log_start":"307'1500","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6882,"log_dups_size":0,"ondisk_log_size":6882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437518336,"num_objects":6676,"num_object_clones":0,"num_object_copies":20028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6636,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8382,"num_write_kb":481856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":2621440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":6676}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":469,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.905612+0000","last_change":"2024-09-13T14:55:58.568852+0000","last_active":"2024-09-13T14:59:14.905612+0000","last_peered":"2024-09-13T14:59:14.905612+0000","last_clean":"2024-09-13T14:59:14.905612+0000","last_became_active":"2024-09-13T14:55:51.169987+0000","last_became_peered":"2024-09-13T14:55:51.169987+0000","last_unstale":"2024-09-13T14:59:14.905612+0000","last_undegraded":"2024-09-13T14:59:14.905612+0000","last_fullsized":"2024-09-13T14:59:14.905612+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.568750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.568750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:56.273429+0000","scrub_duration":0.0074886340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":847,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.912073+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T14:59:14.912073+0000","last_peered":"2024-09-13T14:59:14.912073+0000","last_clean":"2024-09-13T14:59:14.912073+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T14:59:14.912073+0000","last_undegraded":"2024-09-13T14:59:14.912073+0000","last_fullsized":"2024-09-13T14:59:14.912073+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":141,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.911601+0000","last_change":"2024-09-13T14:59:00.930790+0000","last_active":"2024-09-13T14:59:14.911601+0000","last_peered":"2024-09-13T14:59:14.911601+0000","last_clean":"2024-09-13T14:59:14.911601+0000","last_became_active":"2024-09-13T14:59:00.930439+0000","last_became_peered":"2024-09-13T14:59:00.930439+0000","last_unstale":"2024-09-13T14:59:14.911601+0000","last_undegraded":"2024-09-13T14:59:14.911601+0000","last_fullsized":"2024-09-13T14:59:14.911601+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:15.636095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":105,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.908643+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T14:59:14.908643+0000","last_peered":"2024-09-13T14:59:14.908643+0000","last_clean":"2024-09-13T14:59:14.908643+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T14:59:14.908643+0000","last_undegraded":"2024-09-13T14:59:14.908643+0000","last_fullsized":"2024-09-13T14:59:14.908643+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"321'8278","reported_seq":8327,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:39.078043+0000","last_change":"2024-09-13T14:57:34.836468+0000","last_active":"2024-09-13T14:59:39.078043+0000","last_peered":"2024-09-13T14:59:39.078043+0000","last_clean":"2024-09-13T14:59:39.078043+0000","last_became_active":"2024-09-13T14:57:34.836144+0000","last_became_peered":"2024-09-13T14:57:34.836144+0000","last_unstale":"2024-09-13T14:59:39.078043+0000","last_undegraded":"2024-09-13T14:59:39.078043+0000","last_fullsized":"2024-09-13T14:59:39.078043+0000","mapping_epoch":302,"log_start":"307'1400","ondisk_log_start":"307'1400","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6878,"log_dups_size":0,"ondisk_log_size":6878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:40:48.019738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427687936,"num_objects":6526,"num_object_clones":0,"num_object_copies":19578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8278,"num_write_kb":473728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":659,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.230664+0000","last_change":"2024-09-13T14:55:59.245841+0000","last_active":"2024-09-13T14:59:24.230664+0000","last_peered":"2024-09-13T14:59:24.230664+0000","last_clean":"2024-09-13T14:59:24.230664+0000","last_became_active":"2024-09-13T14:55:51.159487+0000","last_became_peered":"2024-09-13T14:55:51.159487+0000","last_unstale":"2024-09-13T14:59:24.230664+0000","last_undegraded":"2024-09-13T14:59:24.230664+0000","last_fullsized":"2024-09-13T14:59:24.230664+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.245213+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.245213+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:58:06.592954+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":532,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.905475+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T14:59:14.905475+0000","last_peered":"2024-09-13T14:59:14.905475+0000","last_clean":"2024-09-13T14:59:14.905475+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T14:59:14.905475+0000","last_undegraded":"2024-09-13T14:59:14.905475+0000","last_fullsized":"2024-09-13T14:59:14.905475+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":537,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.233596+0000","last_change":"2024-09-13T14:59:00.928965+0000","last_active":"2024-09-13T14:59:24.233596+0000","last_peered":"2024-09-13T14:59:24.233596+0000","last_clean":"2024-09-13T14:59:24.233596+0000","last_became_active":"2024-09-13T14:59:00.928166+0000","last_became_peered":"2024-09-13T14:59:00.928166+0000","last_unstale":"2024-09-13T14:59:24.233596+0000","last_undegraded":"2024-09-13T14:59:24.233596+0000","last_fullsized":"2024-09-13T14:59:24.233596+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:34.878172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":534,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.237033+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T14:59:24.237033+0000","last_peered":"2024-09-13T14:59:24.237033+0000","last_clean":"2024-09-13T14:59:24.237033+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T14:59:24.237033+0000","last_undegraded":"2024-09-13T14:59:24.237033+0000","last_fullsized":"2024-09-13T14:59:24.237033+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"321'8296","reported_seq":8323,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:37.391282+0000","last_change":"2024-09-13T14:57:34.559226+0000","last_active":"2024-09-13T14:59:37.391282+0000","last_peered":"2024-09-13T14:59:37.391282+0000","last_clean":"2024-09-13T14:59:37.391282+0000","last_became_active":"2024-09-13T14:57:34.558860+0000","last_became_peered":"2024-09-13T14:57:34.558860+0000","last_unstale":"2024-09-13T14:59:37.391282+0000","last_undegraded":"2024-09-13T14:59:37.391282+0000","last_fullsized":"2024-09-13T14:59:37.391282+0000","mapping_epoch":302,"log_start":"307'1704","ondisk_log_start":"307'1704","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6592,"log_dups_size":0,"ondisk_log_size":6592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:56:32.571916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430309376,"num_objects":6566,"num_object_clones":0,"num_object_copies":19698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8296,"num_write_kb":475584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":98,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.905986+0000","last_change":"2024-09-13T14:55:59.574827+0000","last_active":"2024-09-13T14:59:14.905986+0000","last_peered":"2024-09-13T14:59:14.905986+0000","last_clean":"2024-09-13T14:59:14.905986+0000","last_became_active":"2024-09-13T14:55:51.162903+0000","last_became_peered":"2024-09-13T14:55:51.162903+0000","last_unstale":"2024-09-13T14:59:14.905986+0000","last_undegraded":"2024-09-13T14:59:14.905986+0000","last_fullsized":"2024-09-13T14:59:14.905986+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.574773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.574773+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:39.074072+0000","scrub_duration":0.001553708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":459,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.233857+0000","last_change":"2024-09-13T14:59:00.950349+0000","last_active":"2024-09-13T14:59:24.233857+0000","last_peered":"2024-09-13T14:59:24.233857+0000","last_clean":"2024-09-13T14:59:24.233857+0000","last_became_active":"2024-09-13T14:59:00.925703+0000","last_became_peered":"2024-09-13T14:59:00.925703+0000","last_unstale":"2024-09-13T14:59:24.233857+0000","last_undegraded":"2024-09-13T14:59:24.233857+0000","last_fullsized":"2024-09-13T14:59:24.233857+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:43.851359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":511,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.231075+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T14:59:24.231075+0000","last_peered":"2024-09-13T14:59:24.231075+0000","last_clean":"2024-09-13T14:59:24.231075+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T14:59:24.231075+0000","last_undegraded":"2024-09-13T14:59:24.231075+0000","last_fullsized":"2024-09-13T14:59:24.231075+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":529,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.231471+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T14:59:24.231471+0000","last_peered":"2024-09-13T14:59:24.231471+0000","last_clean":"2024-09-13T14:59:24.231471+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T14:59:24.231471+0000","last_undegraded":"2024-09-13T14:59:24.231471+0000","last_fullsized":"2024-09-13T14:59:24.231471+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"321'8252","reported_seq":8301,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:39.133508+0000","last_change":"2024-09-13T14:57:34.564910+0000","last_active":"2024-09-13T14:59:39.133508+0000","last_peered":"2024-09-13T14:59:39.133508+0000","last_clean":"2024-09-13T14:59:39.133508+0000","last_became_active":"2024-09-13T14:57:34.564633+0000","last_became_peered":"2024-09-13T14:57:34.564633+0000","last_unstale":"2024-09-13T14:59:39.133508+0000","last_undegraded":"2024-09-13T14:59:39.133508+0000","last_fullsized":"2024-09-13T14:59:39.133508+0000","mapping_epoch":302,"log_start":"307'1400","ondisk_log_start":"307'1400","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6852,"log_dups_size":0,"ondisk_log_size":6852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:53:38.983345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424411136,"num_objects":6476,"num_object_clones":0,"num_object_copies":19428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8252,"num_write_kb":471296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":241,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.914264+0000","last_change":"2024-09-13T14:56:01.658240+0000","last_active":"2024-09-13T14:59:14.914264+0000","last_peered":"2024-09-13T14:59:14.914264+0000","last_clean":"2024-09-13T14:59:14.914264+0000","last_became_active":"2024-09-13T14:55:51.160480+0000","last_became_peered":"2024-09-13T14:55:51.160480+0000","last_unstale":"2024-09-13T14:59:14.914264+0000","last_undegraded":"2024-09-13T14:59:14.914264+0000","last_fullsized":"2024-09-13T14:59:14.914264+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.658143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.658143+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:19.262955+0000","scrub_duration":0.0038903079999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":947,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.911772+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T14:59:14.911772+0000","last_peered":"2024-09-13T14:59:14.911772+0000","last_clean":"2024-09-13T14:59:14.911772+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T14:59:14.911772+0000","last_undegraded":"2024-09-13T14:59:14.911772+0000","last_fullsized":"2024-09-13T14:59:14.911772+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":306,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.913144+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T14:59:14.913144+0000","last_peered":"2024-09-13T14:59:14.913144+0000","last_clean":"2024-09-13T14:59:14.913144+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T14:59:14.913144+0000","last_undegraded":"2024-09-13T14:59:14.913144+0000","last_fullsized":"2024-09-13T14:59:14.913144+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":535,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.233743+0000","last_change":"2024-09-13T14:59:00.930713+0000","last_active":"2024-09-13T14:59:24.233743+0000","last_peered":"2024-09-13T14:59:24.233743+0000","last_clean":"2024-09-13T14:59:24.233743+0000","last_became_active":"2024-09-13T14:59:00.928260+0000","last_became_peered":"2024-09-13T14:59:00.928260+0000","last_unstale":"2024-09-13T14:59:24.233743+0000","last_undegraded":"2024-09-13T14:59:24.233743+0000","last_fullsized":"2024-09-13T14:59:24.233743+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:07.048853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"321'8230","reported_seq":8262,"reported_epoch":321,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:59:37.362574+0000","last_change":"2024-09-13T14:59:02.370702+0000","last_active":"2024-09-13T14:59:37.362574+0000","last_peered":"2024-09-13T14:59:37.362574+0000","last_clean":"2024-09-13T14:58:59.881570+0000","last_became_active":"2024-09-13T14:59:02.311551+0000","last_became_peered":"2024-09-13T14:59:02.311551+0000","last_unstale":"2024-09-13T14:59:37.362574+0000","last_undegraded":"2024-09-13T14:59:01.972778+0000","last_fullsized":"2024-09-13T14:59:01.916475+0000","mapping_epoch":316,"log_start":"308'1700","ondisk_log_start":"308'1700","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6530,"log_dups_size":0,"ondisk_log_size":6530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:01:25.255932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":431620096,"num_objects":6586,"num_object_clones":0,"num_object_copies":19758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6597,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8230,"num_write_kb":474112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":-11}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":155,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.909080+0000","last_change":"2024-09-13T14:59:00.931152+0000","last_active":"2024-09-13T14:59:14.909080+0000","last_peered":"2024-09-13T14:59:14.909080+0000","last_clean":"2024-09-13T14:59:14.909080+0000","last_became_active":"2024-09-13T14:59:00.930801+0000","last_became_peered":"2024-09-13T14:59:00.930801+0000","last_unstale":"2024-09-13T14:59:14.909080+0000","last_undegraded":"2024-09-13T14:59:14.909080+0000","last_fullsized":"2024-09-13T14:59:14.909080+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.916764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.916764+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:11:50.934406+0000","scrub_duration":0.0035298819999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":312,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.912587+0000","last_change":"2024-09-13T14:59:00.931326+0000","last_active":"2024-09-13T14:59:14.912587+0000","last_peered":"2024-09-13T14:59:14.912587+0000","last_clean":"2024-09-13T14:59:14.912587+0000","last_became_active":"2024-09-13T14:59:00.931011+0000","last_became_peered":"2024-09-13T14:59:00.931011+0000","last_unstale":"2024-09-13T14:59:14.912587+0000","last_undegraded":"2024-09-13T14:59:14.912587+0000","last_fullsized":"2024-09-13T14:59:14.912587+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:24:41.833263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":1013,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.233925+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T14:59:24.233925+0000","last_peered":"2024-09-13T14:59:24.233925+0000","last_clean":"2024-09-13T14:59:24.233925+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T14:59:24.233925+0000","last_undegraded":"2024-09-13T14:59:24.233925+0000","last_fullsized":"2024-09-13T14:59:24.233925+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":485,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.232170+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T14:59:24.232170+0000","last_peered":"2024-09-13T14:59:24.232170+0000","last_clean":"2024-09-13T14:59:24.232170+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T14:59:24.232170+0000","last_undegraded":"2024-09-13T14:59:24.232170+0000","last_fullsized":"2024-09-13T14:59:24.232170+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"321'8266","reported_seq":8293,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:37.111167+0000","last_change":"2024-09-13T14:57:34.841879+0000","last_active":"2024-09-13T14:59:37.111167+0000","last_peered":"2024-09-13T14:59:37.111167+0000","last_clean":"2024-09-13T14:59:37.111167+0000","last_became_active":"2024-09-13T14:57:34.841546+0000","last_became_peered":"2024-09-13T14:57:34.841546+0000","last_unstale":"2024-09-13T14:59:37.111167+0000","last_undegraded":"2024-09-13T14:59:37.111167+0000","last_fullsized":"2024-09-13T14:59:37.111167+0000","mapping_epoch":302,"log_start":"308'2181","ondisk_log_start":"308'2181","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6085,"log_dups_size":0,"ondisk_log_size":6085,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:36.453841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432930816,"num_objects":6606,"num_object_clones":0,"num_object_copies":19818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8266,"num_write_kb":475904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":145,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.908924+0000","last_change":"2024-09-13T14:59:00.927482+0000","last_active":"2024-09-13T14:59:14.908924+0000","last_peered":"2024-09-13T14:59:14.908924+0000","last_clean":"2024-09-13T14:59:14.908924+0000","last_became_active":"2024-09-13T14:59:00.927101+0000","last_became_peered":"2024-09-13T14:59:00.927101+0000","last_unstale":"2024-09-13T14:59:14.908924+0000","last_undegraded":"2024-09-13T14:59:14.908924+0000","last_fullsized":"2024-09-13T14:59:14.908924+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.997001+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.997001+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:40:40.113760+0000","scrub_duration":0.019401547000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":510,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.233834+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T14:59:24.233834+0000","last_peered":"2024-09-13T14:59:24.233834+0000","last_clean":"2024-09-13T14:59:24.233834+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T14:59:24.233834+0000","last_undegraded":"2024-09-13T14:59:24.233834+0000","last_fullsized":"2024-09-13T14:59:24.233834+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":534,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.909127+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T14:59:14.909127+0000","last_peered":"2024-09-13T14:59:14.909127+0000","last_clean":"2024-09-13T14:59:14.909127+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T14:59:14.909127+0000","last_undegraded":"2024-09-13T14:59:14.909127+0000","last_fullsized":"2024-09-13T14:59:14.909127+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":498,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.237410+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T14:59:24.237410+0000","last_peered":"2024-09-13T14:59:24.237410+0000","last_clean":"2024-09-13T14:59:24.237410+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T14:59:24.237410+0000","last_undegraded":"2024-09-13T14:59:24.237410+0000","last_fullsized":"2024-09-13T14:59:24.237410+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"321'8214","reported_seq":8242,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:37.109209+0000","last_change":"2024-09-13T14:57:34.562351+0000","last_active":"2024-09-13T14:59:37.109209+0000","last_peered":"2024-09-13T14:59:37.109209+0000","last_clean":"2024-09-13T14:59:37.109209+0000","last_became_active":"2024-09-13T14:57:34.561412+0000","last_became_peered":"2024-09-13T14:57:34.561412+0000","last_unstale":"2024-09-13T14:59:37.109209+0000","last_undegraded":"2024-09-13T14:59:37.109209+0000","last_fullsized":"2024-09-13T14:59:37.109209+0000","mapping_epoch":302,"log_start":"308'2124","ondisk_log_start":"308'2124","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6090,"log_dups_size":0,"ondisk_log_size":6090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:37:07.655839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435355672,"num_objects":6644,"num_object_clones":0,"num_object_copies":19932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6644,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8214,"num_write_kb":475393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":272,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.912095+0000","last_change":"2024-09-13T14:55:58.308418+0000","last_active":"2024-09-13T14:59:14.912095+0000","last_peered":"2024-09-13T14:59:14.912095+0000","last_clean":"2024-09-13T14:59:14.912095+0000","last_became_active":"2024-09-13T14:55:51.160771+0000","last_became_peered":"2024-09-13T14:55:51.160771+0000","last_unstale":"2024-09-13T14:59:14.912095+0000","last_undegraded":"2024-09-13T14:59:14.912095+0000","last_fullsized":"2024-09-13T14:59:14.912095+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.308240+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.308240+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:05.665088+0000","scrub_duration":0.0063105979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":42,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.906238+0000","last_change":"2024-09-13T14:59:00.936064+0000","last_active":"2024-09-13T14:59:14.906238+0000","last_peered":"2024-09-13T14:59:14.906238+0000","last_clean":"2024-09-13T14:59:14.906238+0000","last_became_active":"2024-09-13T14:59:00.932311+0000","last_became_peered":"2024-09-13T14:59:00.932311+0000","last_unstale":"2024-09-13T14:59:14.906238+0000","last_undegraded":"2024-09-13T14:59:14.906238+0000","last_fullsized":"2024-09-13T14:59:14.906238+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:16.456004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":868,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.232047+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T14:59:24.232047+0000","last_peered":"2024-09-13T14:59:24.232047+0000","last_clean":"2024-09-13T14:59:24.232047+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T14:59:24.232047+0000","last_undegraded":"2024-09-13T14:59:24.232047+0000","last_fullsized":"2024-09-13T14:59:24.232047+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":43,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.905299+0000","last_change":"2024-09-13T14:59:00.924324+0000","last_active":"2024-09-13T14:59:14.905299+0000","last_peered":"2024-09-13T14:59:14.905299+0000","last_clean":"2024-09-13T14:59:14.905299+0000","last_became_active":"2024-09-13T14:59:00.923798+0000","last_became_peered":"2024-09-13T14:59:00.923798+0000","last_unstale":"2024-09-13T14:59:14.905299+0000","last_undegraded":"2024-09-13T14:59:14.905299+0000","last_fullsized":"2024-09-13T14:59:14.905299+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:10.994877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"321'8425","reported_seq":8484,"reported_epoch":321,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-09-13T14:59:37.164014+0000","last_change":"2024-09-13T14:59:12.908579+0000","last_active":"2024-09-13T14:59:37.164014+0000","last_peered":"2024-09-13T14:59:37.164014+0000","last_clean":"2024-09-13T14:58:59.887259+0000","last_became_active":"2024-09-13T14:59:12.883084+0000","last_became_peered":"2024-09-13T14:59:12.883084+0000","last_unstale":"2024-09-13T14:59:37.164014+0000","last_undegraded":"2024-09-13T14:59:12.880342+0000","last_fullsized":"2024-09-13T14:59:12.880157+0000","mapping_epoch":318,"log_start":"307'1700","ondisk_log_start":"307'1700","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6725,"log_dups_size":0,"ondisk_log_size":6725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:21.422134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":445054976,"num_objects":6791,"num_object_clones":0,"num_object_copies":20373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6791,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8425,"num_write_kb":486912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":6791}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":19,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.912077+0000","last_change":"2024-09-13T14:59:00.925524+0000","last_active":"2024-09-13T14:59:14.912077+0000","last_peered":"2024-09-13T14:59:14.912077+0000","last_clean":"2024-09-13T14:59:14.912077+0000","last_became_active":"2024-09-13T14:59:00.925167+0000","last_became_peered":"2024-09-13T14:59:00.925167+0000","last_unstale":"2024-09-13T14:59:14.912077+0000","last_undegraded":"2024-09-13T14:59:14.912077+0000","last_fullsized":"2024-09-13T14:59:14.912077+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.914801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.914801+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":22,"num_write":4,"num_write_kb":49,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":17890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":520,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.236817+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T14:59:24.236817+0000","last_peered":"2024-09-13T14:59:24.236817+0000","last_clean":"2024-09-13T14:59:24.236817+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T14:59:24.236817+0000","last_undegraded":"2024-09-13T14:59:24.236817+0000","last_fullsized":"2024-09-13T14:59:24.236817+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1134,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.909299+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T14:59:14.909299+0000","last_peered":"2024-09-13T14:59:14.909299+0000","last_clean":"2024-09-13T14:59:14.909299+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T14:59:14.909299+0000","last_undegraded":"2024-09-13T14:59:14.909299+0000","last_fullsized":"2024-09-13T14:59:14.909299+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":288,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.911663+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T14:59:14.911663+0000","last_peered":"2024-09-13T14:59:14.911663+0000","last_clean":"2024-09-13T14:59:14.911663+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T14:59:14.911663+0000","last_undegraded":"2024-09-13T14:59:14.911663+0000","last_fullsized":"2024-09-13T14:59:14.911663+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":305,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.913066+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T14:59:14.913066+0000","last_peered":"2024-09-13T14:59:14.913066+0000","last_clean":"2024-09-13T14:59:14.913066+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T14:59:14.913066+0000","last_undegraded":"2024-09-13T14:59:14.913066+0000","last_fullsized":"2024-09-13T14:59:14.913066+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":527,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.237805+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T14:59:24.237805+0000","last_peered":"2024-09-13T14:59:24.237805+0000","last_clean":"2024-09-13T14:59:24.237805+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T14:59:24.237805+0000","last_undegraded":"2024-09-13T14:59:24.237805+0000","last_fullsized":"2024-09-13T14:59:24.237805+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":509,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.231953+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T14:59:24.231953+0000","last_peered":"2024-09-13T14:59:24.231953+0000","last_clean":"2024-09-13T14:59:24.231953+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T14:59:24.231953+0000","last_undegraded":"2024-09-13T14:59:24.231953+0000","last_fullsized":"2024-09-13T14:59:24.231953+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":311,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.908985+0000","last_change":"2024-09-13T14:59:00.929406+0000","last_active":"2024-09-13T14:59:14.908985+0000","last_peered":"2024-09-13T14:59:14.908985+0000","last_clean":"2024-09-13T14:59:14.908985+0000","last_became_active":"2024-09-13T14:59:00.928524+0000","last_became_peered":"2024-09-13T14:59:00.928524+0000","last_unstale":"2024-09-13T14:59:14.908985+0000","last_undegraded":"2024-09-13T14:59:14.908985+0000","last_fullsized":"2024-09-13T14:59:14.908985+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:10.609448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":87,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.233698+0000","last_change":"2024-09-13T14:56:01.235655+0000","last_active":"2024-09-13T14:59:24.233698+0000","last_peered":"2024-09-13T14:59:24.233698+0000","last_clean":"2024-09-13T14:59:24.233698+0000","last_became_active":"2024-09-13T14:55:55.195703+0000","last_became_peered":"2024-09-13T14:55:55.195703+0000","last_unstale":"2024-09-13T14:59:24.233698+0000","last_undegraded":"2024-09-13T14:59:24.233698+0000","last_fullsized":"2024-09-13T14:59:24.233698+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.235497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.235497+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:55.951927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":285,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.912255+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T14:59:14.912255+0000","last_peered":"2024-09-13T14:59:14.912255+0000","last_clean":"2024-09-13T14:59:14.912255+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T14:59:14.912255+0000","last_undegraded":"2024-09-13T14:59:14.912255+0000","last_fullsized":"2024-09-13T14:59:14.912255+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":292,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.908680+0000","last_change":"2024-09-13T14:59:00.926586+0000","last_active":"2024-09-13T14:59:14.908680+0000","last_peered":"2024-09-13T14:59:14.908680+0000","last_clean":"2024-09-13T14:59:14.908680+0000","last_became_active":"2024-09-13T14:59:00.926206+0000","last_became_peered":"2024-09-13T14:59:00.926206+0000","last_unstale":"2024-09-13T14:59:14.908680+0000","last_undegraded":"2024-09-13T14:59:14.908680+0000","last_fullsized":"2024-09-13T14:59:14.908680+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:17:18.805856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":513,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.236466+0000","last_change":"2024-09-13T14:59:00.932503+0000","last_active":"2024-09-13T14:59:24.236466+0000","last_peered":"2024-09-13T14:59:24.236466+0000","last_clean":"2024-09-13T14:59:24.236466+0000","last_became_active":"2024-09-13T14:59:00.931358+0000","last_became_peered":"2024-09-13T14:59:00.931358+0000","last_unstale":"2024-09-13T14:59:24.236466+0000","last_undegraded":"2024-09-13T14:59:24.236466+0000","last_fullsized":"2024-09-13T14:59:24.236466+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:19.226182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":346,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.230814+0000","last_change":"2024-09-13T14:59:00.927060+0000","last_active":"2024-09-13T14:59:24.230814+0000","last_peered":"2024-09-13T14:59:24.230814+0000","last_clean":"2024-09-13T14:59:24.230814+0000","last_became_active":"2024-09-13T14:59:00.925402+0000","last_became_peered":"2024-09-13T14:59:00.925402+0000","last_unstale":"2024-09-13T14:59:24.230814+0000","last_undegraded":"2024-09-13T14:59:24.230814+0000","last_fullsized":"2024-09-13T14:59:24.230814+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:31:08.336446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":105,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.909230+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T14:59:14.909230+0000","last_peered":"2024-09-13T14:59:14.909230+0000","last_clean":"2024-09-13T14:59:14.909230+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T14:59:14.909230+0000","last_undegraded":"2024-09-13T14:59:14.909230+0000","last_fullsized":"2024-09-13T14:59:14.909230+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":104,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.233093+0000","last_change":"2024-09-13T14:59:00.924196+0000","last_active":"2024-09-13T14:59:24.233093+0000","last_peered":"2024-09-13T14:59:24.233093+0000","last_clean":"2024-09-13T14:59:24.233093+0000","last_became_active":"2024-09-13T14:59:00.923901+0000","last_became_peered":"2024-09-13T14:59:00.923901+0000","last_unstale":"2024-09-13T14:59:24.233093+0000","last_undegraded":"2024-09-13T14:59:24.233093+0000","last_fullsized":"2024-09-13T14:59:24.233093+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:01.913804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":537,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.236933+0000","last_change":"2024-09-13T14:59:00.924814+0000","last_active":"2024-09-13T14:59:24.236933+0000","last_peered":"2024-09-13T14:59:24.236933+0000","last_clean":"2024-09-13T14:59:24.236933+0000","last_became_active":"2024-09-13T14:59:00.924501+0000","last_became_peered":"2024-09-13T14:59:00.924501+0000","last_unstale":"2024-09-13T14:59:24.236933+0000","last_undegraded":"2024-09-13T14:59:24.236933+0000","last_fullsized":"2024-09-13T14:59:24.236933+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:11:15.382207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":502,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.230467+0000","last_change":"2024-09-13T14:59:00.925431+0000","last_active":"2024-09-13T14:59:24.230467+0000","last_peered":"2024-09-13T14:59:24.230467+0000","last_clean":"2024-09-13T14:59:24.230467+0000","last_became_active":"2024-09-13T14:59:00.925023+0000","last_became_peered":"2024-09-13T14:59:00.925023+0000","last_unstale":"2024-09-13T14:59:24.230467+0000","last_undegraded":"2024-09-13T14:59:24.230467+0000","last_fullsized":"2024-09-13T14:59:24.230467+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:38:18.686253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":95,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.912233+0000","last_change":"2024-09-13T14:59:00.927250+0000","last_active":"2024-09-13T14:59:14.912233+0000","last_peered":"2024-09-13T14:59:14.912233+0000","last_clean":"2024-09-13T14:59:14.912233+0000","last_became_active":"2024-09-13T14:59:00.926953+0000","last_became_peered":"2024-09-13T14:59:00.926953+0000","last_unstale":"2024-09-13T14:59:14.912233+0000","last_undegraded":"2024-09-13T14:59:14.912233+0000","last_fullsized":"2024-09-13T14:59:14.912233+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:58.306547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":304,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.911794+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T14:59:14.911794+0000","last_peered":"2024-09-13T14:59:14.911794+0000","last_clean":"2024-09-13T14:59:14.911794+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T14:59:14.911794+0000","last_undegraded":"2024-09-13T14:59:14.911794+0000","last_fullsized":"2024-09-13T14:59:14.911794+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":336,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.234092+0000","last_change":"2024-09-13T14:59:00.934412+0000","last_active":"2024-09-13T14:59:24.234092+0000","last_peered":"2024-09-13T14:59:24.234092+0000","last_clean":"2024-09-13T14:59:24.234092+0000","last_became_active":"2024-09-13T14:59:00.932986+0000","last_became_peered":"2024-09-13T14:59:00.932986+0000","last_unstale":"2024-09-13T14:59:24.234092+0000","last_undegraded":"2024-09-13T14:59:24.234092+0000","last_fullsized":"2024-09-13T14:59:24.234092+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:21:44.285324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":499,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.237331+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T14:59:24.237331+0000","last_peered":"2024-09-13T14:59:24.237331+0000","last_clean":"2024-09-13T14:59:24.237331+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T14:59:24.237331+0000","last_undegraded":"2024-09-13T14:59:24.237331+0000","last_fullsized":"2024-09-13T14:59:24.237331+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":312,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.912996+0000","last_change":"2024-09-13T14:59:00.929751+0000","last_active":"2024-09-13T14:59:14.912996+0000","last_peered":"2024-09-13T14:59:14.912996+0000","last_clean":"2024-09-13T14:59:14.912996+0000","last_became_active":"2024-09-13T14:59:00.929456+0000","last_became_peered":"2024-09-13T14:59:00.929456+0000","last_unstale":"2024-09-13T14:59:14.912996+0000","last_undegraded":"2024-09-13T14:59:14.912996+0000","last_fullsized":"2024-09-13T14:59:14.912996+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:11.462853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":287,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.912150+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T14:59:14.912150+0000","last_peered":"2024-09-13T14:59:14.912150+0000","last_clean":"2024-09-13T14:59:14.912150+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T14:59:14.912150+0000","last_undegraded":"2024-09-13T14:59:14.912150+0000","last_fullsized":"2024-09-13T14:59:14.912150+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":312,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.911715+0000","last_change":"2024-09-13T14:59:00.928898+0000","last_active":"2024-09-13T14:59:14.911715+0000","last_peered":"2024-09-13T14:59:14.911715+0000","last_clean":"2024-09-13T14:59:14.911715+0000","last_became_active":"2024-09-13T14:59:00.928585+0000","last_became_peered":"2024-09-13T14:59:00.928585+0000","last_unstale":"2024-09-13T14:59:14.911715+0000","last_undegraded":"2024-09-13T14:59:14.911715+0000","last_fullsized":"2024-09-13T14:59:14.911715+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:26:57.155149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":326,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.912517+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T14:59:14.912517+0000","last_peered":"2024-09-13T14:59:14.912517+0000","last_clean":"2024-09-13T14:59:14.912517+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T14:59:14.912517+0000","last_undegraded":"2024-09-13T14:59:14.912517+0000","last_fullsized":"2024-09-13T14:59:14.912517+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":70,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.233776+0000","last_change":"2024-09-13T14:59:00.929835+0000","last_active":"2024-09-13T14:59:24.233776+0000","last_peered":"2024-09-13T14:59:24.233776+0000","last_clean":"2024-09-13T14:59:24.233776+0000","last_became_active":"2024-09-13T14:59:00.929514+0000","last_became_peered":"2024-09-13T14:59:00.929514+0000","last_unstale":"2024-09-13T14:59:24.233776+0000","last_undegraded":"2024-09-13T14:59:24.233776+0000","last_fullsized":"2024-09-13T14:59:24.233776+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:48.981240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":348,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.908800+0000","last_change":"2024-09-13T14:59:00.925895+0000","last_active":"2024-09-13T14:59:14.908800+0000","last_peered":"2024-09-13T14:59:14.908800+0000","last_clean":"2024-09-13T14:59:14.908800+0000","last_became_active":"2024-09-13T14:59:00.925564+0000","last_became_peered":"2024-09-13T14:59:00.925564+0000","last_unstale":"2024-09-13T14:59:14.908800+0000","last_undegraded":"2024-09-13T14:59:14.908800+0000","last_fullsized":"2024-09-13T14:59:14.908800+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:23.283927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":308,"reported_epoch":320,"state":"active+clean","last_fresh":"2024-09-13T14:59:14.911430+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T14:59:14.911430+0000","last_peered":"2024-09-13T14:59:14.911430+0000","last_clean":"2024-09-13T14:59:14.911430+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T14:59:14.911430+0000","last_undegraded":"2024-09-13T14:59:14.911430+0000","last_fullsized":"2024-09-13T14:59:14.911430+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":551,"reported_epoch":321,"state":"active+clean","last_fresh":"2024-09-13T14:59:24.236365+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T14:59:24.236365+0000","last_peered":"2024-09-13T14:59:24.236365+0000","last_clean":"2024-09-13T14:59:24.236365+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T14:59:24.236365+0000","last_undegraded":"2024-09-13T14:59:24.236365+0000","last_fullsized":"2024-09-13T14:59:24.236365+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":6945374232,"num_objects":105979,"num_object_clones":0,"num_object_copies":317937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47185,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":105979,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":132833,"num_write_kb":7641921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6071,"num_bytes_recovered":397869056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21472227328,"data_stored":21472215112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101795,"ondisk_log_size":101795,"up":48,"acting":40,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":16172,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":608258,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":9,"num_read_kb":22,"num_write":4,"num_write_kb":49,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":280578,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2100,"ondisk_log_size":2100,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397908,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4759368,"kb_used_data":3869888,"kb_used_omap":1,"kb_used_meta":889406,"kb_avail":88985784,"statfs":{"total":95995035648,"available":91121442816,"internally_reserved":0,"allocated":3962765312,"data_stored":3960553544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910752088},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594137,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1595904,"kb_used_data":1294392,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92149248,"statfs":{"total":95995035648,"available":94360829952,"internally_reserved":0,"allocated":1325457408,"data_stored":1323253924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790366,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5055228,"kb_used_data":4170676,"kb_used_omap":1,"kb_used_meta":884542,"kb_avail":88689924,"statfs":{"total":95995035648,"available":90818482176,"internally_reserved":0,"allocated":4270772224,"data_stored":4268568757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":905771352},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986595,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2237240,"kb_used_data":1335792,"kb_used_omap":1,"kb_used_meta":901374,"kb_avail":91507912,"statfs":{"total":95995035648,"available":93704101888,"internally_reserved":0,"allocated":1367851008,"data_stored":1365650800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923007320},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":0,"seq":446676599037,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762570,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2220984,"kb_used_data":1399920,"kb_used_omap":8,"kb_used_meta":821047,"kb_avail":91524168,"statfs":{"total":95995035648,"available":93720748032,"internally_reserved":0,"allocated":1433518080,"data_stored":1431318680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":840752586},"hb_peers":[0,1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926096,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4579700,"kb_used_data":3796012,"kb_used_omap":7,"kb_used_meta":783672,"kb_avail":89165452,"statfs":{"total":95995035648,"available":91305422848,"internally_reserved":0,"allocated":3887116288,"data_stored":3884920101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":802480841},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122336,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2420996,"kb_used_data":1743036,"kb_used_omap":7,"kb_used_meta":677944,"kb_avail":91324156,"statfs":{"total":95995035648,"available":93515935744,"internally_reserved":0,"allocated":1784868864,"data_stored":1782661676,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":694215247},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1779433472,"data_stored":1779433472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3874230272,"data_stored":3874226200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1418919936,"data_stored":1418919936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3548708864,"data_stored":3548708864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1358954496,"data_stored":1358954496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4250210304,"data_stored":4250206232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1313538048,"data_stored":1313538048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3928231936,"data_stored":3928227864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:59:43.448 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:59:43.449 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pg_num 31 2024-09-13T14:59:43.530 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:43.565 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:43.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:43.668 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:43.705 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:43.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:43.806 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:43.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.842 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:43.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:43.943 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:43.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.977 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:43.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:44.078 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:44.104 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T14:59:44.104 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T14:59:44.104 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T14:59:44.104 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T14:59:44.104 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T14:59:44.104 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T14:59:44.104 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T14:59:44.105 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:59:44.105 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:59:44.105 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:59:44.105 INFO:teuthology.orchestra.run.smithi096.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T14:59:44.105 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:59:44.105 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:59:44.105 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:59:44.105 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:59:44.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.107 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:44.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:44.208 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:44.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:43 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2105785995' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:44.249 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:44.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:44.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:43 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2105785995' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:44.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:43 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2105785995' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:44.350 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:44.376 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:44.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:44.478 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:44.505 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:44.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:44.526 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:59:44.607 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:44.647 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:44.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:44.749 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:44.789 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:44.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:44.891 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:44.923 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:44.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:45.025 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:45.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.060 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:45.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:45.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:44 smithi096 ceph-mon[98901]: pgmap v1057: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 45891/308826 objects degraded (14.860%); 8.9 MiB/s, 142 objects/s recovering 2024-09-13T14:59:45.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:44 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 47073/317313 objects degraded (14.835%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:45.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:44 smithi096 ceph-mon[102928]: pgmap v1057: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 45891/308826 objects degraded (14.860%); 8.9 MiB/s, 142 objects/s recovering 2024-09-13T14:59:45.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:44 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 47073/317313 objects degraded (14.835%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:45.161 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:45.187 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:45.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:45.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:44 smithi204 ceph-mon[87071]: pgmap v1057: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 45891/308826 objects degraded (14.860%); 8.9 MiB/s, 142 objects/s recovering 2024-09-13T14:59:45.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:44 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 47073/317313 objects degraded (14.835%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:45.289 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:45.313 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:59:45.313 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:59:45.315 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:45.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:45.417 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:45.447 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:45.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:45.549 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:45.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.586 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:45.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:45.687 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:45.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.726 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:45.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:45.803 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:45.822 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:45.826 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:45.832 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T14:59:45.832 INFO:teuthology.orchestra.run.smithi096.stdout: "size": 20, 2024-09-13T14:59:45.832 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 600, 2024-09-13T14:59:45.832 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T14:59:45.832 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.832 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:214049 8.1 8:8f432ceb:::benchmark_data_smithi096_227542_object88654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T14:59:45.833 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:30.952680+0000", 2024-09-13T14:59:45.833 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 194.87565646600001, 2024-09-13T14:59:45.833 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.129298617, 2024-09-13T14:59:45.833 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.833 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.833 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.833 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.833 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:59:45.833 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:59:45.833 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 214049 2024-09-13T14:59:45.833 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.833 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.834 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.834 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.834 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.952680+0000", 2024-09-13T14:59:45.834 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.834 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.834 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.834 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.834 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.952680+0000", 2024-09-13T14:59:45.834 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.834 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.834 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.834 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.834 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.952681+0000", 2024-09-13T14:59:45.834 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.8299999999999995e-07 2024-09-13T14:59:45.835 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.835 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.835 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.835 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.952685+0000", 2024-09-13T14:59:45.835 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0040000000000001e-06 2024-09-13T14:59:45.835 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.835 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.835 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.835 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.952698+0000", 2024-09-13T14:59:45.835 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3626e-05 2024-09-13T14:59:45.835 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.835 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.835 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.836 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.952705+0000", 2024-09-13T14:59:45.836 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.0339999999999999e-06 2024-09-13T14:59:45.836 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.836 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.836 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.836 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.953333+0000", 2024-09-13T14:59:45.836 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00062725900000000002 2024-09-13T14:59:45.836 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.836 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.836 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.836 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.953584+0000", 2024-09-13T14:59:45.836 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00025121699999999998 2024-09-13T14:59:45.836 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.836 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.837 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:59:45.837 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.953732+0000", 2024-09-13T14:59:45.837 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000147613 2024-09-13T14:59:45.837 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.837 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.837 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.837 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.958440+0000", 2024-09-13T14:59:45.837 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0047083330000000003 2024-09-13T14:59:45.837 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.837 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.837 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.837 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.962904+0000", 2024-09-13T14:59:45.838 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0044642839999999998 2024-09-13T14:59:45.838 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.838 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.838 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.838 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:31.081752+0000", 2024-09-13T14:59:45.838 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.11884774400000001 2024-09-13T14:59:45.838 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.838 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.838 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.838 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:31.081817+0000", 2024-09-13T14:59:45.838 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4684000000000006e-05 2024-09-13T14:59:45.838 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.838 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.838 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.839 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:31.081978+0000", 2024-09-13T14:59:45.839 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000161836 2024-09-13T14:59:45.839 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.839 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.839 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.839 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.839 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.839 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:214058 8.1 8:8f601dc9:::benchmark_data_smithi096_227542_object88663:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T14:59:45.839 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:30.958506+0000", 2024-09-13T14:59:45.839 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 194.869830202, 2024-09-13T14:59:45.839 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.132683099, 2024-09-13T14:59:45.839 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.839 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.840 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.840 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.840 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:59:45.840 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:59:45.840 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 214058 2024-09-13T14:59:45.840 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.840 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.840 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.840 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.840 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.958506+0000", 2024-09-13T14:59:45.840 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.840 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.840 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.841 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.841 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.958506+0000", 2024-09-13T14:59:45.841 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.841 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.841 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.841 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.841 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.958507+0000", 2024-09-13T14:59:45.841 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2899999999999999e-06 2024-09-13T14:59:45.841 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.841 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.841 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.841 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.958512+0000", 2024-09-13T14:59:45.841 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8319999999999996e-06 2024-09-13T14:59:45.842 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.842 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.842 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.842 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.958523+0000", 2024-09-13T14:59:45.842 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0457e-05 2024-09-13T14:59:45.842 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.842 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.842 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.842 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.958529+0000", 2024-09-13T14:59:45.842 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6710000000000002e-06 2024-09-13T14:59:45.842 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.842 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.842 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.842 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.958640+0000", 2024-09-13T14:59:45.843 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011094000000000001 2024-09-13T14:59:45.843 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.843 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.843 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.843 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.958737+0000", 2024-09-13T14:59:45.843 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.6644999999999994e-05 2024-09-13T14:59:45.843 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.843 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.843 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:59:45.843 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.958834+0000", 2024-09-13T14:59:45.843 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7201999999999995e-05 2024-09-13T14:59:45.843 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.844 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.844 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.844 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.962953+0000", 2024-09-13T14:59:45.844 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.004118839 2024-09-13T14:59:45.844 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.844 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.844 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.844 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.963340+0000", 2024-09-13T14:59:45.844 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00038726100000000002 2024-09-13T14:59:45.844 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.844 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.844 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.844 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:31.091046+0000", 2024-09-13T14:59:45.845 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.12770536900000001 2024-09-13T14:59:45.845 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.845 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.845 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.845 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:31.091141+0000", 2024-09-13T14:59:45.845 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.5637999999999995e-05 2024-09-13T14:59:45.845 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.849 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.849 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.849 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:31.091189+0000", 2024-09-13T14:59:45.849 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7954999999999997e-05 2024-09-13T14:59:45.849 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.849 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.849 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.849 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.849 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.849 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:214069 8.1 8:8bb1f952:::benchmark_data_smithi096_227542_object88674:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T14:59:45.849 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:30.969874+0000", 2024-09-13T14:59:45.849 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 194.85846275700001, 2024-09-13T14:59:45.850 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.12350288600000001, 2024-09-13T14:59:45.850 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.850 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.850 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.850 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.850 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:59:45.850 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:59:45.850 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 214069 2024-09-13T14:59:45.851 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.851 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.851 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.851 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.851 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.969874+0000", 2024-09-13T14:59:45.852 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.852 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.852 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.852 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.852 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.969874+0000", 2024-09-13T14:59:45.852 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.852 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.852 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.852 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.853 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.969875+0000", 2024-09-13T14:59:45.853 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0899999999999999e-06 2024-09-13T14:59:45.853 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.853 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.853 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.853 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.969878+0000", 2024-09-13T14:59:45.853 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.799e-06 2024-09-13T14:59:45.853 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.853 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.854 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.854 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.969890+0000", 2024-09-13T14:59:45.854 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1590000000000001e-05 2024-09-13T14:59:45.854 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.854 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.854 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.854 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.969897+0000", 2024-09-13T14:59:45.854 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9070000000000001e-06 2024-09-13T14:59:45.854 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.854 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.854 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.855 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.969951+0000", 2024-09-13T14:59:45.855 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3841999999999997e-05 2024-09-13T14:59:45.855 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.855 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.855 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.855 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.970067+0000", 2024-09-13T14:59:45.855 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011600899999999999 2024-09-13T14:59:45.855 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.855 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.855 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:59:45.855 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.970194+0000", 2024-09-13T14:59:45.856 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012697399999999999 2024-09-13T14:59:45.856 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.856 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.856 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.856 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.973373+0000", 2024-09-13T14:59:45.856 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0031790109999999998 2024-09-13T14:59:45.856 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.856 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.856 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.856 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:30.974073+0000", 2024-09-13T14:59:45.857 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00070004500000000005 2024-09-13T14:59:45.857 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.857 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.857 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.857 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:31.093284+0000", 2024-09-13T14:59:45.857 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.119211647 2024-09-13T14:59:45.857 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.857 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.857 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.857 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:31.093337+0000", 2024-09-13T14:59:45.857 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2169999999999997e-05 2024-09-13T14:59:45.858 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.858 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.858 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.858 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:31.093376+0000", 2024-09-13T14:59:45.858 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9802000000000003e-05 2024-09-13T14:59:45.858 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.859 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.859 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.859 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.859 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.859 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:238715 8.1 8:8940785b:::benchmark_data_smithi096_227542_object113320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T14:59:45.859 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:56.932427+0000", 2024-09-13T14:59:45.859 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 168.89590940900001, 2024-09-13T14:59:45.859 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.241978743, 2024-09-13T14:59:45.860 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.860 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.860 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.860 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.860 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:59:45.860 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:59:45.860 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 238715 2024-09-13T14:59:45.860 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.860 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.861 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.861 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.861 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.932427+0000", 2024-09-13T14:59:45.861 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.861 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.861 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.861 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.861 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.932427+0000", 2024-09-13T14:59:45.861 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.861 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.861 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.861 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.862 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.932428+0000", 2024-09-13T14:59:45.862 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3689999999999999e-06 2024-09-13T14:59:45.862 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.862 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.862 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.862 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.932433+0000", 2024-09-13T14:59:45.862 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9309999999999999e-06 2024-09-13T14:59:45.862 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.862 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.862 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.862 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.932445+0000", 2024-09-13T14:59:45.862 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1946999999999999e-05 2024-09-13T14:59:45.862 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.863 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.863 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.863 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.932452+0000", 2024-09-13T14:59:45.863 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8920000000000003e-06 2024-09-13T14:59:45.863 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.863 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.863 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.863 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.932580+0000", 2024-09-13T14:59:45.863 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012845900000000001 2024-09-13T14:59:45.863 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.863 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.863 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.864 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.932892+0000", 2024-09-13T14:59:45.864 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00031134400000000002 2024-09-13T14:59:45.864 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.864 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.864 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:59:45.864 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.933024+0000", 2024-09-13T14:59:45.864 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000131867 2024-09-13T14:59:45.864 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.864 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.864 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.864 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.945751+0000", 2024-09-13T14:59:45.864 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.012727054 2024-09-13T14:59:45.864 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.864 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.865 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.865 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.945770+0000", 2024-09-13T14:59:45.865 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9511e-05 2024-09-13T14:59:45.865 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.865 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.865 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.865 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.173923+0000", 2024-09-13T14:59:45.865 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.22815306499999999 2024-09-13T14:59:45.865 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.865 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.865 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.865 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.174076+0000", 2024-09-13T14:59:45.866 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00015217400000000001 2024-09-13T14:59:45.866 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.866 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.866 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.866 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.174406+0000", 2024-09-13T14:59:45.866 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00033012999999999999 2024-09-13T14:59:45.866 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.866 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.866 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.866 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.866 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.866 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:238718 8.1 8:83756c58:::benchmark_data_smithi096_227542_object113323:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T14:59:45.866 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:56.936972+0000", 2024-09-13T14:59:45.866 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 168.89136471899999, 2024-09-13T14:59:45.867 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.23953401899999999, 2024-09-13T14:59:45.867 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.867 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.867 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.867 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.867 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:59:45.867 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:59:45.867 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 238718 2024-09-13T14:59:45.867 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.867 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.867 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.867 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.867 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.936972+0000", 2024-09-13T14:59:45.868 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.868 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.868 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.868 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.868 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.936972+0000", 2024-09-13T14:59:45.868 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.868 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.868 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.868 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.868 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.936973+0000", 2024-09-13T14:59:45.868 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1659999999999999e-06 2024-09-13T14:59:45.868 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.868 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.869 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.869 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.936978+0000", 2024-09-13T14:59:45.869 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9570000000000001e-06 2024-09-13T14:59:45.869 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.869 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.869 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.869 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.936990+0000", 2024-09-13T14:59:45.869 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2573e-05 2024-09-13T14:59:45.869 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.869 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.869 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.869 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.936999+0000", 2024-09-13T14:59:45.869 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5439999999999999e-06 2024-09-13T14:59:45.870 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.870 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.870 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.870 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.937184+0000", 2024-09-13T14:59:45.870 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00018521 2024-09-13T14:59:45.870 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.870 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.870 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.870 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.937320+0000", 2024-09-13T14:59:45.871 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013635399999999999 2024-09-13T14:59:45.871 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.871 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.871 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:59:45.871 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.937436+0000", 2024-09-13T14:59:45.871 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011532 2024-09-13T14:59:45.871 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.871 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.871 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.871 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.945776+0000", 2024-09-13T14:59:45.871 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0083401199999999995 2024-09-13T14:59:45.871 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.872 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.872 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.872 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.951747+0000", 2024-09-13T14:59:45.872 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0059707780000000004 2024-09-13T14:59:45.872 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.872 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.872 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.872 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.176404+0000", 2024-09-13T14:59:45.872 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.224657358 2024-09-13T14:59:45.872 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.872 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.872 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.872 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.176460+0000", 2024-09-13T14:59:45.872 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6322e-05 2024-09-13T14:59:45.873 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.873 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.873 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.873 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.176506+0000", 2024-09-13T14:59:45.873 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5317e-05 2024-09-13T14:59:45.873 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.873 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.873 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.873 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.873 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.873 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:238722 8.1 8:8bdc54e3:::benchmark_data_smithi096_227542_object113327:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T14:59:45.873 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:56.939865+0000", 2024-09-13T14:59:45.873 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 168.88847178699999, 2024-09-13T14:59:45.873 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.23677016000000001, 2024-09-13T14:59:45.874 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.874 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.874 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.874 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.874 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:59:45.874 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:59:45.874 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 238722 2024-09-13T14:59:45.874 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.874 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.874 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.874 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.874 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.939865+0000", 2024-09-13T14:59:45.874 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.875 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.875 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.875 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.875 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.939865+0000", 2024-09-13T14:59:45.875 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.875 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.875 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.875 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.875 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.939866+0000", 2024-09-13T14:59:45.875 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.327e-06 2024-09-13T14:59:45.875 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.875 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.875 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.875 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.939870+0000", 2024-09-13T14:59:45.875 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.8709999999999999e-06 2024-09-13T14:59:45.876 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.876 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.876 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.876 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.939880+0000", 2024-09-13T14:59:45.876 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0631e-05 2024-09-13T14:59:45.876 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.876 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.876 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.876 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.939887+0000", 2024-09-13T14:59:45.876 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5259999999999999e-06 2024-09-13T14:59:45.876 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.876 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.876 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.877 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.941827+0000", 2024-09-13T14:59:45.877 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001940591 2024-09-13T14:59:45.877 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.877 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.877 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.877 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.941937+0000", 2024-09-13T14:59:45.877 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000109812 2024-09-13T14:59:45.877 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.877 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.877 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:59:45.877 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.942050+0000", 2024-09-13T14:59:45.877 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011301799999999999 2024-09-13T14:59:45.877 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.877 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.877 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.878 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.946821+0000", 2024-09-13T14:59:45.878 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0047703190000000003 2024-09-13T14:59:45.878 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.878 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.878 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.878 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.951794+0000", 2024-09-13T14:59:45.878 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0049736600000000004 2024-09-13T14:59:45.878 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.878 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.878 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.878 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.176547+0000", 2024-09-13T14:59:45.878 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.224752232 2024-09-13T14:59:45.878 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.878 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.878 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.879 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.176599+0000", 2024-09-13T14:59:45.879 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.27e-05 2024-09-13T14:59:45.879 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.879 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.879 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.879 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.176635+0000", 2024-09-13T14:59:45.879 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.5472999999999997e-05 2024-09-13T14:59:45.879 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.879 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.879 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.879 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.879 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.879 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:238729 8.1 8:86fa6d6f:::benchmark_data_smithi096_227542_object113334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T14:59:45.879 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:56.948519+0000", 2024-09-13T14:59:45.880 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 168.87981721099999, 2024-09-13T14:59:45.880 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.22822959200000001, 2024-09-13T14:59:45.880 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.880 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.880 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.880 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.880 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:59:45.880 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:59:45.880 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 238729 2024-09-13T14:59:45.880 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.880 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.880 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.880 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.880 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.948519+0000", 2024-09-13T14:59:45.881 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.881 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.881 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.881 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.881 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.948519+0000", 2024-09-13T14:59:45.881 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.881 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.881 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.881 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.882 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.948520+0000", 2024-09-13T14:59:45.882 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3319999999999999e-06 2024-09-13T14:59:45.882 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.882 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.882 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.882 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.948525+0000", 2024-09-13T14:59:45.882 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0010000000000001e-06 2024-09-13T14:59:45.882 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.882 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.882 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.882 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.948535+0000", 2024-09-13T14:59:45.882 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.6500000000000008e-06 2024-09-13T14:59:45.882 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.882 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.883 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.883 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.948542+0000", 2024-09-13T14:59:45.883 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7800000000000003e-06 2024-09-13T14:59:45.883 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.883 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.883 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.883 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.948627+0000", 2024-09-13T14:59:45.883 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.4917e-05 2024-09-13T14:59:45.883 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.883 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.883 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.883 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.948741+0000", 2024-09-13T14:59:45.883 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011445799999999999 2024-09-13T14:59:45.884 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.884 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.884 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:59:45.884 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.948864+0000", 2024-09-13T14:59:45.884 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000122533 2024-09-13T14:59:45.884 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.884 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.884 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.884 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.951913+0000", 2024-09-13T14:59:45.884 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0030492509999999998 2024-09-13T14:59:45.884 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.884 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.884 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.885 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.952803+0000", 2024-09-13T14:59:45.885 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00089032900000000003 2024-09-13T14:59:45.885 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.885 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.885 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.885 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.176666+0000", 2024-09-13T14:59:45.885 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.22386224199999999 2024-09-13T14:59:45.885 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.885 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.885 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.885 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.176714+0000", 2024-09-13T14:59:45.885 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8439000000000002e-05 2024-09-13T14:59:45.885 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.886 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.886 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.886 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.176749+0000", 2024-09-13T14:59:45.886 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4659999999999997e-05 2024-09-13T14:59:45.886 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.886 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.886 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.886 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.886 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.886 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:238734 8.1 8:8e21aca4:::benchmark_data_smithi096_227542_object113339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T14:59:45.886 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:56.954599+0000", 2024-09-13T14:59:45.886 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 168.873736853, 2024-09-13T14:59:45.887 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.23521483900000001, 2024-09-13T14:59:45.887 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.887 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.887 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.887 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.887 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:59:45.887 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:59:45.887 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 238734 2024-09-13T14:59:45.887 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.887 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.887 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.887 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.888 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.954599+0000", 2024-09-13T14:59:45.888 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.888 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.888 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.888 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.888 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.954599+0000", 2024-09-13T14:59:45.888 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.888 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.888 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.888 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.888 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.954601+0000", 2024-09-13T14:59:45.888 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.085e-06 2024-09-13T14:59:45.888 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.889 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.889 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.889 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.954605+0000", 2024-09-13T14:59:45.889 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9150000000000004e-06 2024-09-13T14:59:45.889 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.889 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.889 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.889 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.954615+0000", 2024-09-13T14:59:45.889 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2220000000000008e-06 2024-09-13T14:59:45.890 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.890 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.890 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.890 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.954620+0000", 2024-09-13T14:59:45.890 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8720000000000001e-06 2024-09-13T14:59:45.890 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.890 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.890 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.890 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.954675+0000", 2024-09-13T14:59:45.890 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5340000000000002e-05 2024-09-13T14:59:45.890 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.890 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.890 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.891 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.954783+0000", 2024-09-13T14:59:45.891 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000108498 2024-09-13T14:59:45.891 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.891 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.891 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:59:45.891 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.954897+0000", 2024-09-13T14:59:45.891 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000113278 2024-09-13T14:59:45.891 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.891 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.892 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.892 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.958453+0000", 2024-09-13T14:59:45.892 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0035567039999999999 2024-09-13T14:59:45.892 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.892 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.892 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.892 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.958676+0000", 2024-09-13T14:59:45.892 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00022229500000000001 2024-09-13T14:59:45.892 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.892 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.892 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.893 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.189680+0000", 2024-09-13T14:59:45.893 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.23100427400000001 2024-09-13T14:59:45.893 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.893 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.893 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.893 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.189769+0000", 2024-09-13T14:59:45.893 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9409999999999999e-05 2024-09-13T14:59:45.893 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.893 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.893 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.893 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.189814+0000", 2024-09-13T14:59:45.894 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4946e-05 2024-09-13T14:59:45.894 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.894 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.894 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.894 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.894 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.894 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.45749.0:238743 8.1 8:8c289ee8:::benchmark_data_smithi096_227542_object113348:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T14:59:45.894 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:56:56.980800+0000", 2024-09-13T14:59:45.894 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 168.84753658400001, 2024-09-13T14:59:45.894 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.21163926699999999, 2024-09-13T14:59:45.895 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.895 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.895 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.895 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.895 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.45749", 2024-09-13T14:59:45.895 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T14:59:45.895 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 238743 2024-09-13T14:59:45.895 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.895 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.896 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.896 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.896 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.980800+0000", 2024-09-13T14:59:45.896 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.896 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.896 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.896 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.896 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.980800+0000", 2024-09-13T14:59:45.896 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.897 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.897 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.897 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.897 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.980801+0000", 2024-09-13T14:59:45.897 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.093e-06 2024-09-13T14:59:45.897 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.897 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.897 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.897 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.980804+0000", 2024-09-13T14:59:45.897 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.3550000000000001e-06 2024-09-13T14:59:45.897 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.898 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.898 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.898 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.980814+0000", 2024-09-13T14:59:45.898 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.6690000000000001e-06 2024-09-13T14:59:45.898 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.898 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.898 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.898 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.980819+0000", 2024-09-13T14:59:45.898 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.8740000000000003e-06 2024-09-13T14:59:45.898 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.898 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.898 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.898 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.980859+0000", 2024-09-13T14:59:45.898 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0136000000000003e-05 2024-09-13T14:59:45.899 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.899 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.899 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.899 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.980932+0000", 2024-09-13T14:59:45.899 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2922000000000004e-05 2024-09-13T14:59:45.899 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.899 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.899 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:59:45.899 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.981001+0000", 2024-09-13T14:59:45.899 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9320999999999996e-05 2024-09-13T14:59:45.899 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.899 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.899 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.900 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.983786+0000", 2024-09-13T14:59:45.900 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0027852110000000001 2024-09-13T14:59:45.900 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.900 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.903 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.903 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:56.984165+0000", 2024-09-13T14:59:45.903 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00037890800000000001 2024-09-13T14:59:45.903 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.903 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.903 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.903 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.192347+0000", 2024-09-13T14:59:45.903 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.20818134099999999 2024-09-13T14:59:45.903 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.903 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.903 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.904 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.192396+0000", 2024-09-13T14:59:45.904 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9679000000000003e-05 2024-09-13T14:59:45.904 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.904 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.904 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.904 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:56:57.192439+0000", 2024-09-13T14:59:45.904 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2758e-05 2024-09-13T14:59:45.904 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.904 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.904 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.904 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.904 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.904 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:78821 9.f e312/302 9:f5944826:::benchmark_data_smithi096_256909_object78820:head v 312'4927, mlcod=312'4923)", 2024-09-13T14:59:45.905 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:58:34.313988+0000", 2024-09-13T14:59:45.905 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 71.514348261999999, 2024-09-13T14:59:45.905 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.104358561, 2024-09-13T14:59:45.905 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.905 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.905 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.905 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.905 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.905 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.905 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.313988+0000", 2024-09-13T14:59:45.905 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.905 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.905 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.905 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.906 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.313988+0000", 2024-09-13T14:59:45.906 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.906 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.906 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.906 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.906 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.313989+0000", 2024-09-13T14:59:45.906 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9299999999999997e-07 2024-09-13T14:59:45.906 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.906 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.906 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.906 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.314074+0000", 2024-09-13T14:59:45.907 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5175999999999999e-05 2024-09-13T14:59:45.907 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.907 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.907 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.907 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.404607+0000", 2024-09-13T14:59:45.907 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.090533407999999996 2024-09-13T14:59:45.907 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.907 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.907 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.907 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.404616+0000", 2024-09-13T14:59:45.907 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1799999999999996e-06 2024-09-13T14:59:45.907 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.907 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.908 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.908 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.404809+0000", 2024-09-13T14:59:45.908 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00019318800000000001 2024-09-13T14:59:45.908 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.908 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.908 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.908 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.404830+0000", 2024-09-13T14:59:45.908 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0822000000000002e-05 2024-09-13T14:59:45.908 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.909 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.909 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.909 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.418309+0000", 2024-09-13T14:59:45.909 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.013479560999999999 2024-09-13T14:59:45.909 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.909 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.909 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.909 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.418347+0000", 2024-09-13T14:59:45.909 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7432999999999999e-05 2024-09-13T14:59:45.909 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.909 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.909 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.909 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.910 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.910 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.46040.0:79400 9.0 9:0b73bf0e:::benchmark_data_smithi096_256909_object79399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T14:59:45.910 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:58:34.846859+0000", 2024-09-13T14:59:45.910 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 70.981477290000001, 2024-09-13T14:59:45.910 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.10379059, 2024-09-13T14:59:45.910 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.910 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.910 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.910 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.910 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.46040", 2024-09-13T14:59:45.910 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:59:45.910 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 79400 2024-09-13T14:59:45.910 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.911 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.911 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.911 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.911 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.846859+0000", 2024-09-13T14:59:45.911 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.911 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.911 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.911 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.911 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.846859+0000", 2024-09-13T14:59:45.911 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.911 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.911 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.911 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.912 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.846860+0000", 2024-09-13T14:59:45.912 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4130000000000001e-06 2024-09-13T14:59:45.912 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.912 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.912 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.912 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.846882+0000", 2024-09-13T14:59:45.912 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1875999999999999e-05 2024-09-13T14:59:45.912 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.912 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.912 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.912 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.846895+0000", 2024-09-13T14:59:45.912 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2225000000000001e-05 2024-09-13T14:59:45.912 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.913 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.913 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.913 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.846902+0000", 2024-09-13T14:59:45.913 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9800000000000001e-06 2024-09-13T14:59:45.913 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.913 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.913 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.913 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.846959+0000", 2024-09-13T14:59:45.913 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7018999999999997e-05 2024-09-13T14:59:45.913 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.913 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.913 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.913 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.847090+0000", 2024-09-13T14:59:45.914 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013183499999999999 2024-09-13T14:59:45.914 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.914 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.914 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,6", 2024-09-13T14:59:45.914 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.847207+0000", 2024-09-13T14:59:45.914 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011705900000000001 2024-09-13T14:59:45.914 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.914 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.914 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.914 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.851288+0000", 2024-09-13T14:59:45.914 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0040808479999999998 2024-09-13T14:59:45.914 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.914 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.914 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.915 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.854565+0000", 2024-09-13T14:59:45.915 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.003276242 2024-09-13T14:59:45.915 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.915 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.915 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.915 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.950559+0000", 2024-09-13T14:59:45.915 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.095994128999999997 2024-09-13T14:59:45.915 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.915 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.915 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.915 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.950608+0000", 2024-09-13T14:59:45.915 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9567000000000002e-05 2024-09-13T14:59:45.915 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.916 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.916 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.916 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:34.950650+0000", 2024-09-13T14:59:45.916 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1396999999999997e-05 2024-09-13T14:59:45.916 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.916 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.916 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.916 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.916 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.916 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.46040.0:88714 9.0 9:01cfaf0b:::benchmark_data_smithi096_256909_object88713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T14:59:45.916 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:58:42.928397+0000", 2024-09-13T14:59:45.916 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 62.899939685, 2024-09-13T14:59:45.916 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.153194477, 2024-09-13T14:59:45.917 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.917 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.917 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.917 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.917 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.46040", 2024-09-13T14:59:45.917 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:59:45.917 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 88714 2024-09-13T14:59:45.917 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.917 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.917 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.917 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.917 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928397+0000", 2024-09-13T14:59:45.917 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.918 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.918 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.918 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.918 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928397+0000", 2024-09-13T14:59:45.918 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.918 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.918 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.918 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.918 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928398+0000", 2024-09-13T14:59:45.918 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.48e-06 2024-09-13T14:59:45.918 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.918 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.918 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.919 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928422+0000", 2024-09-13T14:59:45.919 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4003e-05 2024-09-13T14:59:45.919 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.919 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.919 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.919 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928437+0000", 2024-09-13T14:59:45.919 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4812e-05 2024-09-13T14:59:45.919 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.919 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.919 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.919 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928444+0000", 2024-09-13T14:59:45.919 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9190000000000002e-06 2024-09-13T14:59:45.919 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.919 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.920 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.920 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928501+0000", 2024-09-13T14:59:45.920 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7318999999999998e-05 2024-09-13T14:59:45.920 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.920 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.920 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.920 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928722+0000", 2024-09-13T14:59:45.920 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00022095799999999999 2024-09-13T14:59:45.920 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.920 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.920 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,6", 2024-09-13T14:59:45.920 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928853+0000", 2024-09-13T14:59:45.920 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013066600000000001 2024-09-13T14:59:45.921 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.921 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.921 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.921 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.932043+0000", 2024-09-13T14:59:45.921 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0031903499999999998 2024-09-13T14:59:45.921 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.922 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.922 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.922 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.936231+0000", 2024-09-13T14:59:45.922 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0041874169999999997 2024-09-13T14:59:45.922 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.922 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.922 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.922 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.081464+0000", 2024-09-13T14:59:45.922 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.14523307899999999 2024-09-13T14:59:45.922 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.922 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.922 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.922 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.081530+0000", 2024-09-13T14:59:45.923 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6342999999999998e-05 2024-09-13T14:59:45.923 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.923 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.923 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.923 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.081591+0000", 2024-09-13T14:59:45.923 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1130999999999995e-05 2024-09-13T14:59:45.923 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.923 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.923 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.923 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.923 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.923 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.46040.0:88731 9.0 9:088adfef:::benchmark_data_smithi096_256909_object88730:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T14:59:45.923 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:58:42.940783+0000", 2024-09-13T14:59:45.924 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 62.887552827999997, 2024-09-13T14:59:45.924 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.14892245500000001, 2024-09-13T14:59:45.924 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.924 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.924 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.924 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.924 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.46040", 2024-09-13T14:59:45.924 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:59:45.924 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 88731 2024-09-13T14:59:45.924 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.924 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.924 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.924 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.925 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.940783+0000", 2024-09-13T14:59:45.925 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.925 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.925 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.925 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.925 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.940783+0000", 2024-09-13T14:59:45.925 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.925 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.925 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.925 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.925 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.940785+0000", 2024-09-13T14:59:45.925 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5200000000000001e-06 2024-09-13T14:59:45.925 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.925 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.926 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.926 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.940805+0000", 2024-09-13T14:59:45.926 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0136000000000002e-05 2024-09-13T14:59:45.926 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.926 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.926 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.926 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.940821+0000", 2024-09-13T14:59:45.926 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6191000000000001e-05 2024-09-13T14:59:45.926 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.926 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.926 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.926 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.940829+0000", 2024-09-13T14:59:45.926 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2409999999999998e-06 2024-09-13T14:59:45.927 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.927 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.927 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.927 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.940887+0000", 2024-09-13T14:59:45.927 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7988000000000001e-05 2024-09-13T14:59:45.927 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.927 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.927 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.927 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.941056+0000", 2024-09-13T14:59:45.927 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00016950999999999999 2024-09-13T14:59:45.927 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.927 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.927 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,6", 2024-09-13T14:59:45.927 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.941230+0000", 2024-09-13T14:59:45.928 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00017420700000000001 2024-09-13T14:59:45.928 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.928 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.928 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.928 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.945863+0000", 2024-09-13T14:59:45.928 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.004632349 2024-09-13T14:59:45.928 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.928 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.928 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.928 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.946640+0000", 2024-09-13T14:59:45.928 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00077761300000000002 2024-09-13T14:59:45.928 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.928 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.929 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.929 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.089636+0000", 2024-09-13T14:59:45.929 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.142995493 2024-09-13T14:59:45.929 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.929 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.929 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.929 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.089675+0000", 2024-09-13T14:59:45.929 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9109000000000001e-05 2024-09-13T14:59:45.929 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.929 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.929 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.929 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.089706+0000", 2024-09-13T14:59:45.929 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.1097999999999999e-05 2024-09-13T14:59:45.929 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.930 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.930 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.930 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.930 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.930 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.46040.0:88759 9.0 9:034dd23c:::benchmark_data_smithi096_256909_object88758:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T14:59:45.930 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:58:42.965199+0000", 2024-09-13T14:59:45.930 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 62.863137287000001, 2024-09-13T14:59:45.930 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.12465232900000001, 2024-09-13T14:59:45.930 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.930 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.930 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.930 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.930 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.46040", 2024-09-13T14:59:45.930 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:59:45.931 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 88759 2024-09-13T14:59:45.931 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.931 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.931 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.931 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.931 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.965199+0000", 2024-09-13T14:59:45.931 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.931 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.931 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.931 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.931 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.965199+0000", 2024-09-13T14:59:45.931 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.932 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.932 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.932 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.932 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.965200+0000", 2024-09-13T14:59:45.933 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.3300000000000001e-07 2024-09-13T14:59:45.933 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.933 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.933 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.933 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.965220+0000", 2024-09-13T14:59:45.933 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.001e-05 2024-09-13T14:59:45.933 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.933 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.933 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.933 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.965229+0000", 2024-09-13T14:59:45.933 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9220000000000003e-06 2024-09-13T14:59:45.933 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.934 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.934 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.934 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.965235+0000", 2024-09-13T14:59:45.934 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.268e-06 2024-09-13T14:59:45.934 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.934 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.934 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.934 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.965292+0000", 2024-09-13T14:59:45.934 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6486000000000001e-05 2024-09-13T14:59:45.934 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.934 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.934 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.934 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.965436+0000", 2024-09-13T14:59:45.935 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00014486999999999999 2024-09-13T14:59:45.935 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.935 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.935 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,6", 2024-09-13T14:59:45.935 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.965591+0000", 2024-09-13T14:59:45.935 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000154912 2024-09-13T14:59:45.935 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.935 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.935 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.935 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.968591+0000", 2024-09-13T14:59:45.935 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0029993720000000001 2024-09-13T14:59:45.935 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.935 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.936 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.936 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.969545+0000", 2024-09-13T14:59:45.936 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000954359 2024-09-13T14:59:45.936 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.936 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.936 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.936 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.089762+0000", 2024-09-13T14:59:45.936 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.120217132 2024-09-13T14:59:45.936 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.936 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.936 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.936 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.089814+0000", 2024-09-13T14:59:45.936 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.1600000000000001e-05 2024-09-13T14:59:45.936 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.937 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.937 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.937 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.089851+0000", 2024-09-13T14:59:45.937 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7564999999999997e-05 2024-09-13T14:59:45.937 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.937 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.937 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.937 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.937 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.937 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.46040.0:88764 9.0 9:0ca8b834:::benchmark_data_smithi096_256909_object88763:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T14:59:45.937 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:58:42.970598+0000", 2024-09-13T14:59:45.937 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 62.857738771999998, 2024-09-13T14:59:45.938 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.119335656, 2024-09-13T14:59:45.938 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.938 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.938 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.938 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.938 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.46040", 2024-09-13T14:59:45.938 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:59:45.938 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 88764 2024-09-13T14:59:45.938 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.938 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.938 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.938 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.938 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.970598+0000", 2024-09-13T14:59:45.938 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.939 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.939 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.939 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.939 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.970598+0000", 2024-09-13T14:59:45.939 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.939 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.939 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.939 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.939 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.970599+0000", 2024-09-13T14:59:45.939 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.325e-06 2024-09-13T14:59:45.939 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.939 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.939 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.940 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.970620+0000", 2024-09-13T14:59:45.940 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0839000000000002e-05 2024-09-13T14:59:45.940 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.940 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.940 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.940 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.970635+0000", 2024-09-13T14:59:45.940 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5786000000000001e-05 2024-09-13T14:59:45.940 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.940 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.940 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.940 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.970643+0000", 2024-09-13T14:59:45.940 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1849999999999998e-06 2024-09-13T14:59:45.940 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.941 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.941 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.941 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.970689+0000", 2024-09-13T14:59:45.941 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6180000000000002e-05 2024-09-13T14:59:45.941 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.941 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.941 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.941 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.970811+0000", 2024-09-13T14:59:45.941 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000122552 2024-09-13T14:59:45.941 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.941 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.941 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,6", 2024-09-13T14:59:45.941 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.970950+0000", 2024-09-13T14:59:45.942 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000138121 2024-09-13T14:59:45.942 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.942 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.942 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.942 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.973637+0000", 2024-09-13T14:59:45.942 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0026871590000000002 2024-09-13T14:59:45.942 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.942 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.942 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.942 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.974277+0000", 2024-09-13T14:59:45.942 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00064053500000000002 2024-09-13T14:59:45.942 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.942 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.942 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.943 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.089884+0000", 2024-09-13T14:59:45.943 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.115606902 2024-09-13T14:59:45.943 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.943 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.943 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.943 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.089913+0000", 2024-09-13T14:59:45.943 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.8473999999999999e-05 2024-09-13T14:59:45.943 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.943 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.943 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.943 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.089933+0000", 2024-09-13T14:59:45.943 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0598000000000002e-05 2024-09-13T14:59:45.943 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.944 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.944 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.944 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.944 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.944 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.46040.0:113274 9.2 9:4b27e2aa:::benchmark_data_smithi096_256909_object113273:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-13T14:59:45.944 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:11.876453+0000", 2024-09-13T14:59:45.944 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 33.951883580999997, 2024-09-13T14:59:45.944 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.011522276, 2024-09-13T14:59:45.944 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.944 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.944 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "reached pg", 2024-09-13T14:59:45.944 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.944 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.46040", 2024-09-13T14:59:45.944 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:59:45.945 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 113274 2024-09-13T14:59:45.945 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.945 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.945 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.945 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.945 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.876453+0000", 2024-09-13T14:59:45.945 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.945 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.945 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.945 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.945 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.876453+0000", 2024-09-13T14:59:45.945 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.945 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.945 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.946 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.946 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.876453+0000", 2024-09-13T14:59:45.946 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1100000000000001e-07 2024-09-13T14:59:45.946 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.946 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.946 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.946 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.876470+0000", 2024-09-13T14:59:45.946 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7257000000000001e-05 2024-09-13T14:59:45.946 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.946 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.946 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.946 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.876478+0000", 2024-09-13T14:59:45.946 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3710000000000002e-06 2024-09-13T14:59:45.946 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.946 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.947 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.947 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.876481+0000", 2024-09-13T14:59:45.947 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6260000000000001e-06 2024-09-13T14:59:45.947 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.947 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.947 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.947 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.876545+0000", 2024-09-13T14:59:45.947 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3414e-05 2024-09-13T14:59:45.947 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.947 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.947 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.947 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.876730+0000", 2024-09-13T14:59:45.947 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00018541200000000001 2024-09-13T14:59:45.947 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.947 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.948 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T14:59:45.948 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.876911+0000", 2024-09-13T14:59:45.948 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00018118999999999999 2024-09-13T14:59:45.948 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.948 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.948 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.948 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.880995+0000", 2024-09-13T14:59:45.948 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0040831219999999998 2024-09-13T14:59:45.948 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.948 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.948 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.948 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.883235+0000", 2024-09-13T14:59:45.948 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0022404040000000001 2024-09-13T14:59:45.948 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.949 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.949 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:59:45.949 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.883248+0000", 2024-09-13T14:59:45.949 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2711000000000001e-05 2024-09-13T14:59:45.949 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.949 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.949 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.949 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.887365+0000", 2024-09-13T14:59:45.949 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.00411754 2024-09-13T14:59:45.949 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.949 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.949 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.949 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.887975+0000", 2024-09-13T14:59:45.949 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00060981800000000004 2024-09-13T14:59:45.950 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.950 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.950 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.950 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.950 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.950 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.46040.0:113276 9.2 9:4738da95:::benchmark_data_smithi096_256909_object113275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-13T14:59:45.950 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:11.882339+0000", 2024-09-13T14:59:45.950 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 33.945997716999997, 2024-09-13T14:59:45.950 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0157655720000001, 2024-09-13T14:59:45.950 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.950 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.950 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.950 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.951 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.46040", 2024-09-13T14:59:45.951 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:59:45.951 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 113276 2024-09-13T14:59:45.951 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.951 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.951 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.951 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.951 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.882339+0000", 2024-09-13T14:59:45.951 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.951 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.951 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.951 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.951 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.882339+0000", 2024-09-13T14:59:45.951 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.952 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.952 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.952 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.952 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.882340+0000", 2024-09-13T14:59:45.952 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.362e-06 2024-09-13T14:59:45.952 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.952 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.952 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.952 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.882366+0000", 2024-09-13T14:59:45.952 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6135999999999998e-05 2024-09-13T14:59:45.952 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.952 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.952 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.953 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.882377+0000", 2024-09-13T14:59:45.953 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0947999999999999e-05 2024-09-13T14:59:45.953 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.953 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.953 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.953 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.882386+0000", 2024-09-13T14:59:45.953 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8589999999999994e-06 2024-09-13T14:59:45.953 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.953 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.953 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.953 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.883827+0000", 2024-09-13T14:59:45.953 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001441283 2024-09-13T14:59:45.953 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.954 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.954 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:59:45.954 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.883835+0000", 2024-09-13T14:59:45.954 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.4490000000000002e-06 2024-09-13T14:59:45.954 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.954 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.954 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.954 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.888192+0000", 2024-09-13T14:59:45.954 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0043569619999999 2024-09-13T14:59:45.954 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.954 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.954 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.954 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.888378+0000", 2024-09-13T14:59:45.954 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00018617500000000001 2024-09-13T14:59:45.954 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.955 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.955 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:59:45.955 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.888586+0000", 2024-09-13T14:59:45.955 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000208552 2024-09-13T14:59:45.955 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.955 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.955 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.955 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.897854+0000", 2024-09-13T14:59:45.955 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0092673860000000007 2024-09-13T14:59:45.955 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.955 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.955 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.955 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.897871+0000", 2024-09-13T14:59:45.956 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7082e-05 2024-09-13T14:59:45.956 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.956 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.956 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.956 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.897912+0000", 2024-09-13T14:59:45.956 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0978e-05 2024-09-13T14:59:45.956 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.956 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.956 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.956 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.897990+0000", 2024-09-13T14:59:45.956 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.7955999999999997e-05 2024-09-13T14:59:45.956 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.956 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.957 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.957 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.898104+0000", 2024-09-13T14:59:45.957 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000114444 2024-09-13T14:59:45.957 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.957 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.957 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.957 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.957 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.957 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.46040.0:113291 9.2 9:41c0aeb3:::benchmark_data_smithi096_256909_object113290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-09-13T14:59:45.957 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:11.910433+0000", 2024-09-13T14:59:45.957 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 33.917902922000003, 2024-09-13T14:59:45.957 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.99143278700000004, 2024-09-13T14:59:45.957 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.958 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.958 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.958 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.958 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.46040", 2024-09-13T14:59:45.958 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:59:45.958 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 113291 2024-09-13T14:59:45.958 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.958 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.958 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.958 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.959 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.910433+0000", 2024-09-13T14:59:45.959 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.959 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.959 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.959 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.959 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.910433+0000", 2024-09-13T14:59:45.959 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.959 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.959 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.959 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.959 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.910435+0000", 2024-09-13T14:59:45.959 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0099999999999998e-06 2024-09-13T14:59:45.960 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.960 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.960 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.960 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.910463+0000", 2024-09-13T14:59:45.960 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7192e-05 2024-09-13T14:59:45.960 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.960 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.960 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.960 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.910481+0000", 2024-09-13T14:59:45.960 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8425999999999999e-05 2024-09-13T14:59:45.960 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.961 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.961 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.961 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.910491+0000", 2024-09-13T14:59:45.961 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7999999999999993e-06 2024-09-13T14:59:45.961 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.961 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.961 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.961 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.910548+0000", 2024-09-13T14:59:45.961 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7117999999999998e-05 2024-09-13T14:59:45.961 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.961 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.961 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:59:45.961 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.910566+0000", 2024-09-13T14:59:45.961 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8224999999999999e-05 2024-09-13T14:59:45.962 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.962 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.962 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.962 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.891661+0000", 2024-09-13T14:59:45.962 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.98109449199999998 2024-09-13T14:59:45.962 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.962 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.962 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.962 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.891838+0000", 2024-09-13T14:59:45.962 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000177305 2024-09-13T14:59:45.962 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.962 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.962 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:59:45.963 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.892041+0000", 2024-09-13T14:59:45.963 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00020351000000000001 2024-09-13T14:59:45.963 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.963 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.963 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.963 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.901575+0000", 2024-09-13T14:59:45.963 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.009533293 2024-09-13T14:59:45.963 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.963 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.963 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.963 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.901640+0000", 2024-09-13T14:59:45.963 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5269000000000001e-05 2024-09-13T14:59:45.963 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.963 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.964 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.964 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.901704+0000", 2024-09-13T14:59:45.964 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4220999999999994e-05 2024-09-13T14:59:45.964 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.964 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.964 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.964 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.901764+0000", 2024-09-13T14:59:45.964 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9911999999999998e-05 2024-09-13T14:59:45.964 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.964 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.964 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.964 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.901866+0000", 2024-09-13T14:59:45.964 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000102014 2024-09-13T14:59:45.964 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.965 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.965 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.965 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.965 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.965 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.46040.0:113316 9.2 9:43e7c39e:::benchmark_data_smithi096_256909_object113315:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-09-13T14:59:45.965 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:11.955742+0000", 2024-09-13T14:59:45.965 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 33.872593991000002, 2024-09-13T14:59:45.965 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.94636433600000003, 2024-09-13T14:59:45.965 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.965 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.965 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.965 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.965 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.46040", 2024-09-13T14:59:45.965 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:59:45.966 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 113316 2024-09-13T14:59:45.966 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.966 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.966 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.966 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.966 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.955742+0000", 2024-09-13T14:59:45.966 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.966 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.966 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.966 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.966 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.955742+0000", 2024-09-13T14:59:45.966 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.966 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.966 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.967 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.967 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.955744+0000", 2024-09-13T14:59:45.967 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.443e-06 2024-09-13T14:59:45.967 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.967 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.967 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.967 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.955793+0000", 2024-09-13T14:59:45.967 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9543000000000003e-05 2024-09-13T14:59:45.967 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.967 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.967 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.967 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.955808+0000", 2024-09-13T14:59:45.967 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4905999999999999e-05 2024-09-13T14:59:45.967 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.968 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.968 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.968 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.955815+0000", 2024-09-13T14:59:45.968 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.097e-06 2024-09-13T14:59:45.968 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.968 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.968 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.968 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.955879+0000", 2024-09-13T14:59:45.968 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3307999999999998e-05 2024-09-13T14:59:45.968 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.968 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.968 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:59:45.968 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.955887+0000", 2024-09-13T14:59:45.968 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5830000000000007e-06 2024-09-13T14:59:45.969 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.969 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.969 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.969 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.895107+0000", 2024-09-13T14:59:45.969 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.93922030199999995 2024-09-13T14:59:45.969 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.969 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.969 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.969 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.895222+0000", 2024-09-13T14:59:45.969 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000114844 2024-09-13T14:59:45.969 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.969 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.970 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:59:45.970 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.895331+0000", 2024-09-13T14:59:45.970 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000108693 2024-09-13T14:59:45.970 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.970 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.970 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.970 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.901880+0000", 2024-09-13T14:59:45.970 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0065485200000000004 2024-09-13T14:59:45.970 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.970 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.970 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.970 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.901925+0000", 2024-09-13T14:59:45.970 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.5306000000000003e-05 2024-09-13T14:59:45.971 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.971 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.971 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.971 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.901968+0000", 2024-09-13T14:59:45.971 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3572999999999998e-05 2024-09-13T14:59:45.971 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.971 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.971 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.971 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.902004+0000", 2024-09-13T14:59:45.971 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.5790000000000001e-05 2024-09-13T14:59:45.971 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.971 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.971 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.971 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.902107+0000", 2024-09-13T14:59:45.972 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000102428 2024-09-13T14:59:45.972 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.972 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.972 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.972 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.972 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.972 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.46040.0:113327 9.2 9:45dc0854:::benchmark_data_smithi096_256909_object113326:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-09-13T14:59:45.972 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:11.982166+0000", 2024-09-13T14:59:45.972 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 33.846170014000002, 2024-09-13T14:59:45.972 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.92015086899999998, 2024-09-13T14:59:45.972 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T14:59:45.972 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T14:59:45.972 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:45.973 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T14:59:45.973 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.46040", 2024-09-13T14:59:45.973 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T14:59:45.973 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 113327 2024-09-13T14:59:45.973 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.973 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T14:59:45.973 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.973 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T14:59:45.973 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.982166+0000", 2024-09-13T14:59:45.973 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.973 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.973 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.973 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T14:59:45.973 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.982166+0000", 2024-09-13T14:59:45.974 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T14:59:45.974 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.974 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.974 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T14:59:45.974 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.982168+0000", 2024-09-13T14:59:45.974 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.626e-06 2024-09-13T14:59:45.974 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.974 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.974 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T14:59:45.974 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.982190+0000", 2024-09-13T14:59:45.974 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1903999999999999e-05 2024-09-13T14:59:45.974 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.974 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.974 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T14:59:45.975 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.982206+0000", 2024-09-13T14:59:45.975 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5801e-05 2024-09-13T14:59:45.975 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.975 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.975 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T14:59:45.975 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.982212+0000", 2024-09-13T14:59:45.975 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.866e-06 2024-09-13T14:59:45.975 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.975 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.975 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.975 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.982279+0000", 2024-09-13T14:59:45.975 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.6119999999999997e-05 2024-09-13T14:59:45.975 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.976 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.976 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for peered", 2024-09-13T14:59:45.976 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:11.982290+0000", 2024-09-13T14:59:45.976 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0975000000000001e-05 2024-09-13T14:59:45.976 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.976 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.976 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T14:59:45.976 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.898153+0000", 2024-09-13T14:59:45.976 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.915863553 2024-09-13T14:59:45.976 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.976 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.976 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T14:59:45.976 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.898327+0000", 2024-09-13T14:59:45.977 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000173492 2024-09-13T14:59:45.977 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.977 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.977 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:59:45.977 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.898497+0000", 2024-09-13T14:59:45.977 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00017003999999999999 2024-09-13T14:59:45.977 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.977 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.977 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "op_commit", 2024-09-13T14:59:45.977 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.901937+0000", 2024-09-13T14:59:45.977 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0034404040000000002 2024-09-13T14:59:45.977 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.977 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.977 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.978 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.902159+0000", 2024-09-13T14:59:45.978 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000222387 2024-09-13T14:59:45.978 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.978 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.978 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:45.978 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.902196+0000", 2024-09-13T14:59:45.978 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6727999999999998e-05 2024-09-13T14:59:45.978 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.978 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.978 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T14:59:45.978 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.902234+0000", 2024-09-13T14:59:45.978 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7719999999999998e-05 2024-09-13T14:59:45.978 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T14:59:45.979 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T14:59:45.979 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T14:59:45.979 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:12.902317+0000", 2024-09-13T14:59:45.979 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.3252999999999997e-05 2024-09-13T14:59:45.979 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.979 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.979 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.979 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T14:59:45.979 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T14:59:45.979 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T14:59:45.980 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:45.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:46.081 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:46.113 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:46.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:46.214 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:46.247 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:46.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:46.349 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:46.393 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:46.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:46.495 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:46.532 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:46.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:46.634 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:46.658 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:46.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:46.760 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:46.793 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:46.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:46.895 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:46.939 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:46.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:47.041 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:47.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.081 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:47.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:47.182 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:47.223 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:47.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:47.314 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:59:47.325 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:47.346 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:47.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:47.448 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:47.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:47 smithi204 ceph-mon[87071]: pgmap v1058: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 825 op/s; 45637/305553 objects degraded (14.936%); 8.9 MiB/s, 142 objects/s recovering 2024-09-13T14:59:47.479 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:47.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:47.493 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:47 smithi096 ceph-mon[98901]: pgmap v1058: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 825 op/s; 45637/305553 objects degraded (14.936%); 8.9 MiB/s, 142 objects/s recovering 2024-09-13T14:59:47.494 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:47 smithi096 ceph-mon[102928]: pgmap v1058: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 825 op/s; 45637/305553 objects degraded (14.936%); 8.9 MiB/s, 142 objects/s recovering 2024-09-13T14:59:47.581 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:47.620 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:47.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:47.721 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:47.749 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:47.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:47.851 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:47.883 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:47.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:47.985 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:48.016 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:48.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:48.118 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:48.154 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:48.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:48.256 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:48.293 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:48.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:48.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:59:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:59:47] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.51.0" 2024-09-13T14:59:48.395 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:48.429 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:48.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:48.531 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:48.570 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:48.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:48.672 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:48.724 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:59:48.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.741 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:48.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:48.842 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:48.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.876 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:48.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:48.977 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:49.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.018 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:49.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:49.119 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:49.159 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:49.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:49.261 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:49.301 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:49.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:49.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:49 smithi096 ceph-mon[98901]: pgmap v1059: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 824 op/s; 44664/298497 objects degraded (14.963%); 8.7 MiB/s, 139 objects/s recovering 2024-09-13T14:59:49.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:49 smithi096 ceph-mon[102928]: pgmap v1059: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 824 op/s; 44664/298497 objects degraded (14.963%); 8.7 MiB/s, 139 objects/s recovering 2024-09-13T14:59:49.402 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:49.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.439 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:49.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:49.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:49 smithi204 ceph-mon[87071]: pgmap v1059: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 824 op/s; 44664/298497 objects degraded (14.963%); 8.7 MiB/s, 139 objects/s recovering 2024-09-13T14:59:49.539 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:49.586 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:49.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:49.688 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:49.725 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:49.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:49.827 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:49.855 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:49.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:49.957 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:49.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.988 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:49.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:50.089 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:50.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.125 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:50.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:50.226 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:50.253 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:50.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:50.355 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:50.379 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:50.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:50.407 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:50 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 44664/298497 objects degraded (14.963%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:50.408 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:50 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 44664/298497 objects degraded (14.963%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:50.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:50 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 44664/298497 objects degraded (14.963%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 14:59:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T14:59:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:50.482 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:50.504 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:50.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:50.606 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:50.631 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:50.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:50.733 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:50.757 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:50.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:50.858 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:50.882 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:50.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:50.984 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:51.008 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:51.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:51.069 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:51.094 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:51.110 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:51.132 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:51.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:51.182 INFO:teuthology.orchestra.run.smithi096.stderr:set pool 1 pg_num to 31 2024-09-13T14:59:51.231 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:51 smithi096 ceph-mon[98901]: pgmap v1060: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 805 op/s; 44167/293301 objects degraded (15.059%); 8.5 MiB/s, 136 objects/s recovering 2024-09-13T14:59:51.231 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:51 smithi096 ceph-mon[98901]: osdmap e323: 8 total, 7 up, 7 in 2024-09-13T14:59:51.231 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:51.231 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:51.231 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:51 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1852535649' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-13T14:59:51.231 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:51 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-13T14:59:51.231 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:51 smithi096 ceph-mon[102928]: pgmap v1060: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 805 op/s; 44167/293301 objects degraded (15.059%); 8.5 MiB/s, 136 objects/s recovering 2024-09-13T14:59:51.232 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:51 smithi096 ceph-mon[102928]: osdmap e323: 8 total, 7 up, 7 in 2024-09-13T14:59:51.232 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:51.232 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:51.232 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:51 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1852535649' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-13T14:59:51.232 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:51 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-13T14:59:51.234 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:51.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.257 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:51.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:51.286 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:51 smithi204 ceph-mon[87071]: pgmap v1060: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 805 op/s; 44167/293301 objects degraded (15.059%); 8.5 MiB/s, 136 objects/s recovering 2024-09-13T14:59:51.287 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:51 smithi204 ceph-mon[87071]: osdmap e323: 8 total, 7 up, 7 in 2024-09-13T14:59:51.287 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T14:59:51.287 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:51.287 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:51 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1852535649' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-13T14:59:51.287 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:51 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-13T14:59:51.358 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:51.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.379 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:51.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:51.480 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:51.501 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:51.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:51.603 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:51.627 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:51.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:51.729 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:51.765 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:51.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:51.867 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:51.892 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:51.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:51.993 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:52.032 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:52.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:52.134 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:52.167 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:52.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:52.203 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T14:59:52.269 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:52.298 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:52.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:52.400 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:52.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.429 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:52.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:52 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-09-13T14:59:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:52 smithi204 ceph-mon[87071]: osdmap e324: 8 total, 7 up, 7 in 2024-09-13T14:59:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:52 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/517722487' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:52 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:52.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:52 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-13T14:59:52.531 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:52.551 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:52 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-09-13T14:59:52.552 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:52 smithi096 ceph-mon[98901]: osdmap e324: 8 total, 7 up, 7 in 2024-09-13T14:59:52.552 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:52 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/517722487' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:52.552 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:52 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:52.552 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:52 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-13T14:59:52.552 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:52 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-09-13T14:59:52.552 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:52 smithi096 ceph-mon[102928]: osdmap e324: 8 total, 7 up, 7 in 2024-09-13T14:59:52.552 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:52 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/517722487' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:52.552 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:52 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:52.553 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:52 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-13T14:59:52.559 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:52.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:52.661 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:52.694 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:52.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:52.796 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:52.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.831 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:52.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:52.932 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:52.962 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:52.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:53.064 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:53.101 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:53.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:53.203 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:53.233 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:53.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:53.335 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:53.366 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:53.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:53.468 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:53 smithi204 ceph-mon[87071]: pgmap v1063: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 669 op/s; 43991/292740 objects degraded (15.027%); 6.6 MiB/s, 105 objects/s recovering 2024-09-13T14:59:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:53 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:53 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:53 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-09-13T14:59:53.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:53 smithi204 ceph-mon[87071]: osdmap e325: 8 total, 7 up, 7 in 2024-09-13T14:59:53.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.501 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:53.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:53 smithi096 ceph-mon[98901]: pgmap v1063: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 669 op/s; 43991/292740 objects degraded (15.027%); 6.6 MiB/s, 105 objects/s recovering 2024-09-13T14:59:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:53 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:53 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:53 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-09-13T14:59:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:53 smithi096 ceph-mon[98901]: osdmap e325: 8 total, 7 up, 7 in 2024-09-13T14:59:53.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:53 smithi096 ceph-mon[102928]: pgmap v1063: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 669 op/s; 43991/292740 objects degraded (15.027%); 6.6 MiB/s, 105 objects/s recovering 2024-09-13T14:59:53.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:53 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:53 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:53 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-09-13T14:59:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:53 smithi096 ceph-mon[102928]: osdmap e325: 8 total, 7 up, 7 in 2024-09-13T14:59:53.602 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:53.637 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:53.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:53.739 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:53.780 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:53.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:53.882 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:53.910 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:53.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:54.012 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:54.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.049 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:54.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:54.150 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:54.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.182 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:54.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:54.283 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:54.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.314 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:54.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:54.415 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:54.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.462 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:54.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:54 smithi204 ceph-mon[87071]: osdmap e326: 8 total, 7 up, 7 in 2024-09-13T14:59:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:54 smithi204 ceph-mon[87071]: 1.c scrub starts 2024-09-13T14:59:54.563 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:54.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:54 smithi096 ceph-mon[98901]: osdmap e326: 8 total, 7 up, 7 in 2024-09-13T14:59:54.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:54 smithi096 ceph-mon[98901]: 1.c scrub starts 2024-09-13T14:59:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:54 smithi096 ceph-mon[102928]: osdmap e326: 8 total, 7 up, 7 in 2024-09-13T14:59:54.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:54 smithi096 ceph-mon[102928]: 1.c scrub starts 2024-09-13T14:59:54.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.596 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:54.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:54.697 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:54.727 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:54.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:54.829 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:54.856 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:54.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:54.960 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:54.981 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:54.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:55.083 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:55.121 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:55.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:55.223 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:55.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.253 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:55.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:55.355 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:55.384 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:55.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:55.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:55 smithi204 ceph-mon[87071]: 1.b scrub starts 2024-09-13T14:59:55.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:55 smithi204 ceph-mon[87071]: pgmap v1066: 119 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 9 unknown, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 43132/286209 objects degraded (15.070%); 9.6 MiB/s, 154 objects/s recovering 2024-09-13T14:59:55.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:55 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 43132/286209 objects degraded (15.070%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:55.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:55 smithi204 ceph-mon[87071]: 1.1c scrub starts 2024-09-13T14:59:55.486 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:55.519 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:55.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:55.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:55 smithi096 ceph-mon[98901]: 1.b scrub starts 2024-09-13T14:59:55.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:55 smithi096 ceph-mon[98901]: pgmap v1066: 119 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 9 unknown, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 43132/286209 objects degraded (15.070%); 9.6 MiB/s, 154 objects/s recovering 2024-09-13T14:59:55.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:55 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 43132/286209 objects degraded (15.070%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:55.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:55 smithi096 ceph-mon[98901]: 1.1c scrub starts 2024-09-13T14:59:55.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:55 smithi096 ceph-mon[102928]: 1.b scrub starts 2024-09-13T14:59:55.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:55 smithi096 ceph-mon[102928]: pgmap v1066: 119 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 9 unknown, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 43132/286209 objects degraded (15.070%); 9.6 MiB/s, 154 objects/s recovering 2024-09-13T14:59:55.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:55 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 43132/286209 objects degraded (15.070%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:55.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:55 smithi096 ceph-mon[102928]: 1.1c scrub starts 2024-09-13T14:59:55.621 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:55.645 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:55.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:55.747 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:55.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.775 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:55.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:55.876 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:55.927 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:55.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:56.029 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:56.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.072 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:56.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:56.173 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:56.214 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:56.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:56.316 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:56.345 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:56.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:56.447 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:56.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.476 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:56.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:56.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:56 smithi096 ceph-mon[98901]: 1.1b scrub starts 2024-09-13T14:59:56.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:56 smithi096 ceph-mon[102928]: 1.1b scrub starts 2024-09-13T14:59:56.577 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:56.609 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:56.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:56.712 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:56.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:56 smithi204 ceph-mon[87071]: 1.1b scrub starts 2024-09-13T14:59:56.754 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:56.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:56.856 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:56.888 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:56.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:56.991 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:57.036 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:57.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:57.138 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:57.181 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:57.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:57.283 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:57.322 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:57.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:57.425 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:57.459 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:57.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:57.561 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:57.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:57 smithi096 ceph-mon[98901]: pgmap v1067: 119 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 8 unknown, 3 active+undersized+degraded+remapped+backfill_wait, 2 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 42946/283722 objects degraded (15.137%); 6.5 MiB/s, 104 objects/s recovering 2024-09-13T14:59:57.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:57 smithi096 ceph-mon[102928]: pgmap v1067: 119 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 8 unknown, 3 active+undersized+degraded+remapped+backfill_wait, 2 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 42946/283722 objects degraded (15.137%); 6.5 MiB/s, 104 objects/s recovering 2024-09-13T14:59:57.594 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 6] out_osds: [3] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2024-09-13T14:59:57.594 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:59:57.594 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-09-13T14:59:57.594 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:59:57.594 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:59:57.595 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-09-13T14:59:57.602 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:57.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:57.640 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:57 smithi204 ceph-mon[87071]: pgmap v1067: 119 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 8 unknown, 3 active+undersized+degraded+remapped+backfill_wait, 2 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 42946/283722 objects degraded (15.137%); 6.5 MiB/s, 104 objects/s recovering 2024-09-13T14:59:57.705 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:57.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.749 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:57.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:57.851 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:57.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.888 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:57.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:57.989 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:58.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.033 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:58.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:58.072 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:59:58.134 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:58.162 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:58.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:58.263 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:58.294 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 14:59:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:14:59:57] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.51.0" 2024-09-13T14:59:58.295 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:58.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:58.397 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:58.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.428 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:58.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:58.529 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:58.557 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:58.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:58.659 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:58.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.690 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:58.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:58.791 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:58.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.841 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:58.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:58.942 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:58.992 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:58.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:59.094 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:59.132 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:59.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:59.234 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:59.266 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:59.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:59.368 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:59.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.400 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:59.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:59.501 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:59.550 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:59.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:59.552 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 14:59:59 smithi204 ceph-mon[87071]: pgmap v1068: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 2 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 42534/276366 objects degraded (15.390%); 6.7 MiB/s, 107 objects/s recovering 2024-09-13T14:59:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 14:59:59 smithi096 ceph-mon[98901]: pgmap v1068: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 2 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 42534/276366 objects degraded (15.390%); 6.7 MiB/s, 107 objects/s recovering 2024-09-13T14:59:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 14:59:59 smithi096 ceph-mon[102928]: pgmap v1068: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 2 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 42534/276366 objects degraded (15.390%); 6.7 MiB/s, 107 objects/s recovering 2024-09-13T14:59:59.653 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:59.687 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:59.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:59.789 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:59.826 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T14:59:59.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:59.928 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:59.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.970 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T14:59:59.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:00.071 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:00.103 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:00.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:00.205 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:00.241 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:00.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:00.251 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T15:00:00.251 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T15:00:00.251 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:00.251 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:150573 9.f e326/316 9:fd8c1ff7:::benchmark_data_smithi096_256909_object31165:head v 326'9329, mlcod=0'0)", 2024-09-13T15:00:00.251 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:00:00.216580+0000", 2024-09-13T15:00:00.251 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0074047000000000002, 2024-09-13T15:00:00.252 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0074326540000000003, 2024-09-13T15:00:00.252 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:00.252 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:00.252 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T15:00:00.252 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:00.252 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:00.252 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:00.252 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.216580+0000", 2024-09-13T15:00:00.252 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:00.252 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:00.252 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:00.252 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:00.252 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.216580+0000", 2024-09-13T15:00:00.253 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:00.253 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:00.253 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:00.253 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:00.253 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.216581+0000", 2024-09-13T15:00:00.253 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5200000000000001e-06 2024-09-13T15:00:00.253 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:00.253 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:00.253 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:00.253 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.216589+0000", 2024-09-13T15:00:00.253 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.7109999999999995e-06 2024-09-13T15:00:00.253 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:00.254 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:00.254 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:00.254 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.216607+0000", 2024-09-13T15:00:00.254 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8525e-05 2024-09-13T15:00:00.254 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:00.254 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:00.254 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:00.254 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.216688+0000", 2024-09-13T15:00:00.254 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0859000000000006e-05 2024-09-13T15:00:00.254 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:00.254 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:00.255 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:00.255 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.221958+0000", 2024-09-13T15:00:00.255 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0052694509999999996 2024-09-13T15:00:00.255 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:00.255 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:00.255 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:00.255 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.221989+0000", 2024-09-13T15:00:00.255 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.1266999999999997e-05 2024-09-13T15:00:00.255 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:00.255 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:00.255 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:00.255 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:00.255 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2024-09-13T15:00:00.256 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 1 2024-09-13T15:00:00.256 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T15:00:00.323 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:00:00.343 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:00.353 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[98901]: 1.b scrub starts 2024-09-13T15:00:00.353 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 42534/276366 objects degraded (15.390%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T15:00:00.353 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[98901]: 1.c scrub starts 2024-09-13T15:00:00.353 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[98901]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 42534/276366 objects degraded (15.390%), 7 pgs degraded 2024-09-13T15:00:00.353 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[98901]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T15:00:00.353 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[98901]: [WRN] PG_DEGRADED: Degraded data redundancy: 42534/276366 objects degraded (15.390%), 7 pgs degraded 2024-09-13T15:00:00.353 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[98901]: pg 9.1 is active+recovering+undersized+degraded+remapped, acting [6,7] 2024-09-13T15:00:00.353 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[98901]: pg 9.2 is active+undersized+degraded+remapped+backfill_wait, acting [1,2] 2024-09-13T15:00:00.353 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[98901]: pg 9.4 is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2024-09-13T15:00:00.353 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[98901]: pg 9.7 is active+recovery_wait+undersized+degraded+remapped, acting [5,7] 2024-09-13T15:00:00.354 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[98901]: pg 9.8 is active+undersized+degraded+remapped+backfill_wait, acting [0,5] 2024-09-13T15:00:00.354 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[98901]: pg 9.c is active+undersized+degraded+remapped+backfill_wait, acting [4,7] 2024-09-13T15:00:00.354 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[98901]: pg 9.f is active+recovery_wait+undersized+degraded+remapped, acting [5,1] 2024-09-13T15:00:00.355 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[102928]: 1.b scrub starts 2024-09-13T15:00:00.355 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 42534/276366 objects degraded (15.390%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T15:00:00.355 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[102928]: 1.c scrub starts 2024-09-13T15:00:00.355 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[102928]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 42534/276366 objects degraded (15.390%), 7 pgs degraded 2024-09-13T15:00:00.356 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[102928]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T15:00:00.356 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[102928]: [WRN] PG_DEGRADED: Degraded data redundancy: 42534/276366 objects degraded (15.390%), 7 pgs degraded 2024-09-13T15:00:00.356 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[102928]: pg 9.1 is active+recovering+undersized+degraded+remapped, acting [6,7] 2024-09-13T15:00:00.356 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[102928]: pg 9.2 is active+undersized+degraded+remapped+backfill_wait, acting [1,2] 2024-09-13T15:00:00.356 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[102928]: pg 9.4 is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2024-09-13T15:00:00.356 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[102928]: pg 9.7 is active+recovery_wait+undersized+degraded+remapped, acting [5,7] 2024-09-13T15:00:00.356 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[102928]: pg 9.8 is active+undersized+degraded+remapped+backfill_wait, acting [0,5] 2024-09-13T15:00:00.356 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[102928]: pg 9.c is active+undersized+degraded+remapped+backfill_wait, acting [4,7] 2024-09-13T15:00:00.356 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:00 smithi096 ceph-mon[102928]: pg 9.f is active+recovery_wait+undersized+degraded+remapped, acting [5,1] 2024-09-13T15:00:00.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.391 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:00.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:00.452 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:00:00.452 INFO:teuthology.orchestra.run.smithi204.stdout: "success": "" 2024-09-13T15:00:00.452 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:00:00.467 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:00 smithi204 ceph-mon[87071]: 1.b scrub starts 2024-09-13T15:00:00.467 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:00 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 42534/276366 objects degraded (15.390%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T15:00:00.467 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:00 smithi204 ceph-mon[87071]: 1.c scrub starts 2024-09-13T15:00:00.467 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:00 smithi204 ceph-mon[87071]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 42534/276366 objects degraded (15.390%), 7 pgs degraded 2024-09-13T15:00:00.467 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:00 smithi204 ceph-mon[87071]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T15:00:00.467 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:00 smithi204 ceph-mon[87071]: [WRN] PG_DEGRADED: Degraded data redundancy: 42534/276366 objects degraded (15.390%), 7 pgs degraded 2024-09-13T15:00:00.467 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:00 smithi204 ceph-mon[87071]: pg 9.1 is active+recovering+undersized+degraded+remapped, acting [6,7] 2024-09-13T15:00:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:00 smithi204 ceph-mon[87071]: pg 9.2 is active+undersized+degraded+remapped+backfill_wait, acting [1,2] 2024-09-13T15:00:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:00 smithi204 ceph-mon[87071]: pg 9.4 is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2024-09-13T15:00:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:00 smithi204 ceph-mon[87071]: pg 9.7 is active+recovery_wait+undersized+degraded+remapped, acting [5,7] 2024-09-13T15:00:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:00 smithi204 ceph-mon[87071]: pg 9.8 is active+undersized+degraded+remapped+backfill_wait, acting [0,5] 2024-09-13T15:00:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:00 smithi204 ceph-mon[87071]: pg 9.c is active+undersized+degraded+remapped+backfill_wait, acting [4,7] 2024-09-13T15:00:00.468 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:00 smithi204 ceph-mon[87071]: pg 9.f is active+recovery_wait+undersized+degraded+remapped, acting [5,1] 2024-09-13T15:00:00.468 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:00:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:00:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:00.493 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:00.534 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:00.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:00.637 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:00.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.678 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:00.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:00.779 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:00.789 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:00:00.789 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:00:00.814 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:00.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:00.824 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:00:00.824 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:00:00.847 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:00:00.848 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:00:00.848 INFO:teuthology.orchestra.run.smithi096.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:00:00.881 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:00:00.882 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:00:00.915 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:00.920 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:00:00.920 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:00:00.948 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:00:00.949 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:00:00.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.958 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:00.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:00.978 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:00:00.978 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:00:01.059 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:01.092 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:01.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:01.194 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:01.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.222 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:01.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:01.322 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:01.355 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:01.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:01.386 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T15:00:01.457 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:01.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.506 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:01.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:01 smithi096 ceph-mon[98901]: 1.1b scrub starts 2024-09-13T15:00:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:01 smithi096 ceph-mon[98901]: pgmap v1069: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 923 op/s; 41983/270567 objects degraded (15.517%); 4.5 MiB/s, 71 objects/s recovering 2024-09-13T15:00:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:01 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4175959250' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:01.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:01 smithi096 ceph-mon[98901]: 1.1c scrub starts 2024-09-13T15:00:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:01 smithi096 ceph-mon[102928]: 1.1b scrub starts 2024-09-13T15:00:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:01 smithi096 ceph-mon[102928]: pgmap v1069: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 923 op/s; 41983/270567 objects degraded (15.517%); 4.5 MiB/s, 71 objects/s recovering 2024-09-13T15:00:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:01 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4175959250' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:01 smithi096 ceph-mon[102928]: 1.1c scrub starts 2024-09-13T15:00:01.607 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:01.630 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:01.639 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:01.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:01 smithi204 ceph-mon[87071]: 1.1b scrub starts 2024-09-13T15:00:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:01 smithi204 ceph-mon[87071]: pgmap v1069: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 923 op/s; 41983/270567 objects degraded (15.517%); 4.5 MiB/s, 71 objects/s recovering 2024-09-13T15:00:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:01 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4175959250' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:01 smithi204 ceph-mon[87071]: 1.1c scrub starts 2024-09-13T15:00:01.741 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:01.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.774 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:01.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:01.876 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:01.922 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:01.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:02.024 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:02.058 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:02.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:02.153 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:00:02.153 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:00:02.160 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:02.196 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:02.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:02.298 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:02.325 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:02.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:02.427 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:02.458 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:02.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:02.560 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:02.591 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:02.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:02.693 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:02.725 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:02.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:02.827 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:02.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.866 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:02.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:02.964 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:02.967 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:02.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.999 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:02.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:03.100 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:03.141 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:03.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:03.244 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:03.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.278 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:03.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:03.379 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:03.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.417 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:03.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:03.518 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:03.525 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:03 smithi096 ceph-mon[98901]: pgmap v1070: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 41855/269793 objects degraded (15.514%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:00:03.526 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:03 smithi096 ceph-mon[102928]: pgmap v1070: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 41855/269793 objects degraded (15.514%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:00:03.555 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:03.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:03.657 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:03.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.693 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:03.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:03 smithi204 ceph-mon[87071]: pgmap v1070: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 41855/269793 objects degraded (15.514%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:00:03.794 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:03.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.839 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:03.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:03.940 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:03.971 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:03.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:04.073 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:04.109 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:04.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:04.154 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:00:04.211 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:04.242 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:04.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:04.345 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:04.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.381 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:04.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:04.482 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:04.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.516 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:04.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:04.617 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:04.651 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:04.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:04.753 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:04.789 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:04.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:04.891 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:04.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.934 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:04.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:05.035 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:05.064 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:05.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:05.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:05.641 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T15:00:05.641 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:05 smithi096 ceph-mon[98901]: pgmap v1071: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 833 op/s; 41294/260712 objects degraded (15.839%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T15:00:05.641 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:05 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 41294/260712 objects degraded (15.839%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:05.641 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:05 smithi096 ceph-mon[98901]: 1.c scrub starts 2024-09-13T15:00:05.641 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:05 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1053967815' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:05.641 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:05 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:05.642 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:05 smithi096 ceph-mon[102928]: pgmap v1071: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 833 op/s; 41294/260712 objects degraded (15.839%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T15:00:05.642 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:05 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 41294/260712 objects degraded (15.839%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:05.642 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:05 smithi096 ceph-mon[102928]: 1.c scrub starts 2024-09-13T15:00:05.642 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:05 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1053967815' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:05.642 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:05 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:05 smithi204 ceph-mon[87071]: pgmap v1071: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 833 op/s; 41294/260712 objects degraded (15.839%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T15:00:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:05 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 41294/260712 objects degraded (15.839%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:05 smithi204 ceph-mon[87071]: 1.c scrub starts 2024-09-13T15:00:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:05 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1053967815' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:05 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:05.742 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:05.767 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:05.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:05.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:05.870 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:05.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:05.909 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:05.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:06.010 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:06.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.042 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:06.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:06.144 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:06.147 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:06.182 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:06.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:06.284 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:06.322 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 6] out_osds: [3] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2024-09-13T15:00:06.322 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:00:06.323 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-09-13T15:00:06.323 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T15:00:06.323 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T15:00:06.323 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-09-13T15:00:06.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:06 smithi096 ceph-mon[98901]: 1.b scrub starts 2024-09-13T15:00:06.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:06 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:06.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:06 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:00:06.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:06 smithi096 ceph-mon[98901]: osdmap e327: 8 total, 7 up, 7 in 2024-09-13T15:00:06.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:06.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:06 smithi096 ceph-mon[102928]: 1.b scrub starts 2024-09-13T15:00:06.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:06 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:06.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:06 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:00:06.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:06 smithi096 ceph-mon[102928]: osdmap e327: 8 total, 7 up, 7 in 2024-09-13T15:00:06.325 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:06.330 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:06.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:06.433 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:06.468 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:06.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:06.569 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:06.605 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:06.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:06.706 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:06 smithi204 ceph-mon[87071]: 1.b scrub starts 2024-09-13T15:00:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:06 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:06 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:00:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:06 smithi204 ceph-mon[87071]: osdmap e327: 8 total, 7 up, 7 in 2024-09-13T15:00:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:06.733 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:06.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:06.835 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:06.884 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:06.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:06.986 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:07.016 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:07.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:07.118 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:07.148 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:07.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:07.250 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:07.278 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:07.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:07.380 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:07.410 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:07.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:07.512 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:07.530 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:07.552 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:07.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:07 smithi096 ceph-mon[98901]: 1.1b scrub starts 2024-09-13T15:00:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:07 smithi096 ceph-mon[98901]: 1.1b scrub ok 2024-09-13T15:00:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:07 smithi096 ceph-mon[98901]: pgmap v1073: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 878 op/s; 40996/257370 objects degraded (15.929%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:00:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:07 smithi096 ceph-mon[98901]: 1.1c scrub starts 2024-09-13T15:00:07.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:07 smithi096 ceph-mon[98901]: 1.1c scrub ok 2024-09-13T15:00:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:07 smithi096 ceph-mon[102928]: 1.1b scrub starts 2024-09-13T15:00:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:07 smithi096 ceph-mon[102928]: 1.1b scrub ok 2024-09-13T15:00:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:07 smithi096 ceph-mon[102928]: pgmap v1073: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 878 op/s; 40996/257370 objects degraded (15.929%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:00:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:07 smithi096 ceph-mon[102928]: 1.1c scrub starts 2024-09-13T15:00:07.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:07 smithi096 ceph-mon[102928]: 1.1c scrub ok 2024-09-13T15:00:07.654 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:07.699 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:07.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:07 smithi204 ceph-mon[87071]: 1.1b scrub starts 2024-09-13T15:00:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:07 smithi204 ceph-mon[87071]: 1.1b scrub ok 2024-09-13T15:00:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:07 smithi204 ceph-mon[87071]: pgmap v1073: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 878 op/s; 40996/257370 objects degraded (15.929%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:00:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:07 smithi204 ceph-mon[87071]: 1.1c scrub starts 2024-09-13T15:00:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:07 smithi204 ceph-mon[87071]: 1.1c scrub ok 2024-09-13T15:00:07.801 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:07.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.833 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:07.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:07.934 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:07.970 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:07.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:08.072 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:08.103 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:08.103 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:00:08.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:08.205 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:08.242 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:08.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:08.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:00:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:00:07] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.51.0" 2024-09-13T15:00:08.344 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:08.382 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:08.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:08.484 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:08.523 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:08.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:08.624 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:08.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.663 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:08.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:08.764 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:08.802 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:08.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:08.864 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:00:08.864 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:00:08.904 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:08.941 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:08.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:09.043 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:09.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.065 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:09.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:09.166 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:09.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.232 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:09.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:09.333 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:09.378 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:09.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:09.479 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:09.507 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:09.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:09 smithi096 ceph-mon[98901]: pgmap v1074: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 889 op/s; 40585/249669 objects degraded (16.256%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:00:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:09 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:00:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:09 smithi096 ceph-mon[102928]: pgmap v1074: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 889 op/s; 40585/249669 objects degraded (16.256%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:00:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:09 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:00:09.610 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:09.637 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:09.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:09 smithi204 ceph-mon[87071]: pgmap v1074: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 889 op/s; 40585/249669 objects degraded (16.256%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:00:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:09 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:00:09.738 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:09.779 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:09.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:09.881 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:09.924 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:09.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:10.026 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:10.067 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:10.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:10.169 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:10.203 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:10.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:10.305 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:10.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:10 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 40585/249669 objects degraded (16.256%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:10.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:10 smithi096 ceph-mon[98901]: 1.c scrub starts 2024-09-13T15:00:10.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:10 smithi096 ceph-mon[98901]: 1.c scrub ok 2024-09-13T15:00:10.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:10 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 40585/249669 objects degraded (16.256%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:10.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:10 smithi096 ceph-mon[102928]: 1.c scrub starts 2024-09-13T15:00:10.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:10 smithi096 ceph-mon[102928]: 1.c scrub ok 2024-09-13T15:00:10.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.338 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:10.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:10.439 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:10.468 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:10 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 40585/249669 objects degraded (16.256%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:10 smithi204 ceph-mon[87071]: 1.c scrub starts 2024-09-13T15:00:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:10 smithi204 ceph-mon[87071]: 1.c scrub ok 2024-09-13T15:00:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:00:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:00:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:10.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:10.491 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:10.503 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:00:10.503 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:00:10.522 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:00:10.522 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:00:10.538 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:00:10.538 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:00:10.539 INFO:teuthology.orchestra.run.smithi096.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:00:10.571 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:10.606 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:00:10.606 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:00:10.608 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:10.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:10.635 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:00:10.635 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:00:10.661 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:00:10.662 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:00:10.693 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:00:10.693 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:00:10.709 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:10.747 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:10.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:10.849 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:10.873 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:10.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:10.975 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:11.001 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:11.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:11.103 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:11.129 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:11.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:11.230 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:11.263 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:11.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:11.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:11 smithi096 ceph-mon[102928]: pgmap v1075: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 39976/243567 objects degraded (16.413%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:00:11.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:11 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2491799718' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:11.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:11 smithi096 ceph-mon[98901]: pgmap v1075: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 39976/243567 objects degraded (16.413%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:00:11.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:11 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2491799718' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:11.365 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:11.400 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:11.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:11.502 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:11.535 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:11.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:11.637 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:11.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.660 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:11.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:11 smithi204 ceph-mon[87071]: pgmap v1075: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 39976/243567 objects degraded (16.413%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:00:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:11 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2491799718' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:11.761 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:11.788 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:11.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:11.890 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:11.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.920 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:11.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:11.938 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:00:11.938 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:00:12.021 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:12.072 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:12.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:12.174 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:12.211 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:12.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:12.313 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:12.349 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:12.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:12.452 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:12.491 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:12.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:12.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:12 smithi096 ceph-mon[98901]: 1.b scrub starts 2024-09-13T15:00:12.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:12 smithi096 ceph-mon[98901]: 1.b scrub ok 2024-09-13T15:00:12.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:12 smithi096 ceph-mon[102928]: 1.b scrub starts 2024-09-13T15:00:12.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:12 smithi096 ceph-mon[102928]: 1.b scrub ok 2024-09-13T15:00:12.593 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:12.625 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:12.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:12.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:12 smithi204 ceph-mon[87071]: 1.b scrub starts 2024-09-13T15:00:12.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:12 smithi204 ceph-mon[87071]: 1.b scrub ok 2024-09-13T15:00:12.727 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:12.756 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:12.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:12.808 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T15:00:12.858 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:12.893 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:12.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:12.995 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:13.032 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:13.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:13.133 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:13.170 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:13.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:13.272 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:13.309 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:13.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:13.410 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:13.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.443 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:13.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:13.544 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:13.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:13 smithi096 ceph-mon[98901]: pgmap v1076: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 39826/242799 objects degraded (16.403%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:00:13.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:13 smithi096 ceph-mon[102928]: pgmap v1076: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 39826/242799 objects degraded (16.403%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:00:13.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.586 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:13.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:13.687 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:13.710 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:13.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:13.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:13 smithi204 ceph-mon[87071]: pgmap v1076: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 39826/242799 objects degraded (16.403%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:00:13.812 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:13.841 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:13.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:13.939 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:00:13.943 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:13.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.978 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:13.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:14.079 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:14.111 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:14.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:14.213 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T15:00:14.213 INFO:teuthology.orchestra.run.smithi096.stdout: "success": "" 2024-09-13T15:00:14.213 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T15:00:14.213 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:14.252 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:14.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:14.354 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:14.389 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:14.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:14.491 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:14.529 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:14.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:14.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:14 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 39826/242799 objects degraded (16.403%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:14.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:14 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 39826/242799 objects degraded (16.403%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:14.631 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:14.659 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:14.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:14.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:14 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 39826/242799 objects degraded (16.403%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:14.761 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:14.803 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:14.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:14.904 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:14.936 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:14.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:15.038 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:15.073 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:15.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:15.175 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:15.235 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:15.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:15.337 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:15.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.381 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:15.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:15.482 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:15.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.516 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:15.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:15.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:15 smithi096 ceph-mon[98901]: pgmap v1077: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 875 op/s; 39290/234432 objects degraded (16.760%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:00:15.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:15 smithi096 ceph-mon[102928]: pgmap v1077: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 875 op/s; 39290/234432 objects degraded (16.760%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:00:15.617 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:15.652 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:15.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:15.688 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:15 smithi204 ceph-mon[87071]: pgmap v1077: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 875 op/s; 39290/234432 objects degraded (16.760%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:00:15.754 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:15.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.793 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:15.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:15.894 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:15.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.929 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:15.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:16.030 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:16.058 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:16.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:16.160 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:16.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.206 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:16.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:16.307 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:16.359 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:16.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:16.370 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T15:00:16.461 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:16.473 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:16 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/180896526' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:16.474 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:16 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/180896526' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:16.496 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:16.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:16.598 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:16.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.644 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:16.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:16.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:16 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/180896526' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:16.746 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:16.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.786 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:16.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:16.888 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:16.935 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:16.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:17.037 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:17.072 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:17.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:17.174 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:17.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.210 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:17.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:17.310 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:17.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.349 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:17.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:17.450 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:17.506 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:17.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:17.570 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:17 smithi096 ceph-mon[98901]: pgmap v1078: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 797 op/s; 39030/231441 objects degraded (16.864%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:00:17.571 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:17 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:00:17.571 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:17 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/180896526' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:17.571 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:17 smithi096 ceph-mon[98901]: osdmap e328: 8 total, 7 up, 7 in 2024-09-13T15:00:17.571 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:17 smithi096 ceph-mon[102928]: pgmap v1078: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 797 op/s; 39030/231441 objects degraded (16.864%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:00:17.571 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:17 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:00:17.571 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:17 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/180896526' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:17.571 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:17 smithi096 ceph-mon[102928]: osdmap e328: 8 total, 7 up, 7 in 2024-09-13T15:00:17.604 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:17.607 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:17.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.653 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:17.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:17.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:17 smithi204 ceph-mon[87071]: pgmap v1078: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 797 op/s; 39030/231441 objects degraded (16.864%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:00:17.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:17 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:00:17.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:17 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/180896526' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:17.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:17 smithi204 ceph-mon[87071]: osdmap e328: 8 total, 7 up, 7 in 2024-09-13T15:00:17.754 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:17.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.787 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:17.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:17.888 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:17.920 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:17.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:18.022 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:18.061 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:18.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:18.164 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:18.184 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:00:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:00:17] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.51.0" 2024-09-13T15:00:18.203 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:18.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:18.305 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:18.339 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:18.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:18.441 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:18.491 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:18.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:18.593 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:18.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.626 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:18.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:18.727 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:18.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.780 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:18.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:18.881 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:18.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.928 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:18.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:19.029 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:19.071 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:19.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:19.173 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:19.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.224 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:19.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:19.325 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:19.353 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:19.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:19.408 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:19 smithi204 ceph-mon[87071]: pgmap v1080: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 852 op/s; 38599/224082 objects degraded (17.225%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:00:19.408 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:19 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 38599/224082 objects degraded (17.225%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:19.454 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:19.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.492 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:19.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:19.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:19 smithi096 ceph-mon[98901]: pgmap v1080: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 852 op/s; 38599/224082 objects degraded (17.225%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:00:19.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:19 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 38599/224082 objects degraded (17.225%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:19.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:19 smithi096 ceph-mon[102928]: pgmap v1080: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 852 op/s; 38599/224082 objects degraded (17.225%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:00:19.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:19 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 38599/224082 objects degraded (17.225%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:19.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:19.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.625 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:19.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:19.726 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:19.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.764 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:19.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:19.865 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:19.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.897 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:19.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:19.998 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:20.043 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:20.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:20.145 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:20.168 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:20.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:20.272 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:20.306 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:20.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:20.408 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:20.435 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:20.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:00:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:00:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:20.537 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:20.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.568 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:20.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:20.669 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:20.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.698 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:20.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:20.799 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:20.837 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:20.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:20.938 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:20.966 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:20.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:21.069 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:20 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:21.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:20 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:21.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:20 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:21.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:20 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:21.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.121 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:21.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:21.165 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:20 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:21.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:20 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:21.222 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:21.258 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:21.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:21.360 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:21.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.398 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:21.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:21.499 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:21.533 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:21.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:21.635 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:21.658 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:21.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:21.760 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:21.785 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:21.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:21.887 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:21.918 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:21.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:21.953 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:21 smithi096 ceph-mon[102928]: pgmap v1081: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 38101/218532 objects degraded (17.435%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:00:21.954 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:21.954 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:21.954 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:21 smithi096 ceph-mon[98901]: pgmap v1081: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 38101/218532 objects degraded (17.435%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:00:21.954 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:21.954 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:22.020 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:22.053 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:22.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:22.153 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T15:00:22.153 INFO:teuthology.orchestra.run.smithi096.stdout: "size": 20, 2024-09-13T15:00:22.153 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 600, 2024-09-13T15:00:22.153 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T15:00:22.153 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.153 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:88714 9.0 e313/302 9:01cfaf0b:::benchmark_data_smithi096_256909_object88713:head v 313'5644, mlcod=313'5642)", 2024-09-13T15:00:22.153 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:58:42.928948+0000", 2024-09-13T15:00:22.153 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 99.043023790999996, 2024-09-13T15:00:22.153 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15184929999999999, 2024-09-13T15:00:22.154 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.154 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.154 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.154 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.154 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.154 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.154 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928948+0000", 2024-09-13T15:00:22.154 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.154 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.154 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.154 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.154 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928948+0000", 2024-09-13T15:00:22.154 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.155 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.155 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.155 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.155 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928949+0000", 2024-09-13T15:00:22.155 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.356e-06 2024-09-13T15:00:22.155 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.155 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.155 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.155 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928994+0000", 2024-09-13T15:00:22.155 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4551000000000001e-05 2024-09-13T15:00:22.155 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.155 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.155 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.156 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.929009+0000", 2024-09-13T15:00:22.156 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5537e-05 2024-09-13T15:00:22.156 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.156 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.156 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.156 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.929018+0000", 2024-09-13T15:00:22.156 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.3699999999999995e-06 2024-09-13T15:00:22.156 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.156 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.156 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.156 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.929075+0000", 2024-09-13T15:00:22.156 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7500999999999997e-05 2024-09-13T15:00:22.156 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.157 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.157 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.157 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.929094+0000", 2024-09-13T15:00:22.157 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8757999999999999e-05 2024-09-13T15:00:22.157 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.157 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.157 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.157 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080745+0000", 2024-09-13T15:00:22.157 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15165108699999999 2024-09-13T15:00:22.157 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.158 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.158 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.158 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080797+0000", 2024-09-13T15:00:22.158 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2139999999999999e-05 2024-09-13T15:00:22.158 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.158 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.158 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.158 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.158 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.158 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:88718 9.2 e313/302 9:4ec1e6d3:::benchmark_data_smithi096_256909_object88717:head v 313'5533, mlcod=313'5531)", 2024-09-13T15:00:22.158 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:58:42.934191+0000", 2024-09-13T15:00:22.158 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 99.037780588999993, 2024-09-13T15:00:22.158 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15776009699999999, 2024-09-13T15:00:22.159 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.159 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.159 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.159 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.159 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.159 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.159 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.934191+0000", 2024-09-13T15:00:22.159 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.159 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.159 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.159 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.159 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.934191+0000", 2024-09-13T15:00:22.159 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.160 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.160 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.160 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.160 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.934192+0000", 2024-09-13T15:00:22.160 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7199999999999997e-07 2024-09-13T15:00:22.160 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.160 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.160 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.160 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.934264+0000", 2024-09-13T15:00:22.160 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1389000000000004e-05 2024-09-13T15:00:22.161 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.161 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.161 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.161 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080739+0000", 2024-09-13T15:00:22.161 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.14647585900000001 2024-09-13T15:00:22.161 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.161 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.161 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.161 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080754+0000", 2024-09-13T15:00:22.161 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4899000000000001e-05 2024-09-13T15:00:22.161 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.161 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.161 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.162 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080821+0000", 2024-09-13T15:00:22.162 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7006000000000002e-05 2024-09-13T15:00:22.162 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.162 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.162 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.162 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080844+0000", 2024-09-13T15:00:22.162 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.2353000000000001e-05 2024-09-13T15:00:22.162 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.162 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.162 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.162 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.091909+0000", 2024-09-13T15:00:22.162 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011065643999999999 2024-09-13T15:00:22.162 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.162 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.163 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.163 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.091951+0000", 2024-09-13T15:00:22.163 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1974999999999997e-05 2024-09-13T15:00:22.163 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.163 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.163 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.163 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.163 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.163 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:88731 9.0 e313/302 9:088adfef:::benchmark_data_smithi096_256909_object88730:head v 313'5645, mlcod=313'5642)", 2024-09-13T15:00:22.163 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:58:42.941408+0000", 2024-09-13T15:00:22.163 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 99.030563911000002, 2024-09-13T15:00:22.163 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.14806446100000001, 2024-09-13T15:00:22.163 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.163 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.164 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.164 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.164 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.164 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.164 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.941408+0000", 2024-09-13T15:00:22.164 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.164 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.164 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.164 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.164 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.941408+0000", 2024-09-13T15:00:22.164 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.164 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.164 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.164 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.165 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.941409+0000", 2024-09-13T15:00:22.165 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2759999999999999e-06 2024-09-13T15:00:22.165 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.165 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.165 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.165 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.941459+0000", 2024-09-13T15:00:22.165 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9480000000000001e-05 2024-09-13T15:00:22.165 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.165 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.165 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.165 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080738+0000", 2024-09-13T15:00:22.165 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.139279245 2024-09-13T15:00:22.165 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.165 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080754+0000", 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6464e-05 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080823+0000", 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8757000000000006e-05 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080839+0000", 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5909999999999998e-05 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.166 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.167 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.089437+0000", 2024-09-13T15:00:22.167 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0085977299999999996 2024-09-13T15:00:22.167 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.167 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.167 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.167 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.089472+0000", 2024-09-13T15:00:22.167 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.5599000000000003e-05 2024-09-13T15:00:22.167 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.167 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.167 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.167 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.167 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.167 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:108556 9.7 e317/316 9:ef5b8d39:::benchmark_data_smithi096_256909_object108555:head v 317'6740, mlcod=0'0)", 2024-09-13T15:00:22.167 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:05.488790+0000", 2024-09-13T15:00:22.168 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 76.483181727000002, 2024-09-13T15:00:22.168 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.084939100000000003, 2024-09-13T15:00:22.168 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.168 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.168 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.168 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.168 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.168 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.168 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.488790+0000", 2024-09-13T15:00:22.168 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.168 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.168 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.168 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.169 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.488790+0000", 2024-09-13T15:00:22.169 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.169 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.169 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.169 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.169 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.488790+0000", 2024-09-13T15:00:22.169 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0900000000000002e-07 2024-09-13T15:00:22.169 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.169 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.169 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.169 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.488817+0000", 2024-09-13T15:00:22.169 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6585999999999999e-05 2024-09-13T15:00:22.170 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.170 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.170 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.170 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.488823+0000", 2024-09-13T15:00:22.170 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9030000000000001e-06 2024-09-13T15:00:22.170 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.170 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.170 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.170 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.488828+0000", 2024-09-13T15:00:22.170 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6059999999999996e-06 2024-09-13T15:00:22.171 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.171 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.171 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.171 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.513251+0000", 2024-09-13T15:00:22.171 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.024423022999999999 2024-09-13T15:00:22.171 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.171 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.171 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.171 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.513278+0000", 2024-09-13T15:00:22.171 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7149999999999999e-05 2024-09-13T15:00:22.171 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.171 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.171 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.171 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.573661+0000", 2024-09-13T15:00:22.171 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.060383035000000002 2024-09-13T15:00:22.172 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.172 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.172 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.172 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.573729+0000", 2024-09-13T15:00:22.172 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8387999999999996e-05 2024-09-13T15:00:22.172 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.172 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.172 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.172 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.172 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.172 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:108558 9.7 e317/316 9:ebd4bfb0:::benchmark_data_smithi096_256909_object108557:head v 317'6741, mlcod=0'0)", 2024-09-13T15:00:22.172 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:05.492548+0000", 2024-09-13T15:00:22.172 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 76.479423870999995, 2024-09-13T15:00:22.172 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.081248148000000006, 2024-09-13T15:00:22.173 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.173 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.173 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.173 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.173 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.173 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.173 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.492548+0000", 2024-09-13T15:00:22.173 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.173 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.173 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.173 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.173 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.492548+0000", 2024-09-13T15:00:22.173 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.173 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.174 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.174 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.174 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.492566+0000", 2024-09-13T15:00:22.174 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8050000000000002e-05 2024-09-13T15:00:22.174 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.174 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.174 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.174 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.492627+0000", 2024-09-13T15:00:22.174 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1187999999999998e-05 2024-09-13T15:00:22.174 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.174 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.174 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.174 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.492639+0000", 2024-09-13T15:00:22.174 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1542999999999999e-05 2024-09-13T15:00:22.174 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.175 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.175 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.175 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.492646+0000", 2024-09-13T15:00:22.175 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2949999999999996e-06 2024-09-13T15:00:22.175 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.175 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.175 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.175 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.523103+0000", 2024-09-13T15:00:22.175 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.030457070999999999 2024-09-13T15:00:22.175 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.175 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.175 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.175 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.523128+0000", 2024-09-13T15:00:22.175 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.5097e-05 2024-09-13T15:00:22.176 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.176 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.176 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.176 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.573734+0000", 2024-09-13T15:00:22.176 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.050605495 2024-09-13T15:00:22.176 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.176 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.176 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.176 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:05.573796+0000", 2024-09-13T15:00:22.176 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2409000000000005e-05 2024-09-13T15:00:22.176 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.176 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.176 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.176 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.177 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.177 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:120681 9.7 e320/316 9:ef43f058:::benchmark_data_smithi096_256909_object1273:head v 320'7516, mlcod=0'0)", 2024-09-13T15:00:22.178 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:23.190918+0000", 2024-09-13T15:00:22.178 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 58.781054206, 2024-09-13T15:00:22.178 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.076048507000000001, 2024-09-13T15:00:22.178 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.178 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.178 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.178 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.178 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.178 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.178 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.190918+0000", 2024-09-13T15:00:22.178 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.178 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.178 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.178 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.179 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.190918+0000", 2024-09-13T15:00:22.179 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.179 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.179 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.179 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.179 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.190919+0000", 2024-09-13T15:00:22.179 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7100000000000011e-07 2024-09-13T15:00:22.179 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.179 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.179 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.179 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.190925+0000", 2024-09-13T15:00:22.180 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2219999999999999e-06 2024-09-13T15:00:22.180 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.180 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.180 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.180 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.190935+0000", 2024-09-13T15:00:22.180 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0767e-05 2024-09-13T15:00:22.180 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.180 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.180 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.180 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.190943+0000", 2024-09-13T15:00:22.180 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2459999999999997e-06 2024-09-13T15:00:22.180 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.180 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.181 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.181 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.205449+0000", 2024-09-13T15:00:22.181 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.014506491 2024-09-13T15:00:22.181 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.181 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.181 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.181 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.205474+0000", 2024-09-13T15:00:22.181 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4719000000000002e-05 2024-09-13T15:00:22.181 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.181 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.181 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.181 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.266909+0000", 2024-09-13T15:00:22.181 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.061435468999999999 2024-09-13T15:00:22.182 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.182 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.182 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.182 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.266966+0000", 2024-09-13T15:00:22.182 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6622000000000001e-05 2024-09-13T15:00:22.182 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.182 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.182 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.182 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.183 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.183 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:120868 9.f e320/316 9:fddb9ba9:::benchmark_data_smithi096_256909_object1460:head v 320'7499, mlcod=0'0)", 2024-09-13T15:00:22.183 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:23.434599+0000", 2024-09-13T15:00:22.183 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 58.537372857999998, 2024-09-13T15:00:22.183 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.078635469999999999, 2024-09-13T15:00:22.183 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.183 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.183 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.183 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.183 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.183 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.183 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.434599+0000", 2024-09-13T15:00:22.183 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.184 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.184 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.184 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.184 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.434599+0000", 2024-09-13T15:00:22.184 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.184 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.184 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.184 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.184 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.434600+0000", 2024-09-13T15:00:22.184 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2600000000000001e-07 2024-09-13T15:00:22.184 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.184 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.184 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.185 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.434605+0000", 2024-09-13T15:00:22.185 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4940000000000003e-06 2024-09-13T15:00:22.185 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.185 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.185 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.185 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.434618+0000", 2024-09-13T15:00:22.185 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2901999999999999e-05 2024-09-13T15:00:22.185 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.185 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.185 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.185 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.434625+0000", 2024-09-13T15:00:22.185 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3579999999999999e-06 2024-09-13T15:00:22.185 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.185 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.186 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.186 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.482365+0000", 2024-09-13T15:00:22.186 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.047740527999999997 2024-09-13T15:00:22.186 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.186 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.186 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.186 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.482384+0000", 2024-09-13T15:00:22.186 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9335e-05 2024-09-13T15:00:22.186 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.186 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.186 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.186 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.513163+0000", 2024-09-13T15:00:22.186 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.03077891 2024-09-13T15:00:22.186 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.187 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.187 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.187 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.513234+0000", 2024-09-13T15:00:22.187 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1017000000000001e-05 2024-09-13T15:00:22.187 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.187 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.187 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.187 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.187 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.187 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:120877 9.f e320/316 9:f0ff22f2:::benchmark_data_smithi096_256909_object1469:head v 320'7500, mlcod=0'0)", 2024-09-13T15:00:22.187 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:23.437942+0000", 2024-09-13T15:00:22.187 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 58.534029287999999, 2024-09-13T15:00:22.187 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.088773064999999998, 2024-09-13T15:00:22.188 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.188 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.188 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.188 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.188 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.188 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.188 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.437942+0000", 2024-09-13T15:00:22.188 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.188 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.188 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.188 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.188 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.437942+0000", 2024-09-13T15:00:22.188 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.188 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.189 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.189 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.189 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.437943+0000", 2024-09-13T15:00:22.189 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.8100000000000002e-07 2024-09-13T15:00:22.189 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.189 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.189 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.189 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.437949+0000", 2024-09-13T15:00:22.189 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8150000000000002e-06 2024-09-13T15:00:22.189 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.189 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.189 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.189 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.437962+0000", 2024-09-13T15:00:22.189 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2673e-05 2024-09-13T15:00:22.190 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.190 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.190 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.190 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.437969+0000", 2024-09-13T15:00:22.190 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.627e-06 2024-09-13T15:00:22.190 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.190 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.190 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.190 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.489624+0000", 2024-09-13T15:00:22.190 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.051655049000000001 2024-09-13T15:00:22.190 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.190 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.190 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.190 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.489648+0000", 2024-09-13T15:00:22.191 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3193999999999998e-05 2024-09-13T15:00:22.191 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.191 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.191 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.191 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.526655+0000", 2024-09-13T15:00:22.191 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.037007557000000003 2024-09-13T15:00:22.191 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.191 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.191 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.191 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.526716+0000", 2024-09-13T15:00:22.191 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0368999999999997e-05 2024-09-13T15:00:22.191 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.191 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.191 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.192 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.192 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.192 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:120878 9.7 e320/316 9:e44adc74:::benchmark_data_smithi096_256909_object1470:head v 320'7531, mlcod=0'0)", 2024-09-13T15:00:22.192 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:23.440886+0000", 2024-09-13T15:00:22.192 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 58.531086101, 2024-09-13T15:00:22.192 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.078429521000000002, 2024-09-13T15:00:22.192 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.192 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.192 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.192 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.192 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.192 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.192 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.440886+0000", 2024-09-13T15:00:22.193 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.193 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.193 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.193 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.193 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.440886+0000", 2024-09-13T15:00:22.193 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.193 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.193 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.193 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.193 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.440886+0000", 2024-09-13T15:00:22.193 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9100000000000003e-07 2024-09-13T15:00:22.193 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.193 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.193 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.194 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.440892+0000", 2024-09-13T15:00:22.194 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4700000000000001e-06 2024-09-13T15:00:22.194 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.194 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.194 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.194 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.440906+0000", 2024-09-13T15:00:22.194 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4166e-05 2024-09-13T15:00:22.194 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.194 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.194 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.194 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.440914+0000", 2024-09-13T15:00:22.194 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.7209999999999994e-06 2024-09-13T15:00:22.194 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.194 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.195 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.195 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.489613+0000", 2024-09-13T15:00:22.195 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.048698996000000001 2024-09-13T15:00:22.195 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.195 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.195 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.195 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.489642+0000", 2024-09-13T15:00:22.195 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.8778000000000001e-05 2024-09-13T15:00:22.195 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.195 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.195 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.195 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.519265+0000", 2024-09-13T15:00:22.195 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.029623535999999999 2024-09-13T15:00:22.195 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.196 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.196 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.196 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.519315+0000", 2024-09-13T15:00:22.196 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0062999999999998e-05 2024-09-13T15:00:22.196 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.196 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.196 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.196 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.196 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.196 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:125635 9.7 e321/316 9:e2b86dc3:::benchmark_data_smithi096_256909_object6227:head v 321'7844, mlcod=0'0)", 2024-09-13T15:00:22.196 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:29.236486+0000", 2024-09-13T15:00:22.196 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 52.735485683, 2024-09-13T15:00:22.196 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.079551944999999999, 2024-09-13T15:00:22.196 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.197 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.197 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.197 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.197 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.197 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.197 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:29.236486+0000", 2024-09-13T15:00:22.197 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.197 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.197 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.197 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.197 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:29.236486+0000", 2024-09-13T15:00:22.197 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.197 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.197 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.198 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.198 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:29.236487+0000", 2024-09-13T15:00:22.198 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.254e-06 2024-09-13T15:00:22.198 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.198 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.198 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.198 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:29.236495+0000", 2024-09-13T15:00:22.198 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9019999999999999e-06 2024-09-13T15:00:22.198 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.198 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.198 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.198 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:29.236511+0000", 2024-09-13T15:00:22.198 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5477000000000001e-05 2024-09-13T15:00:22.198 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.199 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.199 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.199 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:29.236519+0000", 2024-09-13T15:00:22.199 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6859999999999995e-06 2024-09-13T15:00:22.199 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.199 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.199 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.199 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:29.247518+0000", 2024-09-13T15:00:22.199 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.010999093999999999 2024-09-13T15:00:22.200 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.200 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.200 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.200 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:29.247539+0000", 2024-09-13T15:00:22.200 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0523e-05 2024-09-13T15:00:22.200 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.200 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.200 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.200 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:29.315982+0000", 2024-09-13T15:00:22.200 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.068443005000000001 2024-09-13T15:00:22.200 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.200 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.200 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.200 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:29.316038+0000", 2024-09-13T15:00:22.201 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6004000000000001e-05 2024-09-13T15:00:22.201 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.201 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.201 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.201 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.201 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.201 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:138492 9.7 e322/316 9:e3eefcc8:::benchmark_data_smithi096_256909_object19084:head v 322'8655, mlcod=0'0)", 2024-09-13T15:00:22.201 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:44.787976+0000", 2024-09-13T15:00:22.201 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 37.183995324000001, 2024-09-13T15:00:22.201 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.086300818000000001, 2024-09-13T15:00:22.201 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.201 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.201 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.201 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.202 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.202 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.202 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.787976+0000", 2024-09-13T15:00:22.202 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.202 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.202 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.202 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.202 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.787976+0000", 2024-09-13T15:00:22.202 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.202 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.202 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.202 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.202 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.787981+0000", 2024-09-13T15:00:22.202 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6859999999999997e-06 2024-09-13T15:00:22.203 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.203 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.203 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.203 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.787993+0000", 2024-09-13T15:00:22.203 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2300000000000001e-05 2024-09-13T15:00:22.203 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.203 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.203 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.203 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.788013+0000", 2024-09-13T15:00:22.203 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9915000000000002e-05 2024-09-13T15:00:22.203 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.203 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.203 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.203 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.788024+0000", 2024-09-13T15:00:22.204 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0769e-05 2024-09-13T15:00:22.204 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.204 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.204 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.204 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.788881+0000", 2024-09-13T15:00:22.204 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00085665700000000003 2024-09-13T15:00:22.204 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.204 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.204 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.204 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.788909+0000", 2024-09-13T15:00:22.204 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.8045e-05 2024-09-13T15:00:22.204 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.204 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.204 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.205 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.874170+0000", 2024-09-13T15:00:22.205 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.085261233000000006 2024-09-13T15:00:22.205 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.205 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.205 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.205 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.874277+0000", 2024-09-13T15:00:22.205 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00010721300000000001 2024-09-13T15:00:22.205 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.205 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.205 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.205 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.205 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.205 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:138493 9.7 e322/316 9:e9eb11ec:::benchmark_data_smithi096_256909_object19085:head v 322'8656, mlcod=0'0)", 2024-09-13T15:00:22.205 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:44.793152+0000", 2024-09-13T15:00:22.206 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 37.178819302000001, 2024-09-13T15:00:22.206 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.081190279000000004, 2024-09-13T15:00:22.206 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.206 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.206 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.206 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.206 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.206 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.206 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.793152+0000", 2024-09-13T15:00:22.206 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.206 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.206 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.206 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.206 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.793152+0000", 2024-09-13T15:00:22.207 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.207 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.207 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.207 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.207 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.793154+0000", 2024-09-13T15:00:22.207 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2929999999999999e-06 2024-09-13T15:00:22.207 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.207 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.207 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.207 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.793163+0000", 2024-09-13T15:00:22.207 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8359999999999998e-06 2024-09-13T15:00:22.207 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.207 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.207 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.207 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.793174+0000", 2024-09-13T15:00:22.208 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1833e-05 2024-09-13T15:00:22.208 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.208 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.208 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.208 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.793183+0000", 2024-09-13T15:00:22.208 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6219999999999998e-06 2024-09-13T15:00:22.208 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.208 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.208 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.208 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.797114+0000", 2024-09-13T15:00:22.208 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0039306699999999998 2024-09-13T15:00:22.208 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.208 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.209 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.209 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.797139+0000", 2024-09-13T15:00:22.209 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4915000000000001e-05 2024-09-13T15:00:22.209 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.209 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.209 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.209 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.874282+0000", 2024-09-13T15:00:22.209 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.077143122999999994 2024-09-13T15:00:22.209 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.209 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.209 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.209 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.874343+0000", 2024-09-13T15:00:22.209 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0986999999999997e-05 2024-09-13T15:00:22.209 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.210 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.210 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.210 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.210 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.210 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:138495 9.7 e322/316 9:ea201663:::benchmark_data_smithi096_256909_object19087:head v 322'8657, mlcod=0'0)", 2024-09-13T15:00:22.210 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:44.796628+0000", 2024-09-13T15:00:22.210 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 37.175344140999997, 2024-09-13T15:00:22.210 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.077782212000000003, 2024-09-13T15:00:22.210 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.210 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.210 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.210 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.210 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.210 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.211 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.796628+0000", 2024-09-13T15:00:22.211 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.211 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.211 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.211 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.211 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.796628+0000", 2024-09-13T15:00:22.211 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.211 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.211 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.211 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.211 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.796629+0000", 2024-09-13T15:00:22.211 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2300000000000001e-06 2024-09-13T15:00:22.211 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.211 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.212 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.212 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.796637+0000", 2024-09-13T15:00:22.212 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1650000000000006e-06 2024-09-13T15:00:22.212 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.212 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.212 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.212 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.796651+0000", 2024-09-13T15:00:22.212 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.395e-05 2024-09-13T15:00:22.212 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.212 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.212 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.212 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.796657+0000", 2024-09-13T15:00:22.212 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5420000000000002e-06 2024-09-13T15:00:22.213 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.213 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.213 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.213 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.806255+0000", 2024-09-13T15:00:22.213 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0095976670000000007 2024-09-13T15:00:22.213 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.213 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.213 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.213 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.806284+0000", 2024-09-13T15:00:22.213 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.9217000000000002e-05 2024-09-13T15:00:22.213 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.213 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.213 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.213 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.874345+0000", 2024-09-13T15:00:22.214 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.068061026999999996 2024-09-13T15:00:22.214 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.214 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.214 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.214 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:44.874410+0000", 2024-09-13T15:00:22.214 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4413999999999997e-05 2024-09-13T15:00:22.214 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.214 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.214 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.214 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.214 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.214 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:141786 9.7 e322/316 9:e95090d8:::benchmark_data_smithi096_256909_object22378:head v 322'8857, mlcod=0'0)", 2024-09-13T15:00:22.214 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:48.969969+0000", 2024-09-13T15:00:22.214 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 33.002002605000001, 2024-09-13T15:00:22.215 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.077893631000000005, 2024-09-13T15:00:22.215 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.215 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.215 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.215 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.215 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.215 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.215 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:48.969969+0000", 2024-09-13T15:00:22.215 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.215 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.215 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.215 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.215 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:48.969969+0000", 2024-09-13T15:00:22.215 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.216 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.216 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.216 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.216 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:48.969970+0000", 2024-09-13T15:00:22.216 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.7700000000000004e-07 2024-09-13T15:00:22.216 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.216 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.216 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.216 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:48.969976+0000", 2024-09-13T15:00:22.216 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7690000000000002e-06 2024-09-13T15:00:22.216 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.216 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.216 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.216 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:48.969989+0000", 2024-09-13T15:00:22.217 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3472000000000001e-05 2024-09-13T15:00:22.217 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.217 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.217 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.217 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:48.969996+0000", 2024-09-13T15:00:22.217 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.567e-06 2024-09-13T15:00:22.217 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.217 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.217 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.217 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:48.980459+0000", 2024-09-13T15:00:22.217 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.010462832 2024-09-13T15:00:22.217 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.217 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.217 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.218 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:48.980483+0000", 2024-09-13T15:00:22.218 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4060999999999999e-05 2024-09-13T15:00:22.218 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.218 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.218 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.218 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:49.047667+0000", 2024-09-13T15:00:22.218 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.067184095999999999 2024-09-13T15:00:22.218 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.218 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.218 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.218 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:49.047863+0000", 2024-09-13T15:00:22.218 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000196057 2024-09-13T15:00:22.218 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.218 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.218 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.219 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.219 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.219 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:143982 9.7 e325/316 9:e4b0a382:::benchmark_data_smithi096_256909_object24574:head v 325'8993, mlcod=0'0)", 2024-09-13T15:00:22.219 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:52.746712+0000", 2024-09-13T15:00:22.219 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 29.225259648000002, 2024-09-13T15:00:22.219 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.079537343999999996, 2024-09-13T15:00:22.219 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.219 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.219 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.219 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.219 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.219 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.219 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:52.746712+0000", 2024-09-13T15:00:22.219 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.220 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.220 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.220 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.220 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:52.746712+0000", 2024-09-13T15:00:22.220 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.220 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.220 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.220 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.220 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:52.746714+0000", 2024-09-13T15:00:22.220 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.858e-06 2024-09-13T15:00:22.220 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.220 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.220 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.220 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:52.746728+0000", 2024-09-13T15:00:22.221 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3872e-05 2024-09-13T15:00:22.221 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.221 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.221 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.221 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:52.746742+0000", 2024-09-13T15:00:22.221 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4129e-05 2024-09-13T15:00:22.221 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.221 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.221 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.221 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:52.746750+0000", 2024-09-13T15:00:22.221 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.191e-06 2024-09-13T15:00:22.221 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.221 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.221 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.222 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:52.761682+0000", 2024-09-13T15:00:22.222 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.014931387000000001 2024-09-13T15:00:22.222 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.222 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.222 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.222 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:52.761701+0000", 2024-09-13T15:00:22.222 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9607e-05 2024-09-13T15:00:22.222 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.222 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.222 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.222 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:52.826190+0000", 2024-09-13T15:00:22.222 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.064488807999999995 2024-09-13T15:00:22.222 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.222 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.223 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.223 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:52.826249+0000", 2024-09-13T15:00:22.223 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9491999999999999e-05 2024-09-13T15:00:22.223 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.223 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.223 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.223 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.223 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.223 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:150996 9.f e326/316 9:f57cb092:::benchmark_data_smithi096_256909_object31588:head v 326'9354, mlcod=0'0)", 2024-09-13T15:00:22.223 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:00:00.733450+0000", 2024-09-13T15:00:22.223 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 21.238521663, 2024-09-13T15:00:22.223 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.086451852999999995, 2024-09-13T15:00:22.223 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.223 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.224 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.224 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.224 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.224 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.224 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.733450+0000", 2024-09-13T15:00:22.224 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.224 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.224 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.224 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.224 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.733450+0000", 2024-09-13T15:00:22.224 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.224 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.224 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.225 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.225 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.733450+0000", 2024-09-13T15:00:22.225 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7800000000000002e-07 2024-09-13T15:00:22.225 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.225 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.225 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.225 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.733454+0000", 2024-09-13T15:00:22.225 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.7009999999999998e-06 2024-09-13T15:00:22.225 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.225 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.225 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.225 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.733461+0000", 2024-09-13T15:00:22.225 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5470000000000002e-06 2024-09-13T15:00:22.225 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.226 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.226 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.226 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.733465+0000", 2024-09-13T15:00:22.226 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6870000000000002e-06 2024-09-13T15:00:22.226 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.226 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.226 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.226 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.752770+0000", 2024-09-13T15:00:22.226 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.019304623999999999 2024-09-13T15:00:22.226 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.226 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.226 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.226 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.752792+0000", 2024-09-13T15:00:22.226 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.2291000000000001e-05 2024-09-13T15:00:22.227 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.227 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.227 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.227 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.819727+0000", 2024-09-13T15:00:22.227 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.066934878000000003 2024-09-13T15:00:22.227 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.227 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.227 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.227 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:00.819902+0000", 2024-09-13T15:00:22.227 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000174747 2024-09-13T15:00:22.227 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.227 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.227 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.227 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.228 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.228 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:160054 9.7 e327/316 9:e841b783:::benchmark_data_smithi096_256909_object40646:head v 327'9956, mlcod=0'0)", 2024-09-13T15:00:22.228 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:00:11.011063+0000", 2024-09-13T15:00:22.228 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 10.960908419999999, 2024-09-13T15:00:22.228 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.077728173999999997, 2024-09-13T15:00:22.228 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.228 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.228 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.228 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.228 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.228 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.228 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:11.011063+0000", 2024-09-13T15:00:22.228 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.229 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.229 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.229 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.229 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:11.011063+0000", 2024-09-13T15:00:22.229 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.229 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.229 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.229 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.229 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:11.011064+0000", 2024-09-13T15:00:22.229 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0410000000000001e-06 2024-09-13T15:00:22.229 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.229 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.229 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.229 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:11.011071+0000", 2024-09-13T15:00:22.230 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7229999999999998e-06 2024-09-13T15:00:22.230 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.230 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.230 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.230 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:11.011082+0000", 2024-09-13T15:00:22.230 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1262e-05 2024-09-13T15:00:22.230 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.230 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.230 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.230 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:11.011091+0000", 2024-09-13T15:00:22.230 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5450000000000003e-06 2024-09-13T15:00:22.230 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.230 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.230 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.231 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:11.046130+0000", 2024-09-13T15:00:22.231 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.035039044999999998 2024-09-13T15:00:22.231 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.231 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.231 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.231 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:11.046153+0000", 2024-09-13T15:00:22.231 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3436999999999999e-05 2024-09-13T15:00:22.231 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.231 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.231 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.231 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:11.088755+0000", 2024-09-13T15:00:22.231 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.042601722000000002 2024-09-13T15:00:22.231 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.231 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.232 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.232 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:11.088791+0000", 2024-09-13T15:00:22.232 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.6399000000000002e-05 2024-09-13T15:00:22.232 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.232 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.232 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.232 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.232 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.232 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:165844 9.f e328/316 9:f1ddce99:::benchmark_data_smithi096_256909_object46436:head v 328'10269, mlcod=0'0)", 2024-09-13T15:00:22.232 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:00:17.892835+0000", 2024-09-13T15:00:22.232 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 4.079136246, 2024-09-13T15:00:22.232 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.081734345, 2024-09-13T15:00:22.232 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.233 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.233 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.233 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.233 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.233 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.233 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.892835+0000", 2024-09-13T15:00:22.233 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.233 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.233 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.233 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.233 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.892835+0000", 2024-09-13T15:00:22.233 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.233 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.233 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.233 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.234 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.892836+0000", 2024-09-13T15:00:22.234 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2800000000000002e-07 2024-09-13T15:00:22.234 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.234 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.234 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.234 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.892840+0000", 2024-09-13T15:00:22.234 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2230000000000001e-06 2024-09-13T15:00:22.234 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.234 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.234 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.234 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.892849+0000", 2024-09-13T15:00:22.234 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0860000000000008e-06 2024-09-13T15:00:22.234 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.234 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.235 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.235 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.892854+0000", 2024-09-13T15:00:22.235 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9030000000000003e-06 2024-09-13T15:00:22.235 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.235 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.235 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.235 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.907013+0000", 2024-09-13T15:00:22.235 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.014158651 2024-09-13T15:00:22.235 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.235 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.235 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.235 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.907033+0000", 2024-09-13T15:00:22.235 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9802000000000001e-05 2024-09-13T15:00:22.236 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.236 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.236 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.236 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.974510+0000", 2024-09-13T15:00:22.236 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.067477391999999997 2024-09-13T15:00:22.236 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.236 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.236 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.236 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.974570+0000", 2024-09-13T15:00:22.236 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9859999999999999e-05 2024-09-13T15:00:22.236 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.236 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.236 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.236 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.237 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.237 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:165856 9.7 e328/316 9:ea8b5f71:::benchmark_data_smithi096_256909_object46448:head v 328'10338, mlcod=0'0)", 2024-09-13T15:00:22.237 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:00:17.899442+0000", 2024-09-13T15:00:22.237 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 4.0725295780000001, 2024-09-13T15:00:22.237 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.082683965999999998, 2024-09-13T15:00:22.237 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.237 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.237 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.237 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.237 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.237 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.237 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.899442+0000", 2024-09-13T15:00:22.237 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.237 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.238 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.238 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.238 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.899442+0000", 2024-09-13T15:00:22.238 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.238 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.238 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.238 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.238 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.899443+0000", 2024-09-13T15:00:22.238 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6799999999999999e-07 2024-09-13T15:00:22.238 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.238 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.238 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.238 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.899449+0000", 2024-09-13T15:00:22.238 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5659999999999997e-06 2024-09-13T15:00:22.238 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.239 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.239 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.239 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.899459+0000", 2024-09-13T15:00:22.239 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0516999999999999e-05 2024-09-13T15:00:22.239 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.239 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.239 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.239 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.899466+0000", 2024-09-13T15:00:22.239 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2930000000000003e-06 2024-09-13T15:00:22.239 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.239 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.239 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.239 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.915462+0000", 2024-09-13T15:00:22.239 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.015995420999999999 2024-09-13T15:00:22.240 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.240 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.240 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.240 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.915486+0000", 2024-09-13T15:00:22.240 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4165e-05 2024-09-13T15:00:22.240 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.240 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.240 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.240 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.981970+0000", 2024-09-13T15:00:22.240 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.066484302999999995 2024-09-13T15:00:22.240 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.240 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.240 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.241 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.982126+0000", 2024-09-13T15:00:22.241 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000155833 2024-09-13T15:00:22.241 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.241 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.241 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.241 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.241 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.241 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:165861 9.7 e328/316 9:e95f5668:::benchmark_data_smithi096_256909_object46453:head v 328'10339, mlcod=0'0)", 2024-09-13T15:00:22.241 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:00:17.906051+0000", 2024-09-13T15:00:22.241 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 4.0659210979999996, 2024-09-13T15:00:22.241 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.076194695000000007, 2024-09-13T15:00:22.241 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:00:22.241 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:00:22.241 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:22.241 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:00:22.242 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.242 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:00:22.242 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.906051+0000", 2024-09-13T15:00:22.242 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.242 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.242 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.242 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:00:22.242 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.906051+0000", 2024-09-13T15:00:22.242 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:00:22.242 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.242 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.242 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:00:22.242 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.906052+0000", 2024-09-13T15:00:22.242 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.023e-06 2024-09-13T15:00:22.243 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.243 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.243 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:00:22.243 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.906059+0000", 2024-09-13T15:00:22.243 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3810000000000001e-06 2024-09-13T15:00:22.243 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.243 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.243 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:00:22.243 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.906075+0000", 2024-09-13T15:00:22.243 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5478e-05 2024-09-13T15:00:22.243 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.243 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.243 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:00:22.243 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.906082+0000", 2024-09-13T15:00:22.244 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.729e-06 2024-09-13T15:00:22.244 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.244 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.244 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:00:22.244 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.922430+0000", 2024-09-13T15:00:22.244 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.016347807999999998 2024-09-13T15:00:22.244 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.244 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.244 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:00:22.244 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.922456+0000", 2024-09-13T15:00:22.244 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.5448000000000001e-05 2024-09-13T15:00:22.244 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.244 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.244 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:00:22.245 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.982132+0000", 2024-09-13T15:00:22.245 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.059676242999999997 2024-09-13T15:00:22.245 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:00:22.245 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:00:22.245 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:00:22.245 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:17.982245+0000", 2024-09-13T15:00:22.245 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000113585 2024-09-13T15:00:22.245 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.245 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.245 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.245 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:00:22.245 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:00:22.245 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T15:00:22.246 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:22.246 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:21 smithi204 ceph-mon[87071]: pgmap v1081: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 38101/218532 objects degraded (17.435%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:00:22.247 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:22.247 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:22.295 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:22.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:22.397 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:22.424 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:22.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:22.526 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:22.553 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:22.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:22.613 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 6] out_osds: [3] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2024-09-13T15:00:22.613 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:00:22.613 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2024-09-13T15:00:22.614 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:00:22.655 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:22.695 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:22.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:22.797 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:22.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.832 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:22.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:22.933 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:22.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.969 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:22.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:23.071 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:23.098 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:23.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:23.119 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:00:23.157 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T15:00:23.201 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:23.237 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:23.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:23.339 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:23.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.377 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:23.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:23.449 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:23.479 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:23.513 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:23.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:23.590 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:00:23.615 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:23.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.642 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:23.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:23.743 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:23.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.787 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:23.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:23.888 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:23.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.947 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:23.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:24.048 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:24.081 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:23 smithi204 ceph-mon[87071]: pgmap v1082: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 37964/217731 objects degraded (17.436%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:00:24.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.088 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:24.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:24.190 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:24.197 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:23 smithi096 ceph-mon[98901]: pgmap v1082: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 37964/217731 objects degraded (17.436%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:00:24.198 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:23 smithi096 ceph-mon[102928]: pgmap v1082: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 37964/217731 objects degraded (17.436%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:00:24.237 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:24.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:24.339 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:24.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.378 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:24.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:24.479 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:24.523 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:24.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:24.625 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:24.656 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:24.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:24.758 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:24.788 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:24.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:24.890 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:24.923 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:24.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:25.025 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:25.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.063 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:25.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:25.164 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:25.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.201 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:25.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:25.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:24 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 37964/217731 objects degraded (17.436%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:25.302 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:25.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:24 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 37964/217731 objects degraded (17.436%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:25.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:24 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 37964/217731 objects degraded (17.436%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:25.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.332 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:25.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:25.433 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:25.473 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:25.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:25.575 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:25.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.614 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:25.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:25.715 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:25.750 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:25.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:25.789 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:25.823 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:00:25.824 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:00:25.851 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:00:25.851 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:00:25.852 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:25.880 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:25.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:25.883 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:00:25.884 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:00:25.884 INFO:teuthology.orchestra.run.smithi096.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:00:25.907 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:00:25.907 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:00:25.929 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:00:25.929 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:00:25.953 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:00:25.953 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:00:25.984 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:26.005 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:26.008 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:00:26.008 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:00:26.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:26.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:25 smithi096 ceph-mon[98901]: pgmap v1083: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 838 op/s; 37441/209259 objects degraded (17.892%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:00:26.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:25 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3576625798' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:26.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:25 smithi096 ceph-mon[102928]: pgmap v1083: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 838 op/s; 37441/209259 objects degraded (17.892%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:00:26.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:25 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3576625798' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:26.112 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:26.158 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:26.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:26.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:25 smithi204 ceph-mon[87071]: pgmap v1083: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 838 op/s; 37441/209259 objects degraded (17.892%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:00:26.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:25 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3576625798' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:26.260 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:26.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.297 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:26.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:26.398 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:26.437 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:26.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:26.539 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:26.574 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:26.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:26.676 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:26.708 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:26.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:26.809 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:26.846 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:26.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:26.948 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:26.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.986 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:26.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:27.087 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:27.120 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:27.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:27.136 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:00:27.136 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:00:27.221 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:27.250 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:27.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:27.352 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:27.391 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:27.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:27.493 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:27.528 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:27.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:27.630 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:27.662 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:27.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:27.677 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T15:00:27.680 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T15:00:27.764 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:27.798 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:27.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:27.900 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:27.922 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:27.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:28.024 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:28.028 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:27 smithi204 ceph-mon[87071]: pgmap v1084: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 842 op/s; 37161/206160 objects degraded (18.025%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:00:28.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.057 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:28.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:28.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:27 smithi096 ceph-mon[98901]: pgmap v1084: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 842 op/s; 37161/206160 objects degraded (18.025%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:00:28.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:27 smithi096 ceph-mon[102928]: pgmap v1084: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 842 op/s; 37161/206160 objects degraded (18.025%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:00:28.074 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:00:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:00:27] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.51.0" 2024-09-13T15:00:28.158 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:28.183 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:00:28.183 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:00:28.203 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:28.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:28.305 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:28.334 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:28.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:28.436 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:28.472 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:28.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:28.574 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:28.616 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:28.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:28.718 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:28.753 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:28.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:28.855 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:28.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.896 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:28.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:28.994 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T15:00:28.996 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:29.036 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:28 smithi204 ceph-mon[87071]: from='client.36369 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:00:29.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.043 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:29.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:29.070 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":1084,"stamp":"2024-09-13T15:00:26.119307+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4506541511,"num_objects":68720,"num_object_clones":0,"num_object_copies":206160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":37161,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":68720,"num_whiteouts":0,"num_read":6008,"num_read_kb":13034,"num_write":171031,"num_write_kb":7654593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9007,"num_bytes_recovered":591693652,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98167,"ondisk_log_size":98167,"up":357,"acting":350,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16949244,"kb_used_data":11593796,"kb_used_omap":29,"kb_used_meta":5355298,"kb_avail":639266820,"statfs":{"total":671965249536,"available":654609223680,"internally_reserved":0,"allocated":11872047104,"data_stored":11856258917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30654,"internal_metadata":5483825218},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,7,6,6,12,8,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-552271872,"num_objects":-8427,"num_object_clones":0,"num_object_copies":-25281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1869,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8427,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":281,"num_bytes_recovered":18415616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002953"},"pg_stats":[{"pgid":"1.11","version":"82'100","reported_seq":256,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.373854+0000","last_change":"2024-09-13T14:59:00.929470+0000","last_active":"2024-09-13T15:00:16.373854+0000","last_peered":"2024-09-13T15:00:16.373854+0000","last_clean":"2024-09-13T15:00:16.373854+0000","last_became_active":"2024-09-13T14:59:00.928726+0000","last_became_peered":"2024-09-13T14:59:00.928726+0000","last_unstale":"2024-09-13T15:00:16.373854+0000","last_undegraded":"2024-09-13T15:00:16.373854+0000","last_fullsized":"2024-09-13T15:00:16.373854+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.087994+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.087994+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:01:26.573411+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":100,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.372837+0000","last_change":"2024-09-13T14:59:00.935629+0000","last_active":"2024-09-13T15:00:16.372837+0000","last_peered":"2024-09-13T15:00:16.372837+0000","last_clean":"2024-09-13T15:00:16.372837+0000","last_became_active":"2024-09-13T14:59:00.935106+0000","last_became_peered":"2024-09-13T14:59:00.935106+0000","last_unstale":"2024-09-13T15:00:16.372837+0000","last_undegraded":"2024-09-13T15:00:16.372837+0000","last_fullsized":"2024-09-13T15:00:16.372837+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:12.481049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":294,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.291213+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T15:00:05.291213+0000","last_peered":"2024-09-13T15:00:05.291213+0000","last_clean":"2024-09-13T15:00:05.291213+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T15:00:05.291213+0000","last_undegraded":"2024-09-13T15:00:05.291213+0000","last_fullsized":"2024-09-13T15:00:05.291213+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":58,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296115+0000","last_change":"2024-09-13T14:56:04.643672+0000","last_active":"2024-09-13T15:00:05.296115+0000","last_peered":"2024-09-13T15:00:05.296115+0000","last_clean":"2024-09-13T15:00:05.296115+0000","last_became_active":"2024-09-13T14:55:55.185995+0000","last_became_peered":"2024-09-13T14:55:55.185995+0000","last_unstale":"2024-09-13T15:00:05.296115+0000","last_undegraded":"2024-09-13T15:00:05.296115+0000","last_fullsized":"2024-09-13T15:00:05.296115+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.643614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.643614+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:36.495546+0000","scrub_duration":0.001441681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":301,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.295720+0000","last_change":"2024-09-13T14:59:00.926231+0000","last_active":"2024-09-13T15:00:05.295720+0000","last_peered":"2024-09-13T15:00:05.295720+0000","last_clean":"2024-09-13T15:00:05.295720+0000","last_became_active":"2024-09-13T14:59:00.925808+0000","last_became_peered":"2024-09-13T14:59:00.925808+0000","last_unstale":"2024-09-13T15:00:05.295720+0000","last_undegraded":"2024-09-13T15:00:05.295720+0000","last_fullsized":"2024-09-13T15:00:05.295720+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:39.672143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":549,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.373647+0000","last_change":"2024-09-13T14:59:00.923848+0000","last_active":"2024-09-13T15:00:16.373647+0000","last_peered":"2024-09-13T15:00:16.373647+0000","last_clean":"2024-09-13T15:00:16.373647+0000","last_became_active":"2024-09-13T14:59:00.923522+0000","last_became_peered":"2024-09-13T14:59:00.923522+0000","last_unstale":"2024-09-13T15:00:16.373647+0000","last_undegraded":"2024-09-13T15:00:16.373647+0000","last_fullsized":"2024-09-13T15:00:16.373647+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:35:35.267527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":687,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.372112+0000","last_change":"2024-09-13T14:59:00.930294+0000","last_active":"2024-09-13T15:00:16.372112+0000","last_peered":"2024-09-13T15:00:16.372112+0000","last_clean":"2024-09-13T15:00:16.372112+0000","last_became_active":"2024-09-13T14:59:00.929860+0000","last_became_peered":"2024-09-13T14:59:00.929860+0000","last_unstale":"2024-09-13T15:00:16.372112+0000","last_undegraded":"2024-09-13T15:00:16.372112+0000","last_fullsized":"2024-09-13T15:00:16.372112+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.240882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.240882+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:22:46.136779+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":543,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.374406+0000","last_change":"2024-09-13T14:59:00.925448+0000","last_active":"2024-09-13T15:00:16.374406+0000","last_peered":"2024-09-13T15:00:16.374406+0000","last_clean":"2024-09-13T15:00:16.374406+0000","last_became_active":"2024-09-13T14:59:00.925152+0000","last_became_peered":"2024-09-13T14:59:00.925152+0000","last_unstale":"2024-09-13T15:00:16.374406+0000","last_undegraded":"2024-09-13T15:00:16.374406+0000","last_fullsized":"2024-09-13T15:00:16.374406+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:06.703642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":295,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.291220+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T15:00:05.291220+0000","last_peered":"2024-09-13T15:00:05.291220+0000","last_clean":"2024-09-13T15:00:05.291220+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T15:00:05.291220+0000","last_undegraded":"2024-09-13T15:00:05.291220+0000","last_fullsized":"2024-09-13T15:00:05.291220+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":58,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296699+0000","last_change":"2024-09-13T14:56:03.605447+0000","last_active":"2024-09-13T15:00:05.296699+0000","last_peered":"2024-09-13T15:00:05.296699+0000","last_clean":"2024-09-13T15:00:05.296699+0000","last_became_active":"2024-09-13T14:55:55.218939+0000","last_became_peered":"2024-09-13T14:55:55.218939+0000","last_unstale":"2024-09-13T15:00:05.296699+0000","last_undegraded":"2024-09-13T15:00:05.296699+0000","last_fullsized":"2024-09-13T15:00:05.296699+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.605356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.605356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:12.467371+0000","scrub_duration":0.0020269620000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":50,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.291801+0000","last_change":"2024-09-13T14:59:00.925287+0000","last_active":"2024-09-13T15:00:05.291801+0000","last_peered":"2024-09-13T15:00:05.291801+0000","last_clean":"2024-09-13T15:00:05.291801+0000","last_became_active":"2024-09-13T14:59:00.923868+0000","last_became_peered":"2024-09-13T14:59:00.923868+0000","last_unstale":"2024-09-13T15:00:05.291801+0000","last_undegraded":"2024-09-13T15:00:05.291801+0000","last_fullsized":"2024-09-13T15:00:05.291801+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:28.106847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":122,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.293381+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T15:00:05.293381+0000","last_peered":"2024-09-13T15:00:05.293381+0000","last_clean":"2024-09-13T15:00:05.293381+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T15:00:05.293381+0000","last_undegraded":"2024-09-13T15:00:05.293381+0000","last_fullsized":"2024-09-13T15:00:05.293381+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"328'10840","reported_seq":10904,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:24.216124+0000","last_change":"2024-09-13T14:57:34.564761+0000","last_active":"2024-09-13T15:00:24.216124+0000","last_peered":"2024-09-13T15:00:24.216124+0000","last_clean":"2024-09-13T15:00:24.216124+0000","last_became_active":"2024-09-13T14:57:34.564455+0000","last_became_peered":"2024-09-13T14:57:34.564455+0000","last_unstale":"2024-09-13T15:00:24.216124+0000","last_undegraded":"2024-09-13T15:00:24.216124+0000","last_fullsized":"2024-09-13T15:00:24.216124+0000","mapping_epoch":302,"log_start":"310'4412","ondisk_log_start":"310'4412","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6428,"log_dups_size":0,"ondisk_log_size":6428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:06.596934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278659072,"num_objects":4252,"num_object_clones":0,"num_object_copies":12756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10840,"num_write_kb":482944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":68,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.291431+0000","last_change":"2024-09-13T14:59:53.223518+0000","last_active":"2024-09-13T15:00:05.291431+0000","last_peered":"2024-09-13T15:00:05.291431+0000","last_clean":"2024-09-13T15:00:05.291431+0000","last_became_active":"2024-09-13T14:59:53.222306+0000","last_became_peered":"2024-09-13T14:59:53.222306+0000","last_unstale":"2024-09-13T15:00:05.291431+0000","last_undegraded":"2024-09-13T15:00:05.291431+0000","last_fullsized":"2024-09-13T15:00:05.291431+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.654717+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.654717+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0060902070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":513,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.378717+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T15:00:16.378717+0000","last_peered":"2024-09-13T15:00:16.378717+0000","last_clean":"2024-09-13T15:00:16.378717+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T15:00:16.378717+0000","last_undegraded":"2024-09-13T15:00:16.378717+0000","last_fullsized":"2024-09-13T15:00:16.378717+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":314,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296459+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T15:00:05.296459+0000","last_peered":"2024-09-13T15:00:05.296459+0000","last_clean":"2024-09-13T15:00:05.296459+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T15:00:05.296459+0000","last_undegraded":"2024-09-13T15:00:05.296459+0000","last_fullsized":"2024-09-13T15:00:05.296459+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"328'10286","reported_seq":10355,"reported_epoch":328,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T15:00:23.692156+0000","last_change":"2024-09-13T14:59:02.220708+0000","last_active":"2024-09-13T15:00:23.692156+0000","last_peered":"2024-09-13T15:00:23.692156+0000","last_clean":"2024-09-13T14:58:59.891928+0000","last_became_active":"2024-09-13T14:59:02.208127+0000","last_became_peered":"2024-09-13T14:59:02.208127+0000","last_unstale":"2024-09-13T15:00:23.692156+0000","last_undegraded":"2024-09-13T14:59:02.073241+0000","last_fullsized":"2024-09-13T14:59:01.920942+0000","mapping_epoch":316,"log_start":"312'4895","ondisk_log_start":"312'4895","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5391,"log_dups_size":0,"ondisk_log_size":5391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273285120,"num_objects":4170,"num_object_clones":0,"num_object_copies":12510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6402,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10286,"num_write_kb":462592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":-2232}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":471,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.374995+0000","last_change":"2024-09-13T15:00:10.016332+0000","last_active":"2024-09-13T15:00:16.374995+0000","last_peered":"2024-09-13T15:00:16.374995+0000","last_clean":"2024-09-13T15:00:16.374995+0000","last_became_active":"2024-09-13T14:59:53.227343+0000","last_became_peered":"2024-09-13T14:59:53.227343+0000","last_unstale":"2024-09-13T15:00:16.374995+0000","last_undegraded":"2024-09-13T15:00:16.374995+0000","last_fullsized":"2024-09-13T15:00:16.374995+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.016260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.016260+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:56.701927+0000","scrub_duration":71,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":294,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.293210+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T15:00:05.293210+0000","last_peered":"2024-09-13T15:00:05.293210+0000","last_clean":"2024-09-13T15:00:05.293210+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T15:00:05.293210+0000","last_undegraded":"2024-09-13T15:00:05.293210+0000","last_fullsized":"2024-09-13T15:00:05.293210+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":359,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.295524+0000","last_change":"2024-09-13T14:59:00.929566+0000","last_active":"2024-09-13T15:00:05.295524+0000","last_peered":"2024-09-13T15:00:05.295524+0000","last_clean":"2024-09-13T15:00:05.295524+0000","last_became_active":"2024-09-13T14:59:00.929271+0000","last_became_peered":"2024-09-13T14:59:00.929271+0000","last_unstale":"2024-09-13T15:00:05.295524+0000","last_undegraded":"2024-09-13T15:00:05.295524+0000","last_fullsized":"2024-09-13T15:00:05.295524+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:25.655536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"328'10567","reported_seq":10611,"reported_epoch":328,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T15:00:22.485192+0000","last_change":"2024-09-13T14:59:02.370697+0000","last_active":"2024-09-13T15:00:22.485192+0000","last_peered":"2024-09-13T15:00:22.485192+0000","last_clean":"2024-09-13T14:58:59.856641+0000","last_became_active":"2024-09-13T14:59:02.311296+0000","last_became_peered":"2024-09-13T14:59:02.311296+0000","last_unstale":"2024-09-13T15:00:22.485192+0000","last_undegraded":"2024-09-13T14:59:01.964581+0000","last_fullsized":"2024-09-13T14:59:01.916393+0000","mapping_epoch":316,"log_start":"312'4679","ondisk_log_start":"312'4679","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5888,"log_dups_size":0,"ondisk_log_size":5888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:14:42.957367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282918912,"num_objects":4317,"num_object_clones":0,"num_object_copies":12951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10567,"num_write_kb":476288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":-2261}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":161,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.293658+0000","last_change":"2024-09-13T14:56:08.889974+0000","last_active":"2024-09-13T15:00:05.293658+0000","last_peered":"2024-09-13T15:00:05.293658+0000","last_clean":"2024-09-13T15:00:05.293658+0000","last_became_active":"2024-09-13T14:55:55.185755+0000","last_became_peered":"2024-09-13T14:55:55.185755+0000","last_unstale":"2024-09-13T15:00:05.293658+0000","last_undegraded":"2024-09-13T15:00:05.293658+0000","last_fullsized":"2024-09-13T15:00:05.293658+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:08.889814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:08.889814+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:02.911564+0000","scrub_duration":0.0053731569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":304,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.293089+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T15:00:05.293089+0000","last_peered":"2024-09-13T15:00:05.293089+0000","last_clean":"2024-09-13T15:00:05.293089+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T15:00:05.293089+0000","last_undegraded":"2024-09-13T15:00:05.293089+0000","last_fullsized":"2024-09-13T15:00:05.293089+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":122,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.292842+0000","last_change":"2024-09-13T14:51:44.190879+0000","last_active":"2024-09-13T15:00:05.292842+0000","last_peered":"2024-09-13T15:00:05.292842+0000","last_clean":"2024-09-13T15:00:05.292842+0000","last_became_active":"2024-09-13T14:51:44.190617+0000","last_became_peered":"2024-09-13T14:51:44.190617+0000","last_unstale":"2024-09-13T15:00:05.292842+0000","last_undegraded":"2024-09-13T15:00:05.292842+0000","last_fullsized":"2024-09-13T15:00:05.292842+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:19.536298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"328'10773","reported_seq":10807,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:22.504742+0000","last_change":"2024-09-13T14:57:34.556168+0000","last_active":"2024-09-13T15:00:22.504742+0000","last_peered":"2024-09-13T15:00:22.504742+0000","last_clean":"2024-09-13T15:00:22.504742+0000","last_became_active":"2024-09-13T14:57:34.555882+0000","last_became_peered":"2024-09-13T14:57:34.555882+0000","last_unstale":"2024-09-13T15:00:22.504742+0000","last_undegraded":"2024-09-13T15:00:22.504742+0000","last_fullsized":"2024-09-13T15:00:22.504742+0000","mapping_epoch":302,"log_start":"312'4860","ondisk_log_start":"312'4860","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5913,"log_dups_size":0,"ondisk_log_size":5913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:05:49.922717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292487168,"num_objects":4463,"num_object_clones":0,"num_object_copies":13389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10773,"num_write_kb":487552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":78,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.297054+0000","last_change":"2024-09-13T14:59:53.220539+0000","last_active":"2024-09-13T15:00:05.297054+0000","last_peered":"2024-09-13T15:00:05.297054+0000","last_clean":"2024-09-13T15:00:05.297054+0000","last_became_active":"2024-09-13T14:59:53.220175+0000","last_became_peered":"2024-09-13T14:59:53.220175+0000","last_unstale":"2024-09-13T15:00:05.297054+0000","last_undegraded":"2024-09-13T15:00:05.297054+0000","last_fullsized":"2024-09-13T15:00:05.297054+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:02.325629+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:02.325629+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0049176200000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":322,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.295163+0000","last_change":"2024-09-13T14:59:00.930126+0000","last_active":"2024-09-13T15:00:05.295163+0000","last_peered":"2024-09-13T15:00:05.295163+0000","last_clean":"2024-09-13T15:00:05.295163+0000","last_became_active":"2024-09-13T14:59:00.929803+0000","last_became_peered":"2024-09-13T14:59:00.929803+0000","last_unstale":"2024-09-13T15:00:05.295163+0000","last_undegraded":"2024-09-13T15:00:05.295163+0000","last_fullsized":"2024-09-13T15:00:05.295163+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:44.456770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":522,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.374386+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T15:00:16.374386+0000","last_peered":"2024-09-13T15:00:16.374386+0000","last_clean":"2024-09-13T15:00:16.374386+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T15:00:16.374386+0000","last_undegraded":"2024-09-13T15:00:16.374386+0000","last_fullsized":"2024-09-13T15:00:16.374386+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"328'10491","reported_seq":14517,"reported_epoch":328,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T15:00:22.077817+0000","last_change":"2024-09-13T14:59:11.915459+0000","last_active":"2024-09-13T15:00:22.077817+0000","last_peered":"2024-09-13T15:00:22.077817+0000","last_clean":"2024-09-13T14:58:59.874928+0000","last_became_active":"2024-09-13T14:59:02.174454+0000","last_became_peered":"2024-09-13T14:59:02.174454+0000","last_unstale":"2024-09-13T15:00:22.077817+0000","last_undegraded":"2024-09-13T14:59:11.911350+0000","last_fullsized":"2024-09-13T14:59:01.917676+0000","mapping_epoch":316,"log_start":"312'4842","ondisk_log_start":"312'4842","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5649,"log_dups_size":0,"ondisk_log_size":5649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:39.210607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284098560,"num_objects":4335,"num_object_clones":0,"num_object_copies":13005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10491,"num_write_kb":474432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1984,"num_bytes_recovered":130023424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"6,7","objects":-280}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":495,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.290961+0000","last_change":"2024-09-13T14:59:53.234834+0000","last_active":"2024-09-13T15:00:05.290961+0000","last_peered":"2024-09-13T15:00:05.290961+0000","last_clean":"2024-09-13T15:00:05.290961+0000","last_became_active":"2024-09-13T14:59:53.234208+0000","last_became_peered":"2024-09-13T14:59:53.234208+0000","last_unstale":"2024-09-13T15:00:05.290961+0000","last_undegraded":"2024-09-13T15:00:05.290961+0000","last_fullsized":"2024-09-13T15:00:05.290961+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":62,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.374049+0000","last_change":"2024-09-13T14:59:00.936078+0000","last_active":"2024-09-13T15:00:16.374049+0000","last_peered":"2024-09-13T15:00:16.374049+0000","last_clean":"2024-09-13T15:00:16.374049+0000","last_became_active":"2024-09-13T14:59:00.925997+0000","last_became_peered":"2024-09-13T14:59:00.925997+0000","last_unstale":"2024-09-13T15:00:16.374049+0000","last_undegraded":"2024-09-13T15:00:16.374049+0000","last_fullsized":"2024-09-13T15:00:16.374049+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:43.716136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":322,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.295846+0000","last_change":"2024-09-13T14:59:00.934706+0000","last_active":"2024-09-13T15:00:05.295846+0000","last_peered":"2024-09-13T15:00:05.295846+0000","last_clean":"2024-09-13T15:00:05.295846+0000","last_became_active":"2024-09-13T14:59:00.934367+0000","last_became_peered":"2024-09-13T14:59:00.934367+0000","last_unstale":"2024-09-13T15:00:05.295846+0000","last_undegraded":"2024-09-13T15:00:05.295846+0000","last_fullsized":"2024-09-13T15:00:05.295846+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:37:52.860543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"328'10806","reported_seq":10870,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:23.732219+0000","last_change":"2024-09-13T14:57:34.835630+0000","last_active":"2024-09-13T15:00:23.732219+0000","last_peered":"2024-09-13T15:00:23.732219+0000","last_clean":"2024-09-13T15:00:23.732219+0000","last_became_active":"2024-09-13T14:57:34.835209+0000","last_became_peered":"2024-09-13T14:57:34.835209+0000","last_unstale":"2024-09-13T15:00:23.732219+0000","last_undegraded":"2024-09-13T15:00:23.732219+0000","last_fullsized":"2024-09-13T15:00:23.732219+0000","mapping_epoch":302,"log_start":"312'5388","ondisk_log_start":"312'5388","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5418,"log_dups_size":0,"ondisk_log_size":5418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:50.159454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286785536,"num_objects":4376,"num_object_clones":0,"num_object_copies":13128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10806,"num_write_kb":485824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":259,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296212+0000","last_change":"2024-09-13T14:59:53.224000+0000","last_active":"2024-09-13T15:00:05.296212+0000","last_peered":"2024-09-13T15:00:05.296212+0000","last_clean":"2024-09-13T15:00:05.296212+0000","last_became_active":"2024-09-13T14:59:53.223473+0000","last_became_peered":"2024-09-13T14:59:53.223473+0000","last_unstale":"2024-09-13T15:00:05.296212+0000","last_undegraded":"2024-09-13T15:00:05.296212+0000","last_fullsized":"2024-09-13T15:00:05.296212+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":131344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":112,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.292925+0000","last_change":"2024-09-13T14:48:32.200521+0000","last_active":"2024-09-13T15:00:05.292925+0000","last_peered":"2024-09-13T15:00:05.292925+0000","last_clean":"2024-09-13T15:00:05.292925+0000","last_became_active":"2024-09-13T14:48:32.200256+0000","last_became_peered":"2024-09-13T14:48:32.200256+0000","last_unstale":"2024-09-13T15:00:05.292925+0000","last_undegraded":"2024-09-13T15:00:05.292925+0000","last_fullsized":"2024-09-13T15:00:05.292925+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:38.899629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":309,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296483+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T15:00:05.296483+0000","last_peered":"2024-09-13T15:00:05.296483+0000","last_clean":"2024-09-13T15:00:05.296483+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T15:00:05.296483+0000","last_undegraded":"2024-09-13T15:00:05.296483+0000","last_fullsized":"2024-09-13T15:00:05.296483+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"328'10610","reported_seq":24423,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:22.233342+0000","last_change":"2024-09-13T14:59:51.664903+0000","last_active":"2024-09-13T15:00:22.233342+0000","last_peered":"2024-09-13T15:00:22.233342+0000","last_clean":"2024-09-13T15:00:22.233342+0000","last_became_active":"2024-09-13T14:59:51.590211+0000","last_became_peered":"2024-09-13T14:59:51.590211+0000","last_unstale":"2024-09-13T15:00:22.233342+0000","last_undegraded":"2024-09-13T15:00:22.233342+0000","last_fullsized":"2024-09-13T15:00:22.233342+0000","mapping_epoch":323,"log_start":"312'5052","ondisk_log_start":"312'5052","created":302,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5558,"log_dups_size":0,"ondisk_log_size":5558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:28.900449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284295168,"num_objects":4338,"num_object_clones":0,"num_object_copies":13014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10610,"num_write_kb":478336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6872,"num_bytes_recovered":450297856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":704,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.372350+0000","last_change":"2024-09-13T15:00:10.659910+0000","last_active":"2024-09-13T15:00:16.372350+0000","last_peered":"2024-09-13T15:00:16.372350+0000","last_clean":"2024-09-13T15:00:16.372350+0000","last_became_active":"2024-09-13T14:59:53.216739+0000","last_became_peered":"2024-09-13T14:59:53.216739+0000","last_unstale":"2024-09-13T15:00:16.372350+0000","last_undegraded":"2024-09-13T15:00:16.372350+0000","last_fullsized":"2024-09-13T15:00:16.372350+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.659838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.659838+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:41.500826+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":509,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.373752+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T15:00:16.373752+0000","last_peered":"2024-09-13T15:00:16.373752+0000","last_clean":"2024-09-13T15:00:16.373752+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T15:00:16.373752+0000","last_undegraded":"2024-09-13T15:00:16.373752+0000","last_fullsized":"2024-09-13T15:00:16.373752+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":312,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.297193+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T15:00:05.297193+0000","last_peered":"2024-09-13T15:00:05.297193+0000","last_clean":"2024-09-13T15:00:05.297193+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T15:00:05.297193+0000","last_undegraded":"2024-09-13T15:00:05.297193+0000","last_fullsized":"2024-09-13T15:00:05.297193+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"328'10659","reported_seq":10730,"reported_epoch":328,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-09-13T15:00:23.749245+0000","last_change":"2024-09-13T14:59:12.888109+0000","last_active":"2024-09-13T15:00:23.749245+0000","last_peered":"2024-09-13T15:00:23.749245+0000","last_clean":"2024-09-13T14:58:59.863442+0000","last_became_active":"2024-09-13T14:59:12.882847+0000","last_became_peered":"2024-09-13T14:59:12.882847+0000","last_unstale":"2024-09-13T15:00:23.749245+0000","last_undegraded":"2024-09-13T14:59:12.878900+0000","last_fullsized":"2024-09-13T14:59:12.878712+0000","mapping_epoch":318,"log_start":"312'5299","ondisk_log_start":"312'5299","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282787840,"num_objects":4315,"num_object_clones":0,"num_object_copies":12945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4315,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10659,"num_write_kb":479168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":4315}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":248,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296602+0000","last_change":"2024-09-13T14:59:53.233276+0000","last_active":"2024-09-13T15:00:05.296602+0000","last_peered":"2024-09-13T15:00:05.296602+0000","last_clean":"2024-09-13T15:00:05.296602+0000","last_became_active":"2024-09-13T14:59:53.232987+0000","last_became_peered":"2024-09-13T14:59:53.232987+0000","last_unstale":"2024-09-13T15:00:05.296602+0000","last_undegraded":"2024-09-13T15:00:05.296602+0000","last_fullsized":"2024-09-13T15:00:05.296602+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":263,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.371760+0000","last_change":"2024-09-13T14:59:00.931433+0000","last_active":"2024-09-13T15:00:16.371760+0000","last_peered":"2024-09-13T15:00:16.371760+0000","last_clean":"2024-09-13T15:00:16.371760+0000","last_became_active":"2024-09-13T14:59:00.928906+0000","last_became_peered":"2024-09-13T14:59:00.928906+0000","last_unstale":"2024-09-13T15:00:16.371760+0000","last_undegraded":"2024-09-13T15:00:16.371760+0000","last_fullsized":"2024-09-13T15:00:16.371760+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:12.786568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":517,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.372695+0000","last_change":"2024-09-13T14:59:00.925277+0000","last_active":"2024-09-13T15:00:16.372695+0000","last_peered":"2024-09-13T15:00:16.372695+0000","last_clean":"2024-09-13T15:00:16.372695+0000","last_became_active":"2024-09-13T14:59:00.924952+0000","last_became_peered":"2024-09-13T14:59:00.924952+0000","last_unstale":"2024-09-13T15:00:16.372695+0000","last_undegraded":"2024-09-13T15:00:16.372695+0000","last_fullsized":"2024-09-13T15:00:16.372695+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:08.888187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"328'10741","reported_seq":10897,"reported_epoch":328,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-09-13T15:00:24.247415+0000","last_change":"2024-09-13T14:59:12.893143+0000","last_active":"2024-09-13T15:00:24.247415+0000","last_peered":"2024-09-13T15:00:24.247415+0000","last_clean":"2024-09-13T14:58:59.874256+0000","last_became_active":"2024-09-13T14:59:12.882701+0000","last_became_peered":"2024-09-13T14:59:12.882701+0000","last_unstale":"2024-09-13T15:00:24.247415+0000","last_undegraded":"2024-09-13T14:59:12.880389+0000","last_fullsized":"2024-09-13T14:59:12.880226+0000","mapping_epoch":318,"log_start":"310'4270","ondisk_log_start":"310'4270","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6471,"log_dups_size":0,"ondisk_log_size":6471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282918912,"num_objects":4317,"num_object_clones":0,"num_object_copies":12951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4299,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10741,"num_write_kb":481856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":2621440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4317}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":476,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.292252+0000","last_change":"2024-09-13T14:55:58.568852+0000","last_active":"2024-09-13T15:00:05.292252+0000","last_peered":"2024-09-13T15:00:05.292252+0000","last_clean":"2024-09-13T15:00:05.292252+0000","last_became_active":"2024-09-13T14:55:51.169987+0000","last_became_peered":"2024-09-13T14:55:51.169987+0000","last_unstale":"2024-09-13T15:00:05.292252+0000","last_undegraded":"2024-09-13T15:00:05.292252+0000","last_fullsized":"2024-09-13T15:00:05.292252+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.568750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.568750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:56.273429+0000","scrub_duration":0.0074886340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":854,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296341+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T15:00:05.296341+0000","last_peered":"2024-09-13T15:00:05.296341+0000","last_clean":"2024-09-13T15:00:05.296341+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T15:00:05.296341+0000","last_undegraded":"2024-09-13T15:00:05.296341+0000","last_fullsized":"2024-09-13T15:00:05.296341+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":148,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.297105+0000","last_change":"2024-09-13T14:59:00.930790+0000","last_active":"2024-09-13T15:00:05.297105+0000","last_peered":"2024-09-13T15:00:05.297105+0000","last_clean":"2024-09-13T15:00:05.297105+0000","last_became_active":"2024-09-13T14:59:00.930439+0000","last_became_peered":"2024-09-13T14:59:00.930439+0000","last_unstale":"2024-09-13T15:00:05.297105+0000","last_undegraded":"2024-09-13T15:00:05.297105+0000","last_fullsized":"2024-09-13T15:00:05.297105+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:15.636095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":112,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.293160+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T15:00:05.293160+0000","last_peered":"2024-09-13T15:00:05.293160+0000","last_clean":"2024-09-13T15:00:05.293160+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T15:00:05.293160+0000","last_undegraded":"2024-09-13T15:00:05.293160+0000","last_fullsized":"2024-09-13T15:00:05.293160+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"328'10637","reported_seq":10701,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:24.206421+0000","last_change":"2024-09-13T14:57:34.836468+0000","last_active":"2024-09-13T15:00:24.206421+0000","last_peered":"2024-09-13T15:00:24.206421+0000","last_clean":"2024-09-13T15:00:24.206421+0000","last_became_active":"2024-09-13T14:57:34.836144+0000","last_became_peered":"2024-09-13T14:57:34.836144+0000","last_unstale":"2024-09-13T15:00:24.206421+0000","last_undegraded":"2024-09-13T15:00:24.206421+0000","last_fullsized":"2024-09-13T15:00:24.206421+0000","mapping_epoch":302,"log_start":"310'4226","ondisk_log_start":"310'4226","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6411,"log_dups_size":0,"ondisk_log_size":6411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:40:48.019738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273088512,"num_objects":4167,"num_object_clones":0,"num_object_copies":12501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10637,"num_write_kb":473728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":675,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.371975+0000","last_change":"2024-09-13T14:55:59.245841+0000","last_active":"2024-09-13T15:00:16.371975+0000","last_peered":"2024-09-13T15:00:16.371975+0000","last_clean":"2024-09-13T15:00:16.371975+0000","last_became_active":"2024-09-13T14:55:51.159487+0000","last_became_peered":"2024-09-13T14:55:51.159487+0000","last_unstale":"2024-09-13T15:00:16.371975+0000","last_undegraded":"2024-09-13T15:00:16.371975+0000","last_fullsized":"2024-09-13T15:00:16.371975+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.245213+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.245213+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:47.786479+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":539,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.290696+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T15:00:05.290696+0000","last_peered":"2024-09-13T15:00:05.290696+0000","last_clean":"2024-09-13T15:00:05.290696+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T15:00:05.290696+0000","last_undegraded":"2024-09-13T15:00:05.290696+0000","last_fullsized":"2024-09-13T15:00:05.290696+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":551,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.373805+0000","last_change":"2024-09-13T14:59:00.928965+0000","last_active":"2024-09-13T15:00:16.373805+0000","last_peered":"2024-09-13T15:00:16.373805+0000","last_clean":"2024-09-13T15:00:16.373805+0000","last_became_active":"2024-09-13T14:59:00.928166+0000","last_became_peered":"2024-09-13T14:59:00.928166+0000","last_unstale":"2024-09-13T15:00:16.373805+0000","last_undegraded":"2024-09-13T15:00:16.373805+0000","last_fullsized":"2024-09-13T15:00:16.373805+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:34.878172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":548,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.375083+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T15:00:16.375083+0000","last_peered":"2024-09-13T15:00:16.375083+0000","last_clean":"2024-09-13T15:00:16.375083+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T15:00:16.375083+0000","last_undegraded":"2024-09-13T15:00:16.375083+0000","last_fullsized":"2024-09-13T15:00:16.375083+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"328'10651","reported_seq":10685,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:22.504772+0000","last_change":"2024-09-13T14:57:34.559226+0000","last_active":"2024-09-13T15:00:22.504772+0000","last_peered":"2024-09-13T15:00:22.504772+0000","last_clean":"2024-09-13T15:00:22.504772+0000","last_became_active":"2024-09-13T14:57:34.558860+0000","last_became_peered":"2024-09-13T14:57:34.558860+0000","last_unstale":"2024-09-13T15:00:22.504772+0000","last_undegraded":"2024-09-13T15:00:22.504772+0000","last_fullsized":"2024-09-13T15:00:22.504772+0000","mapping_epoch":302,"log_start":"312'4708","ondisk_log_start":"312'4708","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5943,"log_dups_size":0,"ondisk_log_size":5943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:56:32.571916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275972096,"num_objects":4211,"num_object_clones":0,"num_object_copies":12633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10651,"num_write_kb":475584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":105,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.292032+0000","last_change":"2024-09-13T14:55:59.574827+0000","last_active":"2024-09-13T15:00:05.292032+0000","last_peered":"2024-09-13T15:00:05.292032+0000","last_clean":"2024-09-13T15:00:05.292032+0000","last_became_active":"2024-09-13T14:55:51.162903+0000","last_became_peered":"2024-09-13T14:55:51.162903+0000","last_unstale":"2024-09-13T15:00:05.292032+0000","last_undegraded":"2024-09-13T15:00:05.292032+0000","last_fullsized":"2024-09-13T15:00:05.292032+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.574773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.574773+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:39.074072+0000","scrub_duration":0.001553708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":473,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.374105+0000","last_change":"2024-09-13T14:59:00.950349+0000","last_active":"2024-09-13T15:00:16.374105+0000","last_peered":"2024-09-13T15:00:16.374105+0000","last_clean":"2024-09-13T15:00:16.374105+0000","last_became_active":"2024-09-13T14:59:00.925703+0000","last_became_peered":"2024-09-13T14:59:00.925703+0000","last_unstale":"2024-09-13T15:00:16.374105+0000","last_undegraded":"2024-09-13T15:00:16.374105+0000","last_fullsized":"2024-09-13T15:00:16.374105+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:43.851359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":525,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.371852+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T15:00:16.371852+0000","last_peered":"2024-09-13T15:00:16.371852+0000","last_clean":"2024-09-13T15:00:16.371852+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T15:00:16.371852+0000","last_undegraded":"2024-09-13T15:00:16.371852+0000","last_fullsized":"2024-09-13T15:00:16.371852+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":543,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.372603+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T15:00:16.372603+0000","last_peered":"2024-09-13T15:00:16.372603+0000","last_clean":"2024-09-13T15:00:16.372603+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T15:00:16.372603+0000","last_undegraded":"2024-09-13T15:00:16.372603+0000","last_fullsized":"2024-09-13T15:00:16.372603+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"328'10555","reported_seq":10619,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:24.203484+0000","last_change":"2024-09-13T14:57:34.564910+0000","last_active":"2024-09-13T15:00:24.203484+0000","last_peered":"2024-09-13T15:00:24.203484+0000","last_clean":"2024-09-13T15:00:24.203484+0000","last_became_active":"2024-09-13T14:57:34.564633+0000","last_became_peered":"2024-09-13T14:57:34.564633+0000","last_unstale":"2024-09-13T15:00:24.203484+0000","last_undegraded":"2024-09-13T15:00:24.203484+0000","last_fullsized":"2024-09-13T15:00:24.203484+0000","mapping_epoch":302,"log_start":"310'4157","ondisk_log_start":"310'4157","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6398,"log_dups_size":0,"ondisk_log_size":6398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:53:38.983345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273481728,"num_objects":4173,"num_object_clones":0,"num_object_copies":12519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10555,"num_write_kb":471296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":248,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.294666+0000","last_change":"2024-09-13T14:56:01.658240+0000","last_active":"2024-09-13T15:00:05.294666+0000","last_peered":"2024-09-13T15:00:05.294666+0000","last_clean":"2024-09-13T15:00:05.294666+0000","last_became_active":"2024-09-13T14:55:51.160480+0000","last_became_peered":"2024-09-13T14:55:51.160480+0000","last_unstale":"2024-09-13T15:00:05.294666+0000","last_undegraded":"2024-09-13T15:00:05.294666+0000","last_fullsized":"2024-09-13T15:00:05.294666+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.658143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.658143+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:19.262955+0000","scrub_duration":0.0038903079999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":954,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296885+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T15:00:05.296885+0000","last_peered":"2024-09-13T15:00:05.296885+0000","last_clean":"2024-09-13T15:00:05.296885+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T15:00:05.296885+0000","last_undegraded":"2024-09-13T15:00:05.296885+0000","last_fullsized":"2024-09-13T15:00:05.296885+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":313,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296044+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T15:00:05.296044+0000","last_peered":"2024-09-13T15:00:05.296044+0000","last_clean":"2024-09-13T15:00:05.296044+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T15:00:05.296044+0000","last_undegraded":"2024-09-13T15:00:05.296044+0000","last_fullsized":"2024-09-13T15:00:05.296044+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":550,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.372583+0000","last_change":"2024-09-13T14:59:00.930713+0000","last_active":"2024-09-13T15:00:16.372583+0000","last_peered":"2024-09-13T15:00:16.372583+0000","last_clean":"2024-09-13T15:00:16.372583+0000","last_became_active":"2024-09-13T14:59:00.928260+0000","last_became_peered":"2024-09-13T14:59:00.928260+0000","last_unstale":"2024-09-13T15:00:16.372583+0000","last_undegraded":"2024-09-13T15:00:16.372583+0000","last_fullsized":"2024-09-13T15:00:16.372583+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:07.048853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"328'10495","reported_seq":10538,"reported_epoch":328,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T15:00:22.479219+0000","last_change":"2024-09-13T14:59:02.370702+0000","last_active":"2024-09-13T15:00:22.479219+0000","last_peered":"2024-09-13T15:00:22.479219+0000","last_clean":"2024-09-13T14:58:59.881570+0000","last_became_active":"2024-09-13T14:59:02.311551+0000","last_became_peered":"2024-09-13T14:59:02.311551+0000","last_unstale":"2024-09-13T15:00:22.479219+0000","last_undegraded":"2024-09-13T14:59:01.972778+0000","last_fullsized":"2024-09-13T14:59:01.916475+0000","mapping_epoch":316,"log_start":"311'4522","ondisk_log_start":"311'4522","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5973,"log_dups_size":0,"ondisk_log_size":5973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:01:25.255932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283181056,"num_objects":4321,"num_object_clones":0,"num_object_copies":12963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6597,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10495,"num_write_kb":474112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":-2276}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":172,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.293575+0000","last_change":"2024-09-13T14:59:53.249535+0000","last_active":"2024-09-13T15:00:05.293575+0000","last_peered":"2024-09-13T15:00:05.293575+0000","last_clean":"2024-09-13T15:00:05.293575+0000","last_became_active":"2024-09-13T14:59:53.249243+0000","last_became_peered":"2024-09-13T14:59:53.249243+0000","last_unstale":"2024-09-13T15:00:05.293575+0000","last_undegraded":"2024-09-13T15:00:05.293575+0000","last_fullsized":"2024-09-13T15:00:05.293575+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.916764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.916764+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0035298819999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":319,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.295866+0000","last_change":"2024-09-13T14:59:00.931326+0000","last_active":"2024-09-13T15:00:05.295866+0000","last_peered":"2024-09-13T15:00:05.295866+0000","last_clean":"2024-09-13T15:00:05.295866+0000","last_became_active":"2024-09-13T14:59:00.931011+0000","last_became_peered":"2024-09-13T14:59:00.931011+0000","last_unstale":"2024-09-13T15:00:05.295866+0000","last_undegraded":"2024-09-13T15:00:05.295866+0000","last_fullsized":"2024-09-13T15:00:05.295866+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:24:41.833263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":1028,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.372754+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T15:00:16.372754+0000","last_peered":"2024-09-13T15:00:16.372754+0000","last_clean":"2024-09-13T15:00:16.372754+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T15:00:16.372754+0000","last_undegraded":"2024-09-13T15:00:16.372754+0000","last_fullsized":"2024-09-13T15:00:16.372754+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":500,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.372395+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T15:00:16.372395+0000","last_peered":"2024-09-13T15:00:16.372395+0000","last_clean":"2024-09-13T15:00:16.372395+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T15:00:16.372395+0000","last_undegraded":"2024-09-13T15:00:16.372395+0000","last_fullsized":"2024-09-13T15:00:16.372395+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"328'10612","reported_seq":10646,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:22.207611+0000","last_change":"2024-09-13T14:57:34.841879+0000","last_active":"2024-09-13T15:00:22.207611+0000","last_peered":"2024-09-13T15:00:22.207611+0000","last_clean":"2024-09-13T15:00:22.207611+0000","last_became_active":"2024-09-13T14:57:34.841546+0000","last_became_peered":"2024-09-13T14:57:34.841546+0000","last_unstale":"2024-09-13T15:00:22.207611+0000","last_undegraded":"2024-09-13T15:00:22.207611+0000","last_fullsized":"2024-09-13T15:00:22.207611+0000","mapping_epoch":302,"log_start":"312'5051","ondisk_log_start":"312'5051","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5561,"log_dups_size":0,"ondisk_log_size":5561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:36.453841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279183360,"num_objects":4260,"num_object_clones":0,"num_object_copies":12780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10612,"num_write_kb":475904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":162,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.293366+0000","last_change":"2024-09-13T14:59:53.226670+0000","last_active":"2024-09-13T15:00:05.293366+0000","last_peered":"2024-09-13T15:00:05.293366+0000","last_clean":"2024-09-13T15:00:05.293366+0000","last_became_active":"2024-09-13T14:59:53.226288+0000","last_became_peered":"2024-09-13T14:59:53.226288+0000","last_unstale":"2024-09-13T15:00:05.293366+0000","last_undegraded":"2024-09-13T15:00:05.293366+0000","last_fullsized":"2024-09-13T15:00:05.293366+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.997001+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.997001+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.019401547000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":525,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.372671+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T15:00:16.372671+0000","last_peered":"2024-09-13T15:00:16.372671+0000","last_clean":"2024-09-13T15:00:16.372671+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T15:00:16.372671+0000","last_undegraded":"2024-09-13T15:00:16.372671+0000","last_fullsized":"2024-09-13T15:00:16.372671+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":541,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.293169+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T15:00:05.293169+0000","last_peered":"2024-09-13T15:00:05.293169+0000","last_clean":"2024-09-13T15:00:05.293169+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T15:00:05.293169+0000","last_undegraded":"2024-09-13T15:00:05.293169+0000","last_fullsized":"2024-09-13T15:00:05.293169+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":512,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.374203+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T15:00:16.374203+0000","last_peered":"2024-09-13T15:00:16.374203+0000","last_clean":"2024-09-13T15:00:16.374203+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T15:00:16.374203+0000","last_undegraded":"2024-09-13T15:00:16.374203+0000","last_fullsized":"2024-09-13T15:00:16.374203+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"328'10536","reported_seq":10570,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:22.203746+0000","last_change":"2024-09-13T14:57:34.562351+0000","last_active":"2024-09-13T15:00:22.203746+0000","last_peered":"2024-09-13T15:00:22.203746+0000","last_clean":"2024-09-13T15:00:22.203746+0000","last_became_active":"2024-09-13T14:57:34.561412+0000","last_became_peered":"2024-09-13T14:57:34.561412+0000","last_unstale":"2024-09-13T15:00:22.203746+0000","last_undegraded":"2024-09-13T15:00:22.203746+0000","last_fullsized":"2024-09-13T15:00:22.203746+0000","mapping_epoch":302,"log_start":"312'4953","ondisk_log_start":"312'4953","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":5583,"log_dups_size":0,"ondisk_log_size":5583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:37:07.655839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283181080,"num_objects":4322,"num_object_clones":0,"num_object_copies":12966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4322,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10536,"num_write_kb":475393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":289,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296189+0000","last_change":"2024-09-13T14:59:53.220435+0000","last_active":"2024-09-13T15:00:05.296189+0000","last_peered":"2024-09-13T15:00:05.296189+0000","last_clean":"2024-09-13T15:00:05.296189+0000","last_became_active":"2024-09-13T14:59:53.220135+0000","last_became_peered":"2024-09-13T14:59:53.220135+0000","last_unstale":"2024-09-13T15:00:05.296189+0000","last_undegraded":"2024-09-13T15:00:05.296189+0000","last_fullsized":"2024-09-13T15:00:05.296189+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.308240+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.308240+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063105979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":49,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.291949+0000","last_change":"2024-09-13T14:59:00.936064+0000","last_active":"2024-09-13T15:00:05.291949+0000","last_peered":"2024-09-13T15:00:05.291949+0000","last_clean":"2024-09-13T15:00:05.291949+0000","last_became_active":"2024-09-13T14:59:00.932311+0000","last_became_peered":"2024-09-13T14:59:00.932311+0000","last_unstale":"2024-09-13T15:00:05.291949+0000","last_undegraded":"2024-09-13T15:00:05.291949+0000","last_fullsized":"2024-09-13T15:00:05.291949+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:16.456004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":882,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.373092+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T15:00:16.373092+0000","last_peered":"2024-09-13T15:00:16.373092+0000","last_clean":"2024-09-13T15:00:16.373092+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T15:00:16.373092+0000","last_undegraded":"2024-09-13T15:00:16.373092+0000","last_fullsized":"2024-09-13T15:00:16.373092+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":50,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.290521+0000","last_change":"2024-09-13T14:59:00.924324+0000","last_active":"2024-09-13T15:00:05.290521+0000","last_peered":"2024-09-13T15:00:05.290521+0000","last_clean":"2024-09-13T15:00:05.290521+0000","last_became_active":"2024-09-13T14:59:00.923798+0000","last_became_peered":"2024-09-13T14:59:00.923798+0000","last_unstale":"2024-09-13T15:00:05.290521+0000","last_undegraded":"2024-09-13T15:00:05.290521+0000","last_fullsized":"2024-09-13T15:00:05.290521+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:10.994877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"328'10861","reported_seq":10928,"reported_epoch":328,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-09-13T15:00:22.209962+0000","last_change":"2024-09-13T14:59:12.908579+0000","last_active":"2024-09-13T15:00:22.209962+0000","last_peered":"2024-09-13T15:00:22.209962+0000","last_clean":"2024-09-13T14:58:59.887259+0000","last_became_active":"2024-09-13T14:59:12.883084+0000","last_became_peered":"2024-09-13T14:59:12.883084+0000","last_unstale":"2024-09-13T15:00:22.209962+0000","last_undegraded":"2024-09-13T14:59:12.880342+0000","last_fullsized":"2024-09-13T14:59:12.880157+0000","mapping_epoch":318,"log_start":"312'4771","ondisk_log_start":"312'4771","created":302,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:57:33.528970+0000","last_clean_scrub_stamp":"2024-09-13T14:57:33.528970+0000","objects_scrubbed":0,"log_size":6090,"log_dups_size":0,"ondisk_log_size":6090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:21.422134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285409280,"num_objects":4355,"num_object_clones":0,"num_object_copies":13065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4355,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10861,"num_write_kb":486912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":4355}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":26,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.295017+0000","last_change":"2024-09-13T14:59:00.925524+0000","last_active":"2024-09-13T15:00:05.295017+0000","last_peered":"2024-09-13T15:00:05.295017+0000","last_clean":"2024-09-13T15:00:05.295017+0000","last_became_active":"2024-09-13T14:59:00.925167+0000","last_became_peered":"2024-09-13T14:59:00.925167+0000","last_unstale":"2024-09-13T15:00:05.295017+0000","last_undegraded":"2024-09-13T15:00:05.295017+0000","last_fullsized":"2024-09-13T15:00:05.295017+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.914801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.914801+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":22,"num_write":4,"num_write_kb":49,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":17890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":534,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.374242+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T15:00:16.374242+0000","last_peered":"2024-09-13T15:00:16.374242+0000","last_clean":"2024-09-13T15:00:16.374242+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T15:00:16.374242+0000","last_undegraded":"2024-09-13T15:00:16.374242+0000","last_fullsized":"2024-09-13T15:00:16.374242+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1141,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.293805+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T15:00:05.293805+0000","last_peered":"2024-09-13T15:00:05.293805+0000","last_clean":"2024-09-13T15:00:05.293805+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T15:00:05.293805+0000","last_undegraded":"2024-09-13T15:00:05.293805+0000","last_fullsized":"2024-09-13T15:00:05.293805+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":295,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.295241+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T15:00:05.295241+0000","last_peered":"2024-09-13T15:00:05.295241+0000","last_clean":"2024-09-13T15:00:05.295241+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T15:00:05.295241+0000","last_undegraded":"2024-09-13T15:00:05.295241+0000","last_fullsized":"2024-09-13T15:00:05.295241+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"82'100","reported_seq":161,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.293227+0000","last_change":"2024-09-13T14:59:53.227319+0000","last_active":"2024-09-13T15:00:05.293227+0000","last_peered":"2024-09-13T15:00:05.293227+0000","last_clean":"2024-09-13T15:00:05.293227+0000","last_became_active":"2024-09-13T14:59:53.226920+0000","last_became_peered":"2024-09-13T14:59:53.226920+0000","last_unstale":"2024-09-13T15:00:05.293227+0000","last_undegraded":"2024-09-13T15:00:05.293227+0000","last_fullsized":"2024-09-13T15:00:05.293227+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.997001+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.997001+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.019401547000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":312,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.295971+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T15:00:05.295971+0000","last_peered":"2024-09-13T15:00:05.295971+0000","last_clean":"2024-09-13T15:00:05.295971+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T15:00:05.295971+0000","last_undegraded":"2024-09-13T15:00:05.295971+0000","last_fullsized":"2024-09-13T15:00:05.295971+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":541,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.375092+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T15:00:16.375092+0000","last_peered":"2024-09-13T15:00:16.375092+0000","last_clean":"2024-09-13T15:00:16.375092+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T15:00:16.375092+0000","last_undegraded":"2024-09-13T15:00:16.375092+0000","last_fullsized":"2024-09-13T15:00:16.375092+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.17","version":"82'100","reported_seq":171,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.292962+0000","last_change":"2024-09-13T14:59:53.249583+0000","last_active":"2024-09-13T15:00:05.292962+0000","last_peered":"2024-09-13T15:00:05.292962+0000","last_clean":"2024-09-13T15:00:05.292962+0000","last_became_active":"2024-09-13T14:59:53.249177+0000","last_became_peered":"2024-09-13T14:59:53.249177+0000","last_unstale":"2024-09-13T15:00:05.292962+0000","last_undegraded":"2024-09-13T15:00:05.292962+0000","last_fullsized":"2024-09-13T15:00:05.292962+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.916764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.916764+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0035298819999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":523,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.372300+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T15:00:16.372300+0000","last_peered":"2024-09-13T15:00:16.372300+0000","last_clean":"2024-09-13T15:00:16.372300+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T15:00:16.372300+0000","last_undegraded":"2024-09-13T15:00:16.372300+0000","last_fullsized":"2024-09-13T15:00:16.372300+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":318,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.292949+0000","last_change":"2024-09-13T14:59:00.929406+0000","last_active":"2024-09-13T15:00:05.292949+0000","last_peered":"2024-09-13T15:00:05.292949+0000","last_clean":"2024-09-13T15:00:05.292949+0000","last_became_active":"2024-09-13T14:59:00.928524+0000","last_became_peered":"2024-09-13T14:59:00.928524+0000","last_unstale":"2024-09-13T15:00:05.292949+0000","last_undegraded":"2024-09-13T15:00:05.292949+0000","last_fullsized":"2024-09-13T15:00:05.292949+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:10.609448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":103,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.376459+0000","last_change":"2024-09-13T14:56:01.235655+0000","last_active":"2024-09-13T15:00:16.376459+0000","last_peered":"2024-09-13T15:00:16.376459+0000","last_clean":"2024-09-13T15:00:16.376459+0000","last_became_active":"2024-09-13T14:55:55.195703+0000","last_became_peered":"2024-09-13T14:55:55.195703+0000","last_unstale":"2024-09-13T15:00:16.376459+0000","last_undegraded":"2024-09-13T15:00:16.376459+0000","last_fullsized":"2024-09-13T15:00:16.376459+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.235497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.235497+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:39:20.369406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":292,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.295477+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T15:00:05.295477+0000","last_peered":"2024-09-13T15:00:05.295477+0000","last_clean":"2024-09-13T15:00:05.295477+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T15:00:05.295477+0000","last_undegraded":"2024-09-13T15:00:05.295477+0000","last_fullsized":"2024-09-13T15:00:05.295477+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":299,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.291093+0000","last_change":"2024-09-13T14:59:00.926586+0000","last_active":"2024-09-13T15:00:05.291093+0000","last_peered":"2024-09-13T15:00:05.291093+0000","last_clean":"2024-09-13T15:00:05.291093+0000","last_became_active":"2024-09-13T14:59:00.926206+0000","last_became_peered":"2024-09-13T14:59:00.926206+0000","last_unstale":"2024-09-13T15:00:05.291093+0000","last_undegraded":"2024-09-13T15:00:05.291093+0000","last_fullsized":"2024-09-13T15:00:05.291093+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:17:18.805856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.15","version":"82'100","reported_seq":288,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296561+0000","last_change":"2024-09-13T14:59:53.225618+0000","last_active":"2024-09-13T15:00:05.296561+0000","last_peered":"2024-09-13T15:00:05.296561+0000","last_clean":"2024-09-13T15:00:05.296561+0000","last_became_active":"2024-09-13T14:59:53.225333+0000","last_became_peered":"2024-09-13T14:59:53.225333+0000","last_unstale":"2024-09-13T15:00:05.296561+0000","last_undegraded":"2024-09-13T15:00:05.296561+0000","last_fullsized":"2024-09-13T15:00:05.296561+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.308240+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.308240+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063105979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":528,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.375307+0000","last_change":"2024-09-13T14:59:00.932503+0000","last_active":"2024-09-13T15:00:16.375307+0000","last_peered":"2024-09-13T15:00:16.375307+0000","last_clean":"2024-09-13T15:00:16.375307+0000","last_became_active":"2024-09-13T14:59:00.931358+0000","last_became_peered":"2024-09-13T14:59:00.931358+0000","last_unstale":"2024-09-13T15:00:16.375307+0000","last_undegraded":"2024-09-13T15:00:16.375307+0000","last_fullsized":"2024-09-13T15:00:16.375307+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:19.226182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":360,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.371504+0000","last_change":"2024-09-13T14:59:00.927060+0000","last_active":"2024-09-13T15:00:16.371504+0000","last_peered":"2024-09-13T15:00:16.371504+0000","last_clean":"2024-09-13T15:00:16.371504+0000","last_became_active":"2024-09-13T14:59:00.925402+0000","last_became_peered":"2024-09-13T14:59:00.925402+0000","last_unstale":"2024-09-13T15:00:16.371504+0000","last_undegraded":"2024-09-13T15:00:16.371504+0000","last_fullsized":"2024-09-13T15:00:16.371504+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:31:08.336446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"82'100","reported_seq":247,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296621+0000","last_change":"2024-09-13T14:59:53.238757+0000","last_active":"2024-09-13T15:00:05.296621+0000","last_peered":"2024-09-13T15:00:05.296621+0000","last_clean":"2024-09-13T15:00:05.296621+0000","last_became_active":"2024-09-13T14:59:53.238460+0000","last_became_peered":"2024-09-13T14:59:53.238460+0000","last_unstale":"2024-09-13T15:00:05.296621+0000","last_undegraded":"2024-09-13T15:00:05.296621+0000","last_fullsized":"2024-09-13T15:00:05.296621+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:25.861193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:25.861193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042074019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":112,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.293731+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T15:00:05.293731+0000","last_peered":"2024-09-13T15:00:05.293731+0000","last_clean":"2024-09-13T15:00:05.293731+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T15:00:05.293731+0000","last_undegraded":"2024-09-13T15:00:05.293731+0000","last_fullsized":"2024-09-13T15:00:05.293731+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":118,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.373655+0000","last_change":"2024-09-13T14:59:00.924196+0000","last_active":"2024-09-13T15:00:16.373655+0000","last_peered":"2024-09-13T15:00:16.373655+0000","last_clean":"2024-09-13T15:00:16.373655+0000","last_became_active":"2024-09-13T14:59:00.923901+0000","last_became_peered":"2024-09-13T14:59:00.923901+0000","last_unstale":"2024-09-13T15:00:16.373655+0000","last_undegraded":"2024-09-13T15:00:16.373655+0000","last_fullsized":"2024-09-13T15:00:16.373655+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:01.913804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"82'100","reported_seq":700,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.371819+0000","last_change":"2024-09-13T15:00:05.698412+0000","last_active":"2024-09-13T15:00:16.371819+0000","last_peered":"2024-09-13T15:00:16.371819+0000","last_clean":"2024-09-13T15:00:16.371819+0000","last_became_active":"2024-09-13T14:59:53.217818+0000","last_became_peered":"2024-09-13T14:59:53.217818+0000","last_unstale":"2024-09-13T15:00:16.371819+0000","last_undegraded":"2024-09-13T15:00:16.371819+0000","last_fullsized":"2024-09-13T15:00:16.371819+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:05.698273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:05.698273+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:50:41.295785+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":552,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.375191+0000","last_change":"2024-09-13T14:59:00.924814+0000","last_active":"2024-09-13T15:00:16.375191+0000","last_peered":"2024-09-13T15:00:16.375191+0000","last_clean":"2024-09-13T15:00:16.375191+0000","last_became_active":"2024-09-13T14:59:00.924501+0000","last_became_peered":"2024-09-13T14:59:00.924501+0000","last_unstale":"2024-09-13T15:00:16.375191+0000","last_undegraded":"2024-09-13T15:00:16.375191+0000","last_fullsized":"2024-09-13T15:00:16.375191+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:11:15.382207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":516,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.371620+0000","last_change":"2024-09-13T14:59:00.925431+0000","last_active":"2024-09-13T15:00:16.371620+0000","last_peered":"2024-09-13T15:00:16.371620+0000","last_clean":"2024-09-13T15:00:16.371620+0000","last_became_active":"2024-09-13T14:59:00.925023+0000","last_became_peered":"2024-09-13T14:59:00.925023+0000","last_unstale":"2024-09-13T15:00:16.371620+0000","last_undegraded":"2024-09-13T15:00:16.371620+0000","last_fullsized":"2024-09-13T15:00:16.371620+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:38:18.686253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"82'100","reported_seq":259,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296391+0000","last_change":"2024-09-13T14:59:53.247281+0000","last_active":"2024-09-13T15:00:05.296391+0000","last_peered":"2024-09-13T15:00:05.296391+0000","last_clean":"2024-09-13T15:00:05.296391+0000","last_became_active":"2024-09-13T14:59:53.246955+0000","last_became_peered":"2024-09-13T14:59:53.246955+0000","last_unstale":"2024-09-13T15:00:05.296391+0000","last_undegraded":"2024-09-13T15:00:05.296391+0000","last_fullsized":"2024-09-13T15:00:05.296391+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:27.868313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:27.868313+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0099069050000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":131344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":102,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296315+0000","last_change":"2024-09-13T14:59:00.927250+0000","last_active":"2024-09-13T15:00:05.296315+0000","last_peered":"2024-09-13T15:00:05.296315+0000","last_clean":"2024-09-13T15:00:05.296315+0000","last_became_active":"2024-09-13T14:59:00.926953+0000","last_became_peered":"2024-09-13T14:59:00.926953+0000","last_unstale":"2024-09-13T15:00:05.296315+0000","last_undegraded":"2024-09-13T15:00:05.296315+0000","last_fullsized":"2024-09-13T15:00:05.296315+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:58.306547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":311,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296062+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T15:00:05.296062+0000","last_peered":"2024-09-13T15:00:05.296062+0000","last_clean":"2024-09-13T15:00:05.296062+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T15:00:05.296062+0000","last_undegraded":"2024-09-13T15:00:05.296062+0000","last_fullsized":"2024-09-13T15:00:05.296062+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"82'100","reported_seq":494,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.290800+0000","last_change":"2024-09-13T14:59:53.234993+0000","last_active":"2024-09-13T15:00:05.290800+0000","last_peered":"2024-09-13T15:00:05.290800+0000","last_clean":"2024-09-13T15:00:05.290800+0000","last_became_active":"2024-09-13T14:59:53.234573+0000","last_became_peered":"2024-09-13T14:59:53.234573+0000","last_unstale":"2024-09-13T15:00:05.290800+0000","last_undegraded":"2024-09-13T15:00:05.290800+0000","last_fullsized":"2024-09-13T15:00:05.290800+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:40:32.590146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:40:32.590146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0035160629999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":350,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.372970+0000","last_change":"2024-09-13T14:59:00.934412+0000","last_active":"2024-09-13T15:00:16.372970+0000","last_peered":"2024-09-13T15:00:16.372970+0000","last_clean":"2024-09-13T15:00:16.372970+0000","last_became_active":"2024-09-13T14:59:00.932986+0000","last_became_peered":"2024-09-13T14:59:00.932986+0000","last_unstale":"2024-09-13T15:00:16.372970+0000","last_undegraded":"2024-09-13T15:00:16.372970+0000","last_fullsized":"2024-09-13T15:00:16.372970+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:21:44.285324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":513,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.374099+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T15:00:16.374099+0000","last_peered":"2024-09-13T15:00:16.374099+0000","last_clean":"2024-09-13T15:00:16.374099+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T15:00:16.374099+0000","last_undegraded":"2024-09-13T15:00:16.374099+0000","last_fullsized":"2024-09-13T15:00:16.374099+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"82'100","reported_seq":77,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.296881+0000","last_change":"2024-09-13T14:59:53.230150+0000","last_active":"2024-09-13T15:00:05.296881+0000","last_peered":"2024-09-13T15:00:05.296881+0000","last_clean":"2024-09-13T15:00:05.296881+0000","last_became_active":"2024-09-13T14:59:53.229606+0000","last_became_peered":"2024-09-13T14:59:53.229606+0000","last_unstale":"2024-09-13T15:00:05.296881+0000","last_undegraded":"2024-09-13T15:00:05.296881+0000","last_fullsized":"2024-09-13T15:00:05.296881+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:02.325629+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:02.325629+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0049176200000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":319,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.295898+0000","last_change":"2024-09-13T14:59:00.929751+0000","last_active":"2024-09-13T15:00:05.295898+0000","last_peered":"2024-09-13T15:00:05.295898+0000","last_clean":"2024-09-13T15:00:05.295898+0000","last_became_active":"2024-09-13T14:59:00.929456+0000","last_became_peered":"2024-09-13T14:59:00.929456+0000","last_unstale":"2024-09-13T15:00:05.295898+0000","last_undegraded":"2024-09-13T15:00:05.295898+0000","last_fullsized":"2024-09-13T15:00:05.295898+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:11.462853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":294,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.295093+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T15:00:05.295093+0000","last_peered":"2024-09-13T15:00:05.295093+0000","last_clean":"2024-09-13T15:00:05.295093+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T15:00:05.295093+0000","last_undegraded":"2024-09-13T15:00:05.295093+0000","last_fullsized":"2024-09-13T15:00:05.295093+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":319,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.295971+0000","last_change":"2024-09-13T14:59:00.928898+0000","last_active":"2024-09-13T15:00:05.295971+0000","last_peered":"2024-09-13T15:00:05.295971+0000","last_clean":"2024-09-13T15:00:05.295971+0000","last_became_active":"2024-09-13T14:59:00.928585+0000","last_became_peered":"2024-09-13T14:59:00.928585+0000","last_unstale":"2024-09-13T15:00:05.295971+0000","last_undegraded":"2024-09-13T15:00:05.295971+0000","last_fullsized":"2024-09-13T15:00:05.295971+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:26:57.155149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":333,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.295796+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T15:00:05.295796+0000","last_peered":"2024-09-13T15:00:05.295796+0000","last_clean":"2024-09-13T15:00:05.295796+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T15:00:05.295796+0000","last_undegraded":"2024-09-13T15:00:05.295796+0000","last_fullsized":"2024-09-13T15:00:05.295796+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"82'100","reported_seq":467,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.374693+0000","last_change":"2024-09-13T15:00:06.945603+0000","last_active":"2024-09-13T15:00:16.374693+0000","last_peered":"2024-09-13T15:00:16.374693+0000","last_clean":"2024-09-13T15:00:16.374693+0000","last_became_active":"2024-09-13T14:59:53.227869+0000","last_became_peered":"2024-09-13T14:59:53.227869+0000","last_unstale":"2024-09-13T15:00:16.374693+0000","last_undegraded":"2024-09-13T15:00:16.374693+0000","last_fullsized":"2024-09-13T15:00:16.374693+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:06.945525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:06.945525+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:23:33.636855+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":84,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.373937+0000","last_change":"2024-09-13T14:59:00.929835+0000","last_active":"2024-09-13T15:00:16.373937+0000","last_peered":"2024-09-13T15:00:16.373937+0000","last_clean":"2024-09-13T15:00:16.373937+0000","last_became_active":"2024-09-13T14:59:00.929514+0000","last_became_peered":"2024-09-13T14:59:00.929514+0000","last_unstale":"2024-09-13T15:00:16.373937+0000","last_undegraded":"2024-09-13T15:00:16.373937+0000","last_fullsized":"2024-09-13T15:00:16.373937+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:48.981240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":355,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.293230+0000","last_change":"2024-09-13T14:59:00.925895+0000","last_active":"2024-09-13T15:00:05.293230+0000","last_peered":"2024-09-13T15:00:05.293230+0000","last_clean":"2024-09-13T15:00:05.293230+0000","last_became_active":"2024-09-13T14:59:00.925564+0000","last_became_peered":"2024-09-13T14:59:00.925564+0000","last_unstale":"2024-09-13T15:00:05.293230+0000","last_undegraded":"2024-09-13T15:00:05.293230+0000","last_fullsized":"2024-09-13T15:00:05.293230+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:23.283927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1d","version":"82'100","reported_seq":67,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.291303+0000","last_change":"2024-09-13T14:59:53.223903+0000","last_active":"2024-09-13T15:00:05.291303+0000","last_peered":"2024-09-13T15:00:05.291303+0000","last_clean":"2024-09-13T15:00:05.291303+0000","last_became_active":"2024-09-13T14:59:53.223040+0000","last_became_peered":"2024-09-13T14:59:53.223040+0000","last_unstale":"2024-09-13T15:00:05.291303+0000","last_undegraded":"2024-09-13T15:00:05.291303+0000","last_fullsized":"2024-09-13T15:00:05.291303+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.654717+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.654717+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0060902070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":315,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.295288+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T15:00:05.295288+0000","last_peered":"2024-09-13T15:00:05.295288+0000","last_clean":"2024-09-13T15:00:05.295288+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T15:00:05.295288+0000","last_undegraded":"2024-09-13T15:00:05.295288+0000","last_fullsized":"2024-09-13T15:00:05.295288+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":565,"reported_epoch":328,"state":"active+clean","last_fresh":"2024-09-13T15:00:16.375248+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T15:00:16.375248+0000","last_peered":"2024-09-13T15:00:16.375248+0000","last_clean":"2024-09-13T15:00:16.375248+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T15:00:16.375248+0000","last_undegraded":"2024-09-13T15:00:16.375248+0000","last_fullsized":"2024-09-13T15:00:16.375248+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4501733400,"num_objects":68692,"num_object_clones":0,"num_object_copies":206076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":37161,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":68692,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":170120,"num_write_kb":7641921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8897,"num_bytes_recovered":583008256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15313743872,"data_stored":15313731656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94035,"ondisk_log_size":94035,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":16172,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":608258,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":9,"num_read_kb":22,"num_write":4,"num_write_kb":49,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":280578,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397917,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3388792,"kb_used_data":2507632,"kb_used_omap":1,"kb_used_meta":881086,"kb_avail":90356360,"statfs":{"total":95995035648,"available":92524912640,"internally_reserved":0,"allocated":2567815168,"data_stored":2565551045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":902232408},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,3,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594146,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1159152,"kb_used_data":849320,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92586000,"statfs":{"total":95995035648,"available":94808064000,"internally_reserved":0,"allocated":869703680,"data_stored":867447841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790375,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3688812,"kb_used_data":2795940,"kb_used_omap":1,"kb_used_meta":892862,"kb_avail":90056340,"statfs":{"total":95995035648,"available":92217692160,"internally_reserved":0,"allocated":2863042560,"data_stored":2860786738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":914291032},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,2,0,3,6,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986604,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1777384,"kb_used_data":884320,"kb_used_omap":1,"kb_used_meta":893054,"kb_avail":91967768,"statfs":{"total":95995035648,"available":94174994432,"internally_reserved":0,"allocated":905543680,"data_stored":903291117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":914487640},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":0,"seq":446676599037,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762579,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1799656,"kb_used_data":946656,"kb_used_omap":8,"kb_used_meta":852983,"kb_avail":91945496,"statfs":{"total":95995035648,"available":94152187904,"internally_reserved":0,"allocated":969375744,"data_stored":967123989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":873455050},"hb_peers":[0,1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926105,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3285348,"kb_used_data":2465564,"kb_used_omap":7,"kb_used_meta":819768,"kb_avail":90459804,"statfs":{"total":95995035648,"available":92630839296,"internally_reserved":0,"allocated":2524737536,"data_stored":2522488994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":839443145},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122345,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1850100,"kb_used_data":1144364,"kb_used_omap":7,"kb_used_meta":705720,"kb_avail":91895052,"statfs":{"total":95995035648,"available":94100533248,"internally_reserved":0,"allocated":1171828736,"data_stored":1169569193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":722657871},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1162215424,"data_stored":1162215424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2510753792,"data_stored":2510749720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":963313664,"data_stored":963313664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3548708864,"data_stored":3548708864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":896532480,"data_stored":896532480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2831618048,"data_stored":2831613976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":855900160,"data_stored":855900160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2544701440,"data_stored":2544697368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:00:29.077 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_4 pgp_num 16 2024-09-13T15:00:29.137 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:00:29.144 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:29.167 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:28 smithi096 ceph-mon[98901]: from='client.36369 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:00:29.168 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:28 smithi096 ceph-mon[102928]: from='client.36369 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:00:29.182 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:29.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:29.284 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:29.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.322 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:29.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:29.424 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:29.471 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:29.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:29.473 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:00:29.573 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:29.612 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:29.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:29.714 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:29.732 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:29.746 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:29.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.760 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:29.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:29.862 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:29.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.898 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:29.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:29.999 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:30.029 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:30.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:30.131 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:30.145 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:29 smithi204 ceph-mon[87071]: pgmap v1085: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 928 op/s; 36750/198729 objects degraded (18.493%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T15:00:30.146 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:29 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 36750/198729 objects degraded (18.493%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:30.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.168 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:30.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:30.204 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:29 smithi096 ceph-mon[98901]: pgmap v1085: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 928 op/s; 36750/198729 objects degraded (18.493%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T15:00:30.204 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:29 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 36750/198729 objects degraded (18.493%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:30.204 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:29 smithi096 ceph-mon[102928]: pgmap v1085: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 928 op/s; 36750/198729 objects degraded (18.493%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T15:00:30.204 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:29 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 36750/198729 objects degraded (18.493%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:30.268 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:30.295 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:30.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:30.397 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:30.424 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:00:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:00:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:30.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.445 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:30.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:30.546 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:30.573 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:30.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:30.675 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:30.706 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:30.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:30.808 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:30.845 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:30.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:30.910 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T15:00:30.947 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:30.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.987 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:30.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:31.088 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:31.117 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:31.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:31.215 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:30 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1934838938' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:00:31.215 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:30 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:00:31.215 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:30 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1934838938' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:00:31.215 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:30 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:00:31.218 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:31.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:30 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1934838938' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:00:31.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:30 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:00:31.265 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:31.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:31.367 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:31.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.405 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:31.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:31.506 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:31.531 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:31.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:31.633 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:31.659 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:31.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:31.761 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:31.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.799 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:31.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:31.901 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:31.905 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:00:31.905 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:00:31.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.943 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:31.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:32.044 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:32.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:31 smithi096 ceph-mon[98901]: pgmap v1086: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 861 op/s; 36241/193065 objects degraded (18.771%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T15:00:32.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:31 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:32.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:31 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:00:32.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:31 smithi096 ceph-mon[98901]: osdmap e329: 8 total, 7 up, 7 in 2024-09-13T15:00:32.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:31 smithi096 ceph-mon[102928]: pgmap v1086: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 861 op/s; 36241/193065 objects degraded (18.771%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T15:00:32.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:31 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:32.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:31 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:00:32.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:31 smithi096 ceph-mon[102928]: osdmap e329: 8 total, 7 up, 7 in 2024-09-13T15:00:32.076 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:32.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:32.165 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:31 smithi204 ceph-mon[87071]: pgmap v1086: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 861 op/s; 36241/193065 objects degraded (18.771%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T15:00:32.165 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:31 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:32.165 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:31 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:00:32.165 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:31 smithi204 ceph-mon[87071]: osdmap e329: 8 total, 7 up, 7 in 2024-09-13T15:00:32.179 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:32.218 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:32.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:32.320 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:32.354 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:32.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:32.456 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:32.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.496 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:32.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:32.597 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:32.622 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T15:00:32.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.637 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:32.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:32.738 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:32.786 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:32.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:32.888 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:32.916 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:32.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:33.018 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:33.047 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:33.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:33.149 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:33.152 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:32 smithi096 ceph-mon[98901]: pgmap v1088: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 848 op/s; 36082/192264 objects degraded (18.767%); 1.9 MiB/s, 31 objects/s recovering 2024-09-13T15:00:33.152 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:32 smithi096 ceph-mon[102928]: pgmap v1088: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 848 op/s; 36082/192264 objects degraded (18.767%); 1.9 MiB/s, 31 objects/s recovering 2024-09-13T15:00:33.162 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:00:33.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.175 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:33.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:33.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:32 smithi204 ceph-mon[87071]: pgmap v1088: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 848 op/s; 36082/192264 objects degraded (18.767%); 1.9 MiB/s, 31 objects/s recovering 2024-09-13T15:00:33.275 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:33.305 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:33.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:33.407 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:33.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.434 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:33.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:33.535 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:33.570 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:33.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:33.621 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:00:33.621 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:00:33.621 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:00:33.621 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:00:33.621 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:00:33.621 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:00:33.621 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:00:33.621 INFO:teuthology.orchestra.run.smithi096.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:00:33.622 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:00:33.622 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:00:33.622 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:00:33.622 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:00:33.622 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:00:33.624 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:00:33.624 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:00:33.672 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:33.699 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:33.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:33.801 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:33.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.830 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:33.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:33.931 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:33.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.967 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:33.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:34.069 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:34.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.093 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:34.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:34.194 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:34.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:33 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3579383311' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:34.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.230 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:34.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:34.277 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:33 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3579383311' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:34.277 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:33 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3579383311' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:34.319 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:00:34.320 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:00:34.330 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:34.354 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:34.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:34.456 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:34.484 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:34.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:34.586 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:34.620 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:34.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:34.721 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:34.748 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:34.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:34.850 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:34.885 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:34.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:34.987 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:35.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.021 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:35.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:35.122 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:35.151 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:35.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:35.171 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:34 smithi096 ceph-mon[98901]: pgmap v1089: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 35546/183591 objects degraded (19.362%); 1.9 MiB/s, 31 objects/s recovering 2024-09-13T15:00:35.171 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:34 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 36082/192264 objects degraded (18.767%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:35.171 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:34 smithi096 ceph-mon[102928]: pgmap v1089: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 35546/183591 objects degraded (19.362%); 1.9 MiB/s, 31 objects/s recovering 2024-09-13T15:00:35.172 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:34 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 36082/192264 objects degraded (18.767%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:35.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:34 smithi204 ceph-mon[87071]: pgmap v1089: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 35546/183591 objects degraded (19.362%); 1.9 MiB/s, 31 objects/s recovering 2024-09-13T15:00:35.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:34 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 36082/192264 objects degraded (18.767%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:35.253 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:35.284 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:35.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:35.386 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:35.421 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:35.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:35.523 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:35.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.557 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:35.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:35.658 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:35.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.696 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:35.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:35.797 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:35.820 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:35.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:35.921 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:35.943 INFO:teuthology.orchestra.run.smithi096.stderr:set pool 9 pgp_num to 16 2024-09-13T15:00:35.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.957 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:35.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:36.058 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:36.096 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:35 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/541967699' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T15:00:36.096 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:35 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T15:00:36.096 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:36.097 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:36.097 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:35 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/541967699' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T15:00:36.097 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:35 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T15:00:36.097 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:36.097 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:36.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.122 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:36.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:35 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/541967699' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T15:00:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:35 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T15:00:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:36.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:36.223 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:36.263 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:36.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:36.320 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:00:36.365 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:36.402 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:36.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:36.504 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:36.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.531 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:36.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:36.632 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:36.670 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:36.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:36.773 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:36.804 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:36.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:36.905 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:36.914 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:00:36.938 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:36.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:37.040 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:37.066 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:37.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:37.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:36 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T15:00:37.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:36 smithi096 ceph-mon[98901]: osdmap e330: 8 total, 7 up, 7 in 2024-09-13T15:00:37.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:36 smithi096 ceph-mon[98901]: pgmap v1091: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 35266/180636 objects degraded (19.523%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:00:37.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:36 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T15:00:37.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:36 smithi096 ceph-mon[102928]: osdmap e330: 8 total, 7 up, 7 in 2024-09-13T15:00:37.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:36 smithi096 ceph-mon[102928]: pgmap v1091: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 35266/180636 objects degraded (19.523%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:00:37.168 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:37.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.200 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:37.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:36 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T15:00:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:36 smithi204 ceph-mon[87071]: osdmap e330: 8 total, 7 up, 7 in 2024-09-13T15:00:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:36 smithi204 ceph-mon[87071]: pgmap v1091: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 35266/180636 objects degraded (19.523%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:00:37.301 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:37.336 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:37.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:37.438 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:37.466 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:37.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:37.542 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:37.568 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:37.624 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:37.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:37.726 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:37.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.755 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:37.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:37.856 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:37.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.890 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:37.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:37.991 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:38.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.028 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:38.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:38.129 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:38.166 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:38.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:38.249 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:38.268 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:38.289 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:38.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:38.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:00:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:00:37] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.51.0" 2024-09-13T15:00:38.391 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:38.432 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:38.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:38.534 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:38.567 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:38.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:38.669 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:38.699 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:38.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:38.773 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:38.774 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:38.774 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:38.774 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:38.801 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:38.830 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:38.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:38.932 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:38.966 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:38.966 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:38.973 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:38.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:39.075 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:39.105 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:39.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:39.207 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:39.239 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:39.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:39.341 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:39.370 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:39.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:39.472 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:39.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.510 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:39.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:39.611 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:39.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.644 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:39.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:39.745 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:39.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.789 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:39.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:39.890 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:39.900 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:39 smithi204 ceph-mon[87071]: pgmap v1092: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 789 op/s; 34826/174111 objects degraded (20.002%); 2.7 MiB/s, 42 objects/s recovering 2024-09-13T15:00:39.900 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:39 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:00:39.900 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:39 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:00:39.900 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:39 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:39.900 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:39 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 34826/174111 objects degraded (20.002%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:39.938 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:39.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:40.040 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:40.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:39 smithi096 ceph-mon[98901]: pgmap v1092: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 789 op/s; 34826/174111 objects degraded (20.002%); 2.7 MiB/s, 42 objects/s recovering 2024-09-13T15:00:40.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:39 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:00:40.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:39 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:00:40.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:39 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:40.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:39 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 34826/174111 objects degraded (20.002%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:40.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:39 smithi096 ceph-mon[102928]: pgmap v1092: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 789 op/s; 34826/174111 objects degraded (20.002%); 2.7 MiB/s, 42 objects/s recovering 2024-09-13T15:00:40.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:39 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:00:40.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:39 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:00:40.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:39 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:00:40.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:39 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 34826/174111 objects degraded (20.002%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:40.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.090 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:40.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:40.191 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:40.219 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:40.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:40.321 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:40.349 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:40.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:40.444 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:00:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:00:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:40.451 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:40.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.495 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:40.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:40.596 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:40.604 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:00:40.604 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T15:00:40.604 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T15:00:40.604 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T15:00:40.604 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.604 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169258 7.1 7:89ed45e8:::benchmark_data_smithi096_196313_object49127:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T15:00:40.604 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.061777+0000", 2024-09-13T15:00:40.604 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 570.53718253399995, 2024-09-13T15:00:40.604 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2721801959999999, 2024-09-13T15:00:40.605 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.605 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.605 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.605 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:00:40.605 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:00:40.605 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169258 2024-09-13T15:00:40.605 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.606 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.606 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.606 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.606 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061777+0000", 2024-09-13T15:00:40.606 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.606 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.606 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.606 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.606 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061777+0000", 2024-09-13T15:00:40.606 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.606 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.606 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.606 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.607 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061775+0000", 2024-09-13T15:00:40.607 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:40.607 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.607 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.607 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.607 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061780+0000", 2024-09-13T15:00:40.607 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4140000000000002e-06 2024-09-13T15:00:40.607 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.607 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.607 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.607 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061793+0000", 2024-09-13T15:00:40.607 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2444e-05 2024-09-13T15:00:40.607 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.607 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.608 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.608 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061800+0000", 2024-09-13T15:00:40.608 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8789999999999997e-06 2024-09-13T15:00:40.608 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.608 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.608 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.608 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061860+0000", 2024-09-13T15:00:40.608 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0251000000000003e-05 2024-09-13T15:00:40.608 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.608 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.608 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.608 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.061870+0000", 2024-09-13T15:00:40.608 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.8609999999999993e-06 2024-09-13T15:00:40.609 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.609 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.609 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.609 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.049861+0000", 2024-09-13T15:00:40.609 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.98799087500000005 2024-09-13T15:00:40.610 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.610 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.610 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.610 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.049872+0000", 2024-09-13T15:00:40.610 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1044e-05 2024-09-13T15:00:40.610 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.611 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.611 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.611 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.306782+0000", 2024-09-13T15:00:40.611 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.256909821 2024-09-13T15:00:40.611 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.611 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.612 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.612 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.306898+0000", 2024-09-13T15:00:40.612 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000116332 2024-09-13T15:00:40.612 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.612 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.612 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3,6", 2024-09-13T15:00:40.613 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.306989+0000", 2024-09-13T15:00:40.613 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1566999999999999e-05 2024-09-13T15:00:40.613 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.613 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.613 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.613 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.310582+0000", 2024-09-13T15:00:40.613 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035929719999999998 2024-09-13T15:00:40.613 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.613 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.614 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.614 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.318699+0000", 2024-09-13T15:00:40.614 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0081170830000000006 2024-09-13T15:00:40.614 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.614 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.614 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.614 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327316+0000", 2024-09-13T15:00:40.614 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0086162870000000003 2024-09-13T15:00:40.614 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.615 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.615 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.615 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.333834+0000", 2024-09-13T15:00:40.615 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0065179210000000003 2024-09-13T15:00:40.615 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.615 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.615 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.615 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.333866+0000", 2024-09-13T15:00:40.615 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2592000000000002e-05 2024-09-13T15:00:40.616 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.616 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.616 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.616 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.333957+0000", 2024-09-13T15:00:40.616 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.0482000000000005e-05 2024-09-13T15:00:40.617 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.617 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.617 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.617 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.617 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.617 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169259 7.1 7:8a1070f0:::benchmark_data_smithi096_196313_object49128:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T15:00:40.617 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.062386+0000", 2024-09-13T15:00:40.617 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 570.53657333700005, 2024-09-13T15:00:40.617 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.278361297, 2024-09-13T15:00:40.617 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.617 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.617 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.618 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:00:40.618 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:00:40.618 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169259 2024-09-13T15:00:40.618 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.618 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.618 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.618 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.618 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062386+0000", 2024-09-13T15:00:40.618 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.618 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.618 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.618 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.618 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062386+0000", 2024-09-13T15:00:40.618 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.619 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.619 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.619 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.619 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062385+0000", 2024-09-13T15:00:40.619 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:40.619 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.619 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.619 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.619 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062393+0000", 2024-09-13T15:00:40.619 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7110000000000001e-06 2024-09-13T15:00:40.619 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.619 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.620 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.620 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062408+0000", 2024-09-13T15:00:40.620 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4751e-05 2024-09-13T15:00:40.620 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.620 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.620 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.620 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062414+0000", 2024-09-13T15:00:40.621 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0399999999999998e-06 2024-09-13T15:00:40.621 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.621 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.621 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.621 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062473+0000", 2024-09-13T15:00:40.621 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8730999999999997e-05 2024-09-13T15:00:40.621 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.621 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.622 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.622 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.062478+0000", 2024-09-13T15:00:40.622 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5829999999999997e-06 2024-09-13T15:00:40.622 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.622 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.622 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.623 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.049956+0000", 2024-09-13T15:00:40.623 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.98747715400000002 2024-09-13T15:00:40.623 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.623 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.623 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.623 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.049974+0000", 2024-09-13T15:00:40.624 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8345000000000001e-05 2024-09-13T15:00:40.624 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.624 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.624 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.624 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.310608+0000", 2024-09-13T15:00:40.624 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.260634598 2024-09-13T15:00:40.624 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.624 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.625 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.625 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.310728+0000", 2024-09-13T15:00:40.625 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119803 2024-09-13T15:00:40.625 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.625 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.625 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3,6", 2024-09-13T15:00:40.627 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.310805+0000", 2024-09-13T15:00:40.627 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6697000000000001e-05 2024-09-13T15:00:40.627 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.628 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.628 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.628 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.326994+0000", 2024-09-13T15:00:40.628 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.01618909 2024-09-13T15:00:40.628 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.628 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.628 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.628 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327004+0000", 2024-09-13T15:00:40.628 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0219e-05 2024-09-13T15:00:40.628 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.628 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.629 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.629 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327369+0000", 2024-09-13T15:00:40.629 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000365125 2024-09-13T15:00:40.629 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.629 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.629 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.629 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.340622+0000", 2024-09-13T15:00:40.629 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013252956999999999 2024-09-13T15:00:40.629 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.630 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.630 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.630 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.340678+0000", 2024-09-13T15:00:40.630 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5451999999999997e-05 2024-09-13T15:00:40.630 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.630 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.630 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.630 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.340747+0000", 2024-09-13T15:00:40.630 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9251000000000005e-05 2024-09-13T15:00:40.630 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.630 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.630 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.631 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.631 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.631 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169261 7.1 7:8e31827b:::benchmark_data_smithi096_196313_object49130:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T15:00:40.631 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.063536+0000", 2024-09-13T15:00:40.631 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 570.535422864, 2024-09-13T15:00:40.631 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2803364450000001, 2024-09-13T15:00:40.631 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.631 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.631 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.631 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:00:40.631 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:00:40.631 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169261 2024-09-13T15:00:40.632 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.632 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.632 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.632 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.632 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063536+0000", 2024-09-13T15:00:40.632 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.632 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.632 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.632 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.632 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063536+0000", 2024-09-13T15:00:40.632 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.632 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.632 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.633 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.633 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063535+0000", 2024-09-13T15:00:40.633 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:00:40.633 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.633 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.633 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.633 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063541+0000", 2024-09-13T15:00:40.633 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3190000000000002e-06 2024-09-13T15:00:40.633 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.633 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.633 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.633 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063551+0000", 2024-09-13T15:00:40.633 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0671e-05 2024-09-13T15:00:40.633 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.634 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.634 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.634 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063557+0000", 2024-09-13T15:00:40.634 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.874e-06 2024-09-13T15:00:40.634 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.634 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.634 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.634 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063600+0000", 2024-09-13T15:00:40.634 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2549999999999997e-05 2024-09-13T15:00:40.634 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.635 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.635 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.635 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.063606+0000", 2024-09-13T15:00:40.635 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5490000000000003e-06 2024-09-13T15:00:40.635 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.635 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.635 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.635 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050009+0000", 2024-09-13T15:00:40.635 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.98640244499999996 2024-09-13T15:00:40.635 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.635 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.635 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.635 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050017+0000", 2024-09-13T15:00:40.635 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9740000000000002e-06 2024-09-13T15:00:40.636 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.636 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.636 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.636 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.314552+0000", 2024-09-13T15:00:40.636 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2645352220000001 2024-09-13T15:00:40.636 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.636 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.636 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.636 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.314667+0000", 2024-09-13T15:00:40.636 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000115258 2024-09-13T15:00:40.636 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.637 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.637 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3,6", 2024-09-13T15:00:40.637 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.314789+0000", 2024-09-13T15:00:40.637 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012234499999999999 2024-09-13T15:00:40.637 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.637 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.637 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.638 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327061+0000", 2024-09-13T15:00:40.638 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012271855999999999 2024-09-13T15:00:40.638 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.638 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.638 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.638 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327269+0000", 2024-09-13T15:00:40.638 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00020811700000000001 2024-09-13T15:00:40.638 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.638 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.638 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.638 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.340791+0000", 2024-09-13T15:00:40.638 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013521403 2024-09-13T15:00:40.638 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.639 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.639 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.639 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.343708+0000", 2024-09-13T15:00:40.639 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029173559999999999 2024-09-13T15:00:40.639 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.639 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.639 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.639 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.343759+0000", 2024-09-13T15:00:40.639 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0615000000000002e-05 2024-09-13T15:00:40.639 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.639 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.639 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.640 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.343873+0000", 2024-09-13T15:00:40.640 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000113931 2024-09-13T15:00:40.640 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.640 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.640 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.640 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.640 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.640 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169277 7.1 7:866405f8:::benchmark_data_smithi096_196313_object49146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T15:00:40.640 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.098666+0000", 2024-09-13T15:00:40.640 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 570.50029270899995, 2024-09-13T15:00:40.641 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.249298177, 2024-09-13T15:00:40.641 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.641 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.641 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.641 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:00:40.641 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:00:40.641 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169277 2024-09-13T15:00:40.641 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.641 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.641 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.641 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.642 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098666+0000", 2024-09-13T15:00:40.642 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.642 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.642 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.642 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.642 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098666+0000", 2024-09-13T15:00:40.642 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.642 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.642 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.642 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.642 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098665+0000", 2024-09-13T15:00:40.642 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:40.642 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.642 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.642 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.643 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098671+0000", 2024-09-13T15:00:40.643 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7470000000000002e-06 2024-09-13T15:00:40.643 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.643 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.643 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.643 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098683+0000", 2024-09-13T15:00:40.643 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2499e-05 2024-09-13T15:00:40.643 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.643 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.643 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.644 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098689+0000", 2024-09-13T15:00:40.644 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.913e-06 2024-09-13T15:00:40.644 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.644 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.644 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.644 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098749+0000", 2024-09-13T15:00:40.644 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0109000000000003e-05 2024-09-13T15:00:40.644 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.644 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.644 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.644 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.098760+0000", 2024-09-13T15:00:40.644 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1022000000000001e-05 2024-09-13T15:00:40.644 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.645 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.645 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.645 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050038+0000", 2024-09-13T15:00:40.645 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95127815999999998 2024-09-13T15:00:40.645 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.645 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.645 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.646 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050045+0000", 2024-09-13T15:00:40.646 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0580000000000003e-06 2024-09-13T15:00:40.646 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.646 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.646 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.646 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.318726+0000", 2024-09-13T15:00:40.646 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2686819759999999 2024-09-13T15:00:40.646 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.646 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.646 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.646 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.318865+0000", 2024-09-13T15:00:40.646 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013841099999999999 2024-09-13T15:00:40.646 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.646 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.647 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3,6", 2024-09-13T15:00:40.647 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.318984+0000", 2024-09-13T15:00:40.647 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119265 2024-09-13T15:00:40.647 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.647 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.647 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.647 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327276+0000", 2024-09-13T15:00:40.647 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0082922559999999996 2024-09-13T15:00:40.647 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.647 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.648 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.648 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.333803+0000", 2024-09-13T15:00:40.648 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0065261069999999997 2024-09-13T15:00:40.648 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.648 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.648 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.648 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.340820+0000", 2024-09-13T15:00:40.648 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0070172510000000004 2024-09-13T15:00:40.649 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.649 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.649 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.649 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.347790+0000", 2024-09-13T15:00:40.649 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0069697939999999996 2024-09-13T15:00:40.649 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.649 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.649 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.649 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.347845+0000", 2024-09-13T15:00:40.649 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5781e-05 2024-09-13T15:00:40.649 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.649 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.650 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.650 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.347965+0000", 2024-09-13T15:00:40.650 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119197 2024-09-13T15:00:40.650 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.650 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.650 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.650 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.650 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.650 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169282 7.1 7:8cccf50f:::benchmark_data_smithi096_196313_object49151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T15:00:40.650 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.109522+0000", 2024-09-13T15:00:40.650 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 570.48943671699999, 2024-09-13T15:00:40.650 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.246319723, 2024-09-13T15:00:40.650 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.650 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.651 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.651 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:00:40.651 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:00:40.651 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169282 2024-09-13T15:00:40.651 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.651 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.651 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.651 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.651 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109522+0000", 2024-09-13T15:00:40.651 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.651 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.651 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.651 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.651 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109522+0000", 2024-09-13T15:00:40.652 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.652 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.652 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.652 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.652 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109520+0000", 2024-09-13T15:00:40.652 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:00:40.652 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.652 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.652 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.652 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109530+0000", 2024-09-13T15:00:40.652 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1239999999999994e-06 2024-09-13T15:00:40.652 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.652 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.652 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.653 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109550+0000", 2024-09-13T15:00:40.653 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0876000000000001e-05 2024-09-13T15:00:40.653 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.653 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.653 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.653 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109559+0000", 2024-09-13T15:00:40.653 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2709999999999992e-06 2024-09-13T15:00:40.653 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.653 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.653 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.653 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109616+0000", 2024-09-13T15:00:40.653 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7714999999999999e-05 2024-09-13T15:00:40.653 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.654 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.654 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.654 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.109627+0000", 2024-09-13T15:00:40.654 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0054999999999999e-05 2024-09-13T15:00:40.654 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.654 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.654 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.654 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050065+0000", 2024-09-13T15:00:40.654 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.94043828399999996 2024-09-13T15:00:40.654 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.655 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.655 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.655 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050071+0000", 2024-09-13T15:00:40.655 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2489999999999998e-06 2024-09-13T15:00:40.655 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.655 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.655 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.655 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.322884+0000", 2024-09-13T15:00:40.655 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2728127630000001 2024-09-13T15:00:40.655 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.655 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.656 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.656 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.323002+0000", 2024-09-13T15:00:40.656 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011807999999999999 2024-09-13T15:00:40.656 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.656 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.656 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3,6", 2024-09-13T15:00:40.656 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.323120+0000", 2024-09-13T15:00:40.656 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000117837 2024-09-13T15:00:40.656 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.656 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.656 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.656 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.333809+0000", 2024-09-13T15:00:40.657 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010688848000000001 2024-09-13T15:00:40.657 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.657 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.657 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.657 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.333996+0000", 2024-09-13T15:00:40.657 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018746 2024-09-13T15:00:40.657 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.657 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.657 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.657 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.348004+0000", 2024-09-13T15:00:40.658 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.014007798 2024-09-13T15:00:40.658 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.658 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.658 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.658 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.355709+0000", 2024-09-13T15:00:40.658 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0077048639999999996 2024-09-13T15:00:40.658 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.658 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.658 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.658 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.355756+0000", 2024-09-13T15:00:40.658 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6893e-05 2024-09-13T15:00:40.658 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.658 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.658 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.659 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.355842+0000", 2024-09-13T15:00:40.659 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.6512000000000001e-05 2024-09-13T15:00:40.659 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.659 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.659 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.659 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.659 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.659 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169285 7.1 7:8fd36576:::benchmark_data_smithi096_196313_object49154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T15:00:40.659 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.113664+0000", 2024-09-13T15:00:40.659 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 570.48529508700005, 2024-09-13T15:00:40.659 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2423260599999999, 2024-09-13T15:00:40.659 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.660 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.660 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.660 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:00:40.660 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:00:40.660 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169285 2024-09-13T15:00:40.660 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.660 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.660 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.660 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.660 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113664+0000", 2024-09-13T15:00:40.660 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.660 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.660 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.661 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.661 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113664+0000", 2024-09-13T15:00:40.661 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.661 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.661 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.661 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.661 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113662+0000", 2024-09-13T15:00:40.661 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:00:40.661 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.661 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.661 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.661 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113672+0000", 2024-09-13T15:00:40.661 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7110000000000007e-06 2024-09-13T15:00:40.661 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.662 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.662 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.662 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113688+0000", 2024-09-13T15:00:40.662 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6013000000000001e-05 2024-09-13T15:00:40.662 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.662 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.662 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.662 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113695+0000", 2024-09-13T15:00:40.662 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1069999999999999e-06 2024-09-13T15:00:40.662 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.662 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.662 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.662 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113750+0000", 2024-09-13T15:00:40.663 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5210999999999997e-05 2024-09-13T15:00:40.663 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.663 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.663 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.663 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.113758+0000", 2024-09-13T15:00:40.663 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7670000000000003e-06 2024-09-13T15:00:40.663 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.663 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.663 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.663 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050094+0000", 2024-09-13T15:00:40.663 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93633562699999995 2024-09-13T15:00:40.663 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.664 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.664 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.664 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050100+0000", 2024-09-13T15:00:40.664 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8639999999999998e-06 2024-09-13T15:00:40.664 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.664 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.664 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.664 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327406+0000", 2024-09-13T15:00:40.664 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.277305248 2024-09-13T15:00:40.664 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.664 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.664 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.664 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327544+0000", 2024-09-13T15:00:40.665 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013814900000000001 2024-09-13T15:00:40.665 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.665 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.665 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3,6", 2024-09-13T15:00:40.665 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.327685+0000", 2024-09-13T15:00:40.665 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000140913 2024-09-13T15:00:40.665 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.665 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.665 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.665 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.334001+0000", 2024-09-13T15:00:40.665 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0063161620000000002 2024-09-13T15:00:40.665 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.666 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.666 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.666 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.334030+0000", 2024-09-13T15:00:40.666 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8631000000000001e-05 2024-09-13T15:00:40.666 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.666 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.666 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.666 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.354706+0000", 2024-09-13T15:00:40.666 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.020676019 2024-09-13T15:00:40.667 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.667 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.667 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.667 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.355880+0000", 2024-09-13T15:00:40.667 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001173978 2024-09-13T15:00:40.667 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.667 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.668 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.668 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.355908+0000", 2024-09-13T15:00:40.668 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8975e-05 2024-09-13T15:00:40.668 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.668 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.668 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.668 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.355990+0000", 2024-09-13T15:00:40.668 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1589999999999996e-05 2024-09-13T15:00:40.668 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.668 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.668 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.668 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.669 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.669 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:169295 7.1 7:8b7eab30:::benchmark_data_smithi096_196313_object49164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2024-09-13T15:00:40.669 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:10.141322+0000", 2024-09-13T15:00:40.669 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 570.45763725300003, 2024-09-13T15:00:40.669 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2148848929999998, 2024-09-13T15:00:40.669 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.669 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.669 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.669 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:00:40.669 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:00:40.669 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 169295 2024-09-13T15:00:40.669 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.669 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.670 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.670 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.670 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141322+0000", 2024-09-13T15:00:40.670 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.670 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.670 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.670 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.670 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141322+0000", 2024-09-13T15:00:40.670 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.670 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.670 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.670 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.671 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141320+0000", 2024-09-13T15:00:40.671 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:40.671 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.671 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.671 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.671 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141328+0000", 2024-09-13T15:00:40.671 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6229999999999997e-06 2024-09-13T15:00:40.671 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.671 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.671 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.672 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141341+0000", 2024-09-13T15:00:40.672 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.256e-05 2024-09-13T15:00:40.672 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.672 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.672 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.672 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141346+0000", 2024-09-13T15:00:40.672 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9960000000000001e-06 2024-09-13T15:00:40.672 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.672 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.672 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.672 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141387+0000", 2024-09-13T15:00:40.672 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0958999999999999e-05 2024-09-13T15:00:40.672 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.673 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.673 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.673 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:10.141394+0000", 2024-09-13T15:00:40.673 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1709999999999996e-06 2024-09-13T15:00:40.673 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.673 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.673 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.673 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050123+0000", 2024-09-13T15:00:40.673 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.90872976599999999 2024-09-13T15:00:40.673 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.673 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.673 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.673 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:11.050129+0000", 2024-09-13T15:00:40.674 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3839999999999996e-06 2024-09-13T15:00:40.674 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.674 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.674 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.674 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.331078+0000", 2024-09-13T15:00:40.674 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2809490889999999 2024-09-13T15:00:40.674 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.674 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.674 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.674 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.331165+0000", 2024-09-13T15:00:40.674 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7390000000000002e-05 2024-09-13T15:00:40.675 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.675 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.675 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3,6", 2024-09-13T15:00:40.675 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.331245+0000", 2024-09-13T15:00:40.675 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9727000000000004e-05 2024-09-13T15:00:40.675 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.675 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.675 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.675 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.334050+0000", 2024-09-13T15:00:40.675 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002804529 2024-09-13T15:00:40.676 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.676 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.676 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.676 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.334053+0000", 2024-09-13T15:00:40.676 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6100000000000002e-06 2024-09-13T15:00:40.676 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.676 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.676 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.677 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:40.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:40.678 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.354731+0000", 2024-09-13T15:00:40.678 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.020678215999999999 2024-09-13T15:00:40.678 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.678 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.678 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.678 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.356026+0000", 2024-09-13T15:00:40.678 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001294466 2024-09-13T15:00:40.679 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.679 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.679 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.679 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.356070+0000", 2024-09-13T15:00:40.679 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4295999999999997e-05 2024-09-13T15:00:40.679 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.679 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.679 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.679 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:12.356207+0000", 2024-09-13T15:00:40.679 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013657 2024-09-13T15:00:40.679 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.679 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.679 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.679 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.679 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.680 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45749.0:214039 8.2 8:461d6823:::benchmark_data_smithi096_227542_object88644:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T15:00:40.680 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:56:30.943139+0000", 2024-09-13T15:00:40.680 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 249.65581976600001, 2024-09-13T15:00:40.680 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.138819001, 2024-09-13T15:00:40.680 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.680 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.680 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.680 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45749", 2024-09-13T15:00:40.680 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T15:00:40.680 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 214039 2024-09-13T15:00:40.680 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.680 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.680 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.681 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.681 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.943139+0000", 2024-09-13T15:00:40.681 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.681 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.681 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.681 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.681 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.943139+0000", 2024-09-13T15:00:40.681 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.681 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.681 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.681 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.681 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.943138+0000", 2024-09-13T15:00:40.681 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:40.682 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.682 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.682 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.682 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.943144+0000", 2024-09-13T15:00:40.682 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1500000000000004e-06 2024-09-13T15:00:40.682 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.682 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.682 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.682 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.943155+0000", 2024-09-13T15:00:40.682 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1337e-05 2024-09-13T15:00:40.682 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.682 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.683 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.683 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.943160+0000", 2024-09-13T15:00:40.683 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5469999999999998e-06 2024-09-13T15:00:40.683 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.683 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.683 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.683 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.943196+0000", 2024-09-13T15:00:40.683 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6479000000000001e-05 2024-09-13T15:00:40.683 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.683 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.683 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.684 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.943284+0000", 2024-09-13T15:00:40.684 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7795999999999998e-05 2024-09-13T15:00:40.684 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.684 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.684 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:00:40.684 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.943357+0000", 2024-09-13T15:00:40.684 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2958000000000002e-05 2024-09-13T15:00:40.684 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.684 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.684 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.684 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.948041+0000", 2024-09-13T15:00:40.684 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0046832879999999999 2024-09-13T15:00:40.684 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.685 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.685 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.685 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.950501+0000", 2024-09-13T15:00:40.685 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0024604409999999998 2024-09-13T15:00:40.685 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.685 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.685 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.685 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:31.081837+0000", 2024-09-13T15:00:40.685 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.131336174 2024-09-13T15:00:40.685 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.685 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.685 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.685 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:31.081889+0000", 2024-09-13T15:00:40.686 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1737999999999998e-05 2024-09-13T15:00:40.686 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.686 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.686 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.686 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:31.081958+0000", 2024-09-13T15:00:40.686 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9467000000000004e-05 2024-09-13T15:00:40.686 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.686 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.686 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.687 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.687 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.687 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45749.0:214052 8.2 8:4759b8cd:::benchmark_data_smithi096_227542_object88657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T15:00:40.687 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:56:30.955593+0000", 2024-09-13T15:00:40.687 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 249.643365848, 2024-09-13T15:00:40.687 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.142896519, 2024-09-13T15:00:40.687 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.687 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.687 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.687 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45749", 2024-09-13T15:00:40.687 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T15:00:40.688 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 214052 2024-09-13T15:00:40.688 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.688 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.688 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.688 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.688 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955593+0000", 2024-09-13T15:00:40.688 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.688 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.688 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.688 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.688 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955593+0000", 2024-09-13T15:00:40.688 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.689 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.689 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.689 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.689 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955592+0000", 2024-09-13T15:00:40.689 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:40.689 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.689 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.689 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.689 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955599+0000", 2024-09-13T15:00:40.689 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5920000000000003e-06 2024-09-13T15:00:40.689 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.689 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.689 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.690 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955614+0000", 2024-09-13T15:00:40.690 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4346e-05 2024-09-13T15:00:40.690 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.690 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.690 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.690 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955620+0000", 2024-09-13T15:00:40.690 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2339999999999999e-06 2024-09-13T15:00:40.690 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.690 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.690 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.690 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955663+0000", 2024-09-13T15:00:40.690 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3448999999999997e-05 2024-09-13T15:00:40.690 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.691 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.691 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.691 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955791+0000", 2024-09-13T15:00:40.691 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012775500000000001 2024-09-13T15:00:40.691 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.691 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.691 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:00:40.691 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955936+0000", 2024-09-13T15:00:40.691 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014486100000000001 2024-09-13T15:00:40.691 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.691 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.691 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.692 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.960011+0000", 2024-09-13T15:00:40.692 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.004075431 2024-09-13T15:00:40.692 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.692 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.692 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.692 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.964835+0000", 2024-09-13T15:00:40.692 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0048238700000000001 2024-09-13T15:00:40.692 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.692 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.692 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.692 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:31.098384+0000", 2024-09-13T15:00:40.692 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.13354844499999999 2024-09-13T15:00:40.693 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.693 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.693 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.693 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:31.098445+0000", 2024-09-13T15:00:40.693 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1470000000000006e-05 2024-09-13T15:00:40.693 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.693 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.693 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.693 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:31.098490+0000", 2024-09-13T15:00:40.693 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4700000000000002e-05 2024-09-13T15:00:40.693 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.693 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.694 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.694 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.694 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.694 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45749.0:214061 8.2 8:46719c76:::benchmark_data_smithi096_227542_object88666:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T15:00:40.694 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:56:30.960765+0000", 2024-09-13T15:00:40.695 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 249.638194567, 2024-09-13T15:00:40.695 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.13784170500000001, 2024-09-13T15:00:40.695 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.695 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.695 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.695 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45749", 2024-09-13T15:00:40.695 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T15:00:40.696 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 214061 2024-09-13T15:00:40.696 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.696 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.696 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.696 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.696 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.960765+0000", 2024-09-13T15:00:40.696 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.696 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.696 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.696 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.697 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.960765+0000", 2024-09-13T15:00:40.697 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.697 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.697 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.697 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.697 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.960763+0000", 2024-09-13T15:00:40.698 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:00:40.698 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.698 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.698 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.698 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.960770+0000", 2024-09-13T15:00:40.698 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0160000000000003e-06 2024-09-13T15:00:40.698 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.698 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.698 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.698 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.960785+0000", 2024-09-13T15:00:40.699 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5532999999999998e-05 2024-09-13T15:00:40.699 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.699 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.699 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.699 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.960792+0000", 2024-09-13T15:00:40.699 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1840000000000004e-06 2024-09-13T15:00:40.699 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.699 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.699 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.699 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.960869+0000", 2024-09-13T15:00:40.699 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7594000000000003e-05 2024-09-13T15:00:40.700 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.700 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.700 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.700 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.960990+0000", 2024-09-13T15:00:40.700 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012116999999999999 2024-09-13T15:00:40.700 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.700 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.700 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:00:40.701 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.961109+0000", 2024-09-13T15:00:40.701 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000118263 2024-09-13T15:00:40.701 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.701 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.701 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.701 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.964851+0000", 2024-09-13T15:00:40.701 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0037426759999999999 2024-09-13T15:00:40.701 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.701 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.701 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.701 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.968670+0000", 2024-09-13T15:00:40.701 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0038183549999999998 2024-09-13T15:00:40.701 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.702 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.702 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.702 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:31.098530+0000", 2024-09-13T15:00:40.702 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.12985989000000001 2024-09-13T15:00:40.702 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.702 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.702 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.702 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:31.098572+0000", 2024-09-13T15:00:40.702 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2784000000000002e-05 2024-09-13T15:00:40.702 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.702 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.702 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.702 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:31.098606+0000", 2024-09-13T15:00:40.702 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3923e-05 2024-09-13T15:00:40.703 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.703 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.703 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.703 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.703 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.703 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45749.0:238705 8.2 8:41667315:::benchmark_data_smithi096_227542_object113310:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T15:00:40.703 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:56:56.922681+0000", 2024-09-13T15:00:40.703 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 223.67627806600001, 2024-09-13T15:00:40.703 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.23348930900000001, 2024-09-13T15:00:40.703 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.703 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.703 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.703 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45749", 2024-09-13T15:00:40.703 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T15:00:40.703 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 238705 2024-09-13T15:00:40.704 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.704 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.704 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.704 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.704 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922681+0000", 2024-09-13T15:00:40.704 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.704 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.704 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.704 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.705 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922681+0000", 2024-09-13T15:00:40.705 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.705 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.705 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.705 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.705 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922680+0000", 2024-09-13T15:00:40.705 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:00:40.705 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.705 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.706 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.706 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922686+0000", 2024-09-13T15:00:40.706 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6210000000000001e-06 2024-09-13T15:00:40.706 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.706 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.706 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.706 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922696+0000", 2024-09-13T15:00:40.706 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0036e-05 2024-09-13T15:00:40.706 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.706 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.706 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.706 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922701+0000", 2024-09-13T15:00:40.706 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7219999999999996e-06 2024-09-13T15:00:40.706 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.707 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.707 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.707 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922792+0000", 2024-09-13T15:00:40.707 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1013000000000004e-05 2024-09-13T15:00:40.707 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.707 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.707 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.707 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922916+0000", 2024-09-13T15:00:40.707 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012372800000000001 2024-09-13T15:00:40.707 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.707 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.707 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:00:40.707 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.923042+0000", 2024-09-13T15:00:40.707 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000125974 2024-09-13T15:00:40.708 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.708 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.708 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.708 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.926845+0000", 2024-09-13T15:00:40.708 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0038031290000000001 2024-09-13T15:00:40.708 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.708 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.708 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.708 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.927035+0000", 2024-09-13T15:00:40.708 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00019018600000000001 2024-09-13T15:00:40.708 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.709 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.709 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.709 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.156091+0000", 2024-09-13T15:00:40.709 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.22905610600000001 2024-09-13T15:00:40.709 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.709 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.710 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.710 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.156129+0000", 2024-09-13T15:00:40.710 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7125000000000001e-05 2024-09-13T15:00:40.710 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.710 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.710 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.711 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.156170+0000", 2024-09-13T15:00:40.711 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1739999999999997e-05 2024-09-13T15:00:40.711 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.711 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.711 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.711 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.711 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.711 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45749.0:238719 8.2 8:45fb8e4f:::benchmark_data_smithi096_227542_object113324:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T15:00:40.711 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:56:56.937206+0000", 2024-09-13T15:00:40.711 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 223.66175339399999, 2024-09-13T15:00:40.712 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.23103955300000001, 2024-09-13T15:00:40.712 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.712 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.712 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.712 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45749", 2024-09-13T15:00:40.712 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T15:00:40.712 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 238719 2024-09-13T15:00:40.712 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.712 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.712 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.712 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.712 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937206+0000", 2024-09-13T15:00:40.713 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.713 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.713 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.713 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.713 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937206+0000", 2024-09-13T15:00:40.713 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.713 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.713 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.713 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.714 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937204+0000", 2024-09-13T15:00:40.714 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:40.714 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.714 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.714 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.714 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937212+0000", 2024-09-13T15:00:40.714 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7729999999999999e-06 2024-09-13T15:00:40.714 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.714 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.715 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.715 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937226+0000", 2024-09-13T15:00:40.715 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.434e-05 2024-09-13T15:00:40.715 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.715 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.715 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.715 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937231+0000", 2024-09-13T15:00:40.715 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7489999999999998e-06 2024-09-13T15:00:40.715 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.715 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.715 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.715 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937287+0000", 2024-09-13T15:00:40.716 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5995999999999997e-05 2024-09-13T15:00:40.716 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.716 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.716 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.716 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937384+0000", 2024-09-13T15:00:40.716 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6912999999999999e-05 2024-09-13T15:00:40.716 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.716 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.716 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:00:40.716 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937508+0000", 2024-09-13T15:00:40.716 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000124578 2024-09-13T15:00:40.716 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.717 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.717 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.717 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.940699+0000", 2024-09-13T15:00:40.717 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00319024 2024-09-13T15:00:40.717 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.717 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.717 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.717 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.941399+0000", 2024-09-13T15:00:40.717 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00070025800000000004 2024-09-13T15:00:40.718 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.718 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.718 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.718 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.168173+0000", 2024-09-13T15:00:40.718 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.22677360699999999 2024-09-13T15:00:40.718 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.718 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.719 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.719 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.168206+0000", 2024-09-13T15:00:40.719 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3843000000000001e-05 2024-09-13T15:00:40.719 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.719 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.719 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.719 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.168245+0000", 2024-09-13T15:00:40.719 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8862999999999997e-05 2024-09-13T15:00:40.719 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.719 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.719 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.719 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.720 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.720 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45749.0:238724 8.2 8:4164a1e4:::benchmark_data_smithi096_227542_object113329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T15:00:40.720 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:56:56.942313+0000", 2024-09-13T15:00:40.721 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 223.65664598000001, 2024-09-13T15:00:40.721 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.226006811, 2024-09-13T15:00:40.721 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.721 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.721 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.721 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45749", 2024-09-13T15:00:40.721 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T15:00:40.721 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 238724 2024-09-13T15:00:40.721 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.721 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.721 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.721 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.721 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942313+0000", 2024-09-13T15:00:40.721 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.721 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.722 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.722 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.722 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942313+0000", 2024-09-13T15:00:40.722 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.722 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.722 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.722 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.722 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942312+0000", 2024-09-13T15:00:40.722 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:40.722 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.722 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.722 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.722 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942318+0000", 2024-09-13T15:00:40.722 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4649999999999999e-06 2024-09-13T15:00:40.722 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.723 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.723 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.723 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942332+0000", 2024-09-13T15:00:40.723 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3596e-05 2024-09-13T15:00:40.723 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.723 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.723 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.723 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942337+0000", 2024-09-13T15:00:40.723 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4530000000000001e-06 2024-09-13T15:00:40.723 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.723 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.723 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.723 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942374+0000", 2024-09-13T15:00:40.723 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6749999999999999e-05 2024-09-13T15:00:40.723 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.724 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.724 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.724 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942461+0000", 2024-09-13T15:00:40.724 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.6698000000000002e-05 2024-09-13T15:00:40.724 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.724 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.724 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:00:40.724 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942534+0000", 2024-09-13T15:00:40.724 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.3245999999999996e-05 2024-09-13T15:00:40.724 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.724 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.725 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.725 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.945803+0000", 2024-09-13T15:00:40.725 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003268703 2024-09-13T15:00:40.725 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.725 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.725 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.725 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.946325+0000", 2024-09-13T15:00:40.725 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00052236999999999997 2024-09-13T15:00:40.725 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.725 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.725 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.725 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.168274+0000", 2024-09-13T15:00:40.726 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.221949167 2024-09-13T15:00:40.726 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.726 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.726 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.726 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.168301+0000", 2024-09-13T15:00:40.726 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.6923999999999998e-05 2024-09-13T15:00:40.726 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.726 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.726 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.727 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.168320+0000", 2024-09-13T15:00:40.727 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8893e-05 2024-09-13T15:00:40.727 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.727 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.727 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.727 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.727 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.727 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:113285 9.c 9:3711587a:::benchmark_data_smithi096_256909_object113284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-09-13T15:00:40.727 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:11.892740+0000", 2024-09-13T15:00:40.728 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 88.706219188999995, 2024-09-13T15:00:40.728 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.025987193, 2024-09-13T15:00:40.728 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.728 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.728 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.728 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:40.729 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:40.729 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 113285 2024-09-13T15:00:40.729 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.729 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.729 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.729 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.729 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.892740+0000", 2024-09-13T15:00:40.729 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.729 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.729 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.729 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.729 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.892740+0000", 2024-09-13T15:00:40.729 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.729 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.730 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.730 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.730 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.892738+0000", 2024-09-13T15:00:40.730 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:00:40.730 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.730 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.730 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.730 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.892807+0000", 2024-09-13T15:00:40.730 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9183000000000005e-05 2024-09-13T15:00:40.730 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.730 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.730 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.730 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.892821+0000", 2024-09-13T15:00:40.730 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3861e-05 2024-09-13T15:00:40.731 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.731 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.731 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.731 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.892827+0000", 2024-09-13T15:00:40.731 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2439999999999998e-06 2024-09-13T15:00:40.731 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.731 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.731 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.731 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.894455+0000", 2024-09-13T15:00:40.731 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001627257 2024-09-13T15:00:40.731 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.731 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.731 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.731 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.894464+0000", 2024-09-13T15:00:40.732 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.4809999999999995e-06 2024-09-13T15:00:40.732 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.732 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.732 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.732 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.890832+0000", 2024-09-13T15:00:40.732 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99636818599999999 2024-09-13T15:00:40.732 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.732 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.732 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.732 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.890979+0000", 2024-09-13T15:00:40.732 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014698099999999999 2024-09-13T15:00:40.732 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.732 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.732 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:00:40.733 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.891141+0000", 2024-09-13T15:00:40.733 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016161800000000001 2024-09-13T15:00:40.733 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.733 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.733 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.733 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.908915+0000", 2024-09-13T15:00:40.733 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.017774366999999999 2024-09-13T15:00:40.733 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.733 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.733 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.733 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.908932+0000", 2024-09-13T15:00:40.733 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6820000000000002e-05 2024-09-13T15:00:40.733 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.733 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.734 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.734 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918564+0000", 2024-09-13T15:00:40.734 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0096315329999999994 2024-09-13T15:00:40.734 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.734 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.734 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.734 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918624+0000", 2024-09-13T15:00:40.734 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0449999999999999e-05 2024-09-13T15:00:40.734 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.734 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.734 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.734 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918727+0000", 2024-09-13T15:00:40.734 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000103019 2024-09-13T15:00:40.734 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.735 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.735 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.735 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.735 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.735 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:113315 9.c 9:33905679:::benchmark_data_smithi096_256909_object113314:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-09-13T15:00:40.735 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:11.954719+0000", 2024-09-13T15:00:40.735 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 88.644239948999996, 2024-09-13T15:00:40.735 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.96417554699999997, 2024-09-13T15:00:40.735 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.735 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.735 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.735 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:40.735 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:40.735 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 113315 2024-09-13T15:00:40.736 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.736 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.736 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.736 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.736 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954719+0000", 2024-09-13T15:00:40.736 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.736 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.736 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.736 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.736 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954719+0000", 2024-09-13T15:00:40.736 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.736 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.736 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.736 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.737 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954717+0000", 2024-09-13T15:00:40.737 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:00:40.737 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.737 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.737 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.737 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954773+0000", 2024-09-13T15:00:40.737 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6189e-05 2024-09-13T15:00:40.737 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.737 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.737 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.737 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954792+0000", 2024-09-13T15:00:40.738 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8705000000000001e-05 2024-09-13T15:00:40.738 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.738 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.738 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.738 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954798+0000", 2024-09-13T15:00:40.738 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0990000000000004e-06 2024-09-13T15:00:40.738 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.738 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.738 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.739 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954849+0000", 2024-09-13T15:00:40.739 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0444e-05 2024-09-13T15:00:40.739 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.739 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.739 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.739 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954858+0000", 2024-09-13T15:00:40.740 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8410000000000006e-06 2024-09-13T15:00:40.740 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.740 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.740 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.740 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.895625+0000", 2024-09-13T15:00:40.740 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.94076707699999995 2024-09-13T15:00:40.740 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.740 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.740 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.740 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.895768+0000", 2024-09-13T15:00:40.740 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014290900000000001 2024-09-13T15:00:40.740 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.741 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.741 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:00:40.741 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.895943+0000", 2024-09-13T15:00:40.741 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00017526999999999999 2024-09-13T15:00:40.741 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.741 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.741 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.741 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.908972+0000", 2024-09-13T15:00:40.741 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013028814999999999 2024-09-13T15:00:40.742 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.742 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.742 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.742 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.908979+0000", 2024-09-13T15:00:40.742 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9140000000000002e-06 2024-09-13T15:00:40.742 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.742 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.742 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.742 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918764+0000", 2024-09-13T15:00:40.742 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0097855540000000001 2024-09-13T15:00:40.742 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.742 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.743 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.743 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918808+0000", 2024-09-13T15:00:40.743 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3510999999999997e-05 2024-09-13T15:00:40.743 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.743 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.743 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.743 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918895+0000", 2024-09-13T15:00:40.743 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7065999999999996e-05 2024-09-13T15:00:40.743 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.743 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.743 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.743 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.743 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.744 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:113320 9.c 9:3f46f27a:::benchmark_data_smithi096_256909_object113319:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-09-13T15:00:40.744 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:11.961842+0000", 2024-09-13T15:00:40.744 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 88.637117457000002, 2024-09-13T15:00:40.744 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95725164600000001, 2024-09-13T15:00:40.744 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.744 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.745 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.745 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:40.745 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:40.745 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 113320 2024-09-13T15:00:40.745 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.745 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.745 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.745 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.745 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961842+0000", 2024-09-13T15:00:40.745 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.745 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.745 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.746 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.746 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961842+0000", 2024-09-13T15:00:40.746 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.746 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.746 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.746 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.746 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961840+0000", 2024-09-13T15:00:40.746 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:00:40.746 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.746 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.747 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.747 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961882+0000", 2024-09-13T15:00:40.747 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2212999999999997e-05 2024-09-13T15:00:40.747 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.747 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.747 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.747 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961900+0000", 2024-09-13T15:00:40.747 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8196999999999999e-05 2024-09-13T15:00:40.747 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.747 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.747 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.747 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961908+0000", 2024-09-13T15:00:40.748 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8949999999999998e-06 2024-09-13T15:00:40.748 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.748 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.748 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.748 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961977+0000", 2024-09-13T15:00:40.748 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8628999999999996e-05 2024-09-13T15:00:40.748 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.748 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.748 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.748 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961987+0000", 2024-09-13T15:00:40.748 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.8079999999999999e-06 2024-09-13T15:00:40.749 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.749 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.749 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.749 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.899547+0000", 2024-09-13T15:00:40.749 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93756020399999995 2024-09-13T15:00:40.749 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.749 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.750 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.750 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.899680+0000", 2024-09-13T15:00:40.750 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013346699999999999 2024-09-13T15:00:40.750 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.750 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.750 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:00:40.750 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.899841+0000", 2024-09-13T15:00:40.750 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016104300000000001 2024-09-13T15:00:40.750 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.750 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.750 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.750 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.908988+0000", 2024-09-13T15:00:40.750 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.009146955 2024-09-13T15:00:40.750 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.751 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.751 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.751 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.909031+0000", 2024-09-13T15:00:40.751 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2908000000000003e-05 2024-09-13T15:00:40.751 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.751 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.751 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.751 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918935+0000", 2024-09-13T15:00:40.751 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0099035110000000003 2024-09-13T15:00:40.751 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.751 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.751 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.751 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918993+0000", 2024-09-13T15:00:40.751 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8156999999999999e-05 2024-09-13T15:00:40.751 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.752 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.752 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.752 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.919093+0000", 2024-09-13T15:00:40.752 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000100342 2024-09-13T15:00:40.752 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.752 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.752 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.752 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.752 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.752 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:113329 9.c 9:339064f9:::benchmark_data_smithi096_256909_object113328:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-09-13T15:00:40.752 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:11.989105+0000", 2024-09-13T15:00:40.752 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 88.609854569999996, 2024-09-13T15:00:40.752 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.94653837500000004, 2024-09-13T15:00:40.752 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.752 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.753 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.753 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:40.753 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:40.753 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 113329 2024-09-13T15:00:40.753 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.753 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.753 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.753 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.753 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989105+0000", 2024-09-13T15:00:40.753 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.753 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.753 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.753 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.753 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989105+0000", 2024-09-13T15:00:40.754 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.754 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.754 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.754 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.754 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989103+0000", 2024-09-13T15:00:40.754 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:40.754 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.754 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.754 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.754 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989158+0000", 2024-09-13T15:00:40.754 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5356999999999999e-05 2024-09-13T15:00:40.754 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.755 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.755 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.755 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989170+0000", 2024-09-13T15:00:40.755 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1674999999999999e-05 2024-09-13T15:00:40.755 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.755 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.755 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.755 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989175+0000", 2024-09-13T15:00:40.755 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1270000000000002e-06 2024-09-13T15:00:40.755 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.755 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.755 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.756 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989225+0000", 2024-09-13T15:00:40.756 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9323000000000002e-05 2024-09-13T15:00:40.756 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.756 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.756 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:00:40.756 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989232+0000", 2024-09-13T15:00:40.756 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7409999999999992e-06 2024-09-13T15:00:40.756 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.756 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.756 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.756 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.904069+0000", 2024-09-13T15:00:40.756 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.914836172 2024-09-13T15:00:40.756 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.757 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.757 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.757 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.904202+0000", 2024-09-13T15:00:40.757 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013350999999999999 2024-09-13T15:00:40.757 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.757 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.757 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:00:40.757 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.904370+0000", 2024-09-13T15:00:40.757 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016814400000000001 2024-09-13T15:00:40.757 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.757 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.757 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.757 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.908994+0000", 2024-09-13T15:00:40.758 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0046242970000000003 2024-09-13T15:00:40.758 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.758 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.758 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.758 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.909068+0000", 2024-09-13T15:00:40.758 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.3784999999999999e-05 2024-09-13T15:00:40.758 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.758 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.758 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.758 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.935324+0000", 2024-09-13T15:00:40.758 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.026256152000000001 2024-09-13T15:00:40.758 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.759 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.759 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.759 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.935381+0000", 2024-09-13T15:00:40.759 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6595000000000003e-05 2024-09-13T15:00:40.759 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.759 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.759 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.759 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.935643+0000", 2024-09-13T15:00:40.759 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00026189200000000001 2024-09-13T15:00:40.759 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.760 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.760 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.760 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.760 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.760 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:140559 9.c 9:3844b952:::benchmark_data_smithi096_256909_object21151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:00:40.760 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:47.271238+0000", 2024-09-13T15:00:40.760 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 53.327720931000002, 2024-09-13T15:00:40.760 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.11733141699999999, 2024-09-13T15:00:40.761 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.761 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.761 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.761 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:40.761 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:40.761 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 140559 2024-09-13T15:00:40.761 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.761 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.761 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.761 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.761 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.271238+0000", 2024-09-13T15:00:40.761 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.761 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.761 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.762 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.762 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.271238+0000", 2024-09-13T15:00:40.762 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.762 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.762 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.763 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.763 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.271237+0000", 2024-09-13T15:00:40.763 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:40.763 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.763 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.763 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.763 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.271243+0000", 2024-09-13T15:00:40.763 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0009999999999998e-06 2024-09-13T15:00:40.763 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.763 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.763 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.764 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.271254+0000", 2024-09-13T15:00:40.764 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1139e-05 2024-09-13T15:00:40.764 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.764 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.764 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.764 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.271259+0000", 2024-09-13T15:00:40.764 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5979999999999996e-06 2024-09-13T15:00:40.764 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.764 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.765 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.765 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.271401+0000", 2024-09-13T15:00:40.765 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014212200000000001 2024-09-13T15:00:40.765 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.765 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.765 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.766 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.271513+0000", 2024-09-13T15:00:40.766 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000111736 2024-09-13T15:00:40.766 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.766 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.766 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:00:40.766 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.271627+0000", 2024-09-13T15:00:40.767 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011390199999999999 2024-09-13T15:00:40.767 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.767 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.767 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.767 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.274684+0000", 2024-09-13T15:00:40.767 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0030570279999999998 2024-09-13T15:00:40.768 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.768 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.768 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.768 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.279944+0000", 2024-09-13T15:00:40.768 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0052603499999999996 2024-09-13T15:00:40.768 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.768 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.768 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.769 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.388432+0000", 2024-09-13T15:00:40.769 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.108487109 2024-09-13T15:00:40.769 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.769 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.769 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.769 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.388479+0000", 2024-09-13T15:00:40.769 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7482999999999999e-05 2024-09-13T15:00:40.769 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.769 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.769 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.769 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.388570+0000", 2024-09-13T15:00:40.769 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.0569000000000006e-05 2024-09-13T15:00:40.770 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.770 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.770 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.770 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.770 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.770 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:146251 9.c 9:366e771e:::benchmark_data_smithi096_256909_object26843:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2024-09-13T15:00:40.770 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:55.245493+0000", 2024-09-13T15:00:40.770 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 45.353466535000003, 2024-09-13T15:00:40.771 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.13807438499999999, 2024-09-13T15:00:40.771 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.771 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.771 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.771 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:40.771 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:40.771 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 146251 2024-09-13T15:00:40.771 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.771 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.771 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.771 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.772 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:55.245493+0000", 2024-09-13T15:00:40.772 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.772 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.772 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.772 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.772 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:55.245493+0000", 2024-09-13T15:00:40.772 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.772 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.772 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.773 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.773 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:55.245491+0000", 2024-09-13T15:00:40.773 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:00:40.773 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.773 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.773 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.773 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:55.245497+0000", 2024-09-13T15:00:40.773 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5409999999999997e-06 2024-09-13T15:00:40.773 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.773 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.773 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.774 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:55.245508+0000", 2024-09-13T15:00:40.774 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0638999999999999e-05 2024-09-13T15:00:40.774 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.774 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.774 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.774 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:55.245515+0000", 2024-09-13T15:00:40.774 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5400000000000001e-06 2024-09-13T15:00:40.775 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.775 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.775 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.775 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:55.245570+0000", 2024-09-13T15:00:40.775 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5819000000000002e-05 2024-09-13T15:00:40.775 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.776 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.776 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.776 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:55.245689+0000", 2024-09-13T15:00:40.776 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000118952 2024-09-13T15:00:40.776 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.777 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.777 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:00:40.777 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:55.245808+0000", 2024-09-13T15:00:40.777 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119055 2024-09-13T15:00:40.777 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.777 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.777 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.777 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:55.249228+0000", 2024-09-13T15:00:40.777 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0034192089999999999 2024-09-13T15:00:40.778 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.778 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.778 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.778 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:55.249569+0000", 2024-09-13T15:00:40.778 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00034166699999999998 2024-09-13T15:00:40.778 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.778 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.778 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.778 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:55.383406+0000", 2024-09-13T15:00:40.778 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.13383672799999999 2024-09-13T15:00:40.778 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.778 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.778 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.778 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:55.383462+0000", 2024-09-13T15:00:40.779 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5825999999999997e-05 2024-09-13T15:00:40.779 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.779 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.779 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.779 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:55.383567+0000", 2024-09-13T15:00:40.779 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010513099999999999 2024-09-13T15:00:40.779 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.779 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.779 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.779 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.779 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.779 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:160501 9.c 9:36ca4d86:::benchmark_data_smithi096_256909_object41093:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-09-13T15:00:40.779 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:11.545011+0000", 2024-09-13T15:00:40.779 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 29.053947787999999, 2024-09-13T15:00:40.780 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.11718073499999999, 2024-09-13T15:00:40.780 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:40.780 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:40.780 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:40.780 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:40.780 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:40.780 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 160501 2024-09-13T15:00:40.780 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.780 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:40.780 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.781 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:40.781 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:11.545011+0000", 2024-09-13T15:00:40.781 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.781 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.781 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.781 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:40.781 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:11.545011+0000", 2024-09-13T15:00:40.781 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:40.781 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.781 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.781 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:40.781 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:11.545010+0000", 2024-09-13T15:00:40.781 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:00:40.781 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.782 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.782 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:40.782 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:11.545016+0000", 2024-09-13T15:00:40.782 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.981e-06 2024-09-13T15:00:40.782 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.782 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.782 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:40.782 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:11.545033+0000", 2024-09-13T15:00:40.782 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6918999999999998e-05 2024-09-13T15:00:40.782 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.783 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.783 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:40.783 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:11.545039+0000", 2024-09-13T15:00:40.783 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7490000000000001e-06 2024-09-13T15:00:40.783 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.783 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.783 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:40.783 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:11.545172+0000", 2024-09-13T15:00:40.783 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000132452 2024-09-13T15:00:40.783 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.783 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.783 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:40.784 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:11.545454+0000", 2024-09-13T15:00:40.784 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00028261699999999998 2024-09-13T15:00:40.784 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.784 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.784 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:00:40.784 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:11.545808+0000", 2024-09-13T15:00:40.784 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00035348900000000002 2024-09-13T15:00:40.784 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.784 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.784 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:40.784 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:11.549184+0000", 2024-09-13T15:00:40.785 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0033759459999999999 2024-09-13T15:00:40.785 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.785 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.785 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.785 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:11.549884+0000", 2024-09-13T15:00:40.785 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00070038800000000003 2024-09-13T15:00:40.785 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.785 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.785 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:40.785 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:11.662021+0000", 2024-09-13T15:00:40.785 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.112136603 2024-09-13T15:00:40.785 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.788 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.788 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:40.788 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:11.662072+0000", 2024-09-13T15:00:40.788 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1544e-05 2024-09-13T15:00:40.789 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:40.789 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:40.789 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:40.789 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:11.662192+0000", 2024-09-13T15:00:40.789 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119716 2024-09-13T15:00:40.789 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.789 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.789 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.789 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:40.789 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:40.789 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:00:40.790 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:40.819 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:40.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:40.922 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:40.958 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:40.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:41.060 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:41.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.097 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:41.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:41.199 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:41.239 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:41.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:41.342 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:41.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.377 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:41.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:41.478 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:41.514 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:41.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:41.569 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:00:41.569 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:00:41.600 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:00:41.600 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:00:41.616 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:41.637 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:00:41.638 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:00:41.638 INFO:teuthology.orchestra.run.smithi096.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:00:41.653 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:41.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:41.666 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:00:41.666 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:00:41.666 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:41 smithi096 ceph-mon[98901]: pgmap v1093: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 870 op/s; 34326/168999 objects degraded (20.311%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T15:00:41.666 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:41 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1121232456' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:41.667 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:41 smithi096 ceph-mon[102928]: pgmap v1093: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 870 op/s; 34326/168999 objects degraded (20.311%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T15:00:41.667 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:41 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1121232456' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:41.695 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:00:41.695 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:00:41.721 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:00:41.721 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:00:41.754 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:41.761 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:00:41.761 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:00:41.794 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:41.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:41.842 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 6] out_osds: [3] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2024-09-13T15:00:41.842 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:00:41.843 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-09-13T15:00:41.843 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T15:00:41.843 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T15:00:41.843 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-09-13T15:00:41.879 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:41 smithi204 ceph-mon[87071]: pgmap v1093: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 870 op/s; 34326/168999 objects degraded (20.311%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T15:00:41.879 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:41 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1121232456' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:41.896 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:41.931 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:41.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:42.033 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:42.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.057 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:42.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:42.158 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:42.193 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:42.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:42.295 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:42.317 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:42.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:42.419 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:42.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.453 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:42.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:42.554 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:42.582 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:42.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:42.683 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:42.703 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:00:42.704 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:00:42.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.722 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:42.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:42.823 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:42.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.869 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:42.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:42.970 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:42.999 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:43.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:43.101 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:43.131 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:43.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:43.133 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T15:00:43.232 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:43.249 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:00:43.304 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:43.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:43.406 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:43.432 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:43.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:43.534 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:43.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.573 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:43.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:43.622 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T15:00:43.674 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:43.701 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:43.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:43.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:43 smithi204 ceph-mon[87071]: pgmap v1094: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 775 op/s; 34326/168999 objects degraded (20.311%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:00:43.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:43 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1269267835' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:43.803 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:43.834 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:43.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:43.936 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:43.946 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:43 smithi096 ceph-mon[102928]: pgmap v1094: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 775 op/s; 34326/168999 objects degraded (20.311%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:00:43.946 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:43 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1269267835' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:43.947 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:43 smithi096 ceph-mon[98901]: pgmap v1094: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 775 op/s; 34326/168999 objects degraded (20.311%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:00:43.947 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:43 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1269267835' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:43.974 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:43.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:44.076 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:44.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.112 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:44.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:44.213 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:44.262 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:44.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:44.364 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:44.407 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:44.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:44.510 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:44.552 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:44.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:44.653 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:44.683 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:44.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:44.706 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:00:44.755 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:44 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:44.756 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:44 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1269267835' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:44.756 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:44 smithi096 ceph-mon[102928]: osdmap e331: 8 total, 7 up, 7 in 2024-09-13T15:00:44.756 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:44 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 34326/168999 objects degraded (20.311%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:44.756 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:44 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:44.756 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:44 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1269267835' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:44.756 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:44 smithi096 ceph-mon[98901]: osdmap e331: 8 total, 7 up, 7 in 2024-09-13T15:00:44.756 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:44 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 34326/168999 objects degraded (20.311%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:44.785 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:44.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.821 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:44.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:44.921 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:44 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:44.921 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:44 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1269267835' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:44.922 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:44 smithi204 ceph-mon[87071]: osdmap e331: 8 total, 7 up, 7 in 2024-09-13T15:00:44.922 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:44 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 34326/168999 objects degraded (20.311%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:44.922 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:44.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.962 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:44.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:45.063 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:45.072 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:00:45.081 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:45.091 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:45.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:45.193 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:45.230 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:45.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:45.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:45.359 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:45.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:45.461 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:45.500 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:45.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:45.604 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:45.632 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:45.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:45.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:45 smithi204 ceph-mon[87071]: pgmap v1096: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 33649/160113 objects degraded (21.016%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T15:00:45.733 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:45.763 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:45.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:45.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:45 smithi096 ceph-mon[102928]: pgmap v1096: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 33649/160113 objects degraded (21.016%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T15:00:45.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:45 smithi096 ceph-mon[98901]: pgmap v1096: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 33649/160113 objects degraded (21.016%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T15:00:45.865 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:45.889 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:00:45.889 INFO:teuthology.orchestra.run.smithi204.stdout: "success": "" 2024-09-13T15:00:45.889 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:00:45.909 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:45.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:46.011 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:46.037 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:46.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:46.139 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:46.182 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:46.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:46.284 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:46.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.321 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:46.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:46.410 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:00:46.422 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:46.485 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:46.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:46.587 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:46.621 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:46.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:46.723 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:46.759 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:46.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:46.861 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:46.896 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:46.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:46.998 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:47.021 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:47.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:47.123 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:47.144 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:47.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:47.246 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:47.285 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:47.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:47.386 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:47.414 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:47.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:47.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:47.557 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:47.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:47.659 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:47.690 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:47.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:47.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:47 smithi204 ceph-mon[87071]: pgmap v1097: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 33390/157197 objects degraded (21.241%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T15:00:47.767 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:47 smithi096 ceph-mon[98901]: pgmap v1097: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 33390/157197 objects degraded (21.241%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T15:00:47.767 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:47 smithi096 ceph-mon[102928]: pgmap v1097: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 33390/157197 objects degraded (21.241%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T15:00:47.792 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:47.826 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:47.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:47.927 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:47.957 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:47.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:48.030 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:00:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:00:47] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.51.0" 2024-09-13T15:00:48.059 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:48.089 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:48.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:48.191 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:48.223 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:48.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:48.325 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:48.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.364 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:48.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:48.465 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:48.488 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:48.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:48.590 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:48.611 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:48.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:48.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:48.741 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:48.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:48.843 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:48.865 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:48.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:48.967 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:48.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.992 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:48.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:49.093 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:49.114 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:49.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:49.216 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:49.247 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:49.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:49.349 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:49.379 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:49.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:49.481 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:49.509 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:49.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:49.611 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:49.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.641 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:49.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:49.742 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:49.782 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:49.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:49.884 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:49.916 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:49.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:49.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:49 smithi204 ceph-mon[87071]: pgmap v1098: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 785 op/s; 32940/150537 objects degraded (21.882%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:00:49.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:49 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3503469459' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:49.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:49 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 32940/150537 objects degraded (21.882%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:50.017 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:50.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.055 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:50.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:50.056 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:00:50.056 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:00:50.056 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:49 smithi096 ceph-mon[102928]: pgmap v1098: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 785 op/s; 32940/150537 objects degraded (21.882%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:00:50.056 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:49 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3503469459' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:50.056 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:49 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 32940/150537 objects degraded (21.882%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:50.057 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:00:50.057 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:00:50.057 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:00:50.057 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:00:50.057 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:00:50.057 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:00:50.057 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:00:50.058 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:00:50.058 INFO:teuthology.orchestra.run.smithi096.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:00:50.058 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:00:50.058 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:00:50.058 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:00:50.058 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:00:50.058 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:49 smithi096 ceph-mon[98901]: pgmap v1098: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 785 op/s; 32940/150537 objects degraded (21.882%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:00:50.058 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:49 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3503469459' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:50.058 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:49 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 32940/150537 objects degraded (21.882%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:50.156 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:50.192 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:50.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:50.294 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:50.330 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:50.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:50.432 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:00:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:00:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:50.473 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:50.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:50.510 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:00:50.510 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T15:00:50.510 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.510 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:192056 9.7 9:e4233625:::benchmark_data_smithi096_256909_object72648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e331)", 2024-09-13T15:00:50.510 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:50.477411+0000", 2024-09-13T15:00:50.510 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.029469473999999999, 2024-09-13T15:00:50.511 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.029500302999999999, 2024-09-13T15:00:50.511 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:50.511 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:00:50.511 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:50.511 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:50.511 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:50.511 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 192056 2024-09-13T15:00:50.511 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.511 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:50.511 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.511 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:50.511 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.477411+0000", 2024-09-13T15:00:50.511 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.511 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.512 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.512 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:50.512 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.477411+0000", 2024-09-13T15:00:50.512 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.512 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.512 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.512 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:50.512 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.477408+0000", 2024-09-13T15:00:50.512 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999976 2024-09-13T15:00:50.512 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.512 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.512 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:50.512 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.477418+0000", 2024-09-13T15:00:50.512 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.4709999999999996e-06 2024-09-13T15:00:50.513 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.513 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.513 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:50.513 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.477435+0000", 2024-09-13T15:00:50.513 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7088e-05 2024-09-13T15:00:50.513 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.513 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.513 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:50.513 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.477441+0000", 2024-09-13T15:00:50.513 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6649999999999997e-06 2024-09-13T15:00:50.513 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.513 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.513 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:50.513 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.486537+0000", 2024-09-13T15:00:50.514 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0090953330000000006 2024-09-13T15:00:50.514 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.514 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.514 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:50.514 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.486687+0000", 2024-09-13T15:00:50.514 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015066799999999999 2024-09-13T15:00:50.514 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.514 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.514 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:00:50.514 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.486841+0000", 2024-09-13T15:00:50.514 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000153375 2024-09-13T15:00:50.514 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.514 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.514 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:50.514 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.500754+0000", 2024-09-13T15:00:50.515 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013913282000000001 2024-09-13T15:00:50.515 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.515 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.515 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:50.515 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.506013+0000", 2024-09-13T15:00:50.515 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0052583819999999998 2024-09-13T15:00:50.515 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.515 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:50.515 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.515 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.515 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.515 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:192061 9.f 9:fad48061:::benchmark_data_smithi096_256909_object72653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e331)", 2024-09-13T15:00:50.515 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:50.480873+0000", 2024-09-13T15:00:50.516 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.026006741, 2024-09-13T15:00:50.516 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.026129276, 2024-09-13T15:00:50.516 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:50.516 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:50.516 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:50.516 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:50.516 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:50.516 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 192061 2024-09-13T15:00:50.516 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.516 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:50.516 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.516 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:50.516 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.480873+0000", 2024-09-13T15:00:50.516 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.517 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.517 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.517 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:50.517 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.480873+0000", 2024-09-13T15:00:50.517 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.517 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.517 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.517 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:50.517 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.480872+0000", 2024-09-13T15:00:50.517 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:00:50.517 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.517 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.518 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:50.518 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.480879+0000", 2024-09-13T15:00:50.518 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1049999999999997e-06 2024-09-13T15:00:50.518 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.518 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.518 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:50.518 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.480890+0000", 2024-09-13T15:00:50.518 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1395e-05 2024-09-13T15:00:50.518 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.518 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.518 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:50.518 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.480896+0000", 2024-09-13T15:00:50.518 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4870000000000002e-06 2024-09-13T15:00:50.518 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.518 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.519 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:50.519 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.484033+0000", 2024-09-13T15:00:50.519 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0031378130000000001 2024-09-13T15:00:50.519 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.519 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.519 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:50.519 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.484167+0000", 2024-09-13T15:00:50.519 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013311400000000001 2024-09-13T15:00:50.519 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.519 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.519 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:00:50.519 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.484302+0000", 2024-09-13T15:00:50.519 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013587199999999999 2024-09-13T15:00:50.520 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.520 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.520 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:50.520 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.490200+0000", 2024-09-13T15:00:50.520 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.005897322 2024-09-13T15:00:50.520 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.520 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.520 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:50.520 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.500604+0000", 2024-09-13T15:00:50.520 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010404544 2024-09-13T15:00:50.520 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.520 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.520 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:50.520 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504207+0000", 2024-09-13T15:00:50.521 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0036024759999999999 2024-09-13T15:00:50.521 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.521 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.521 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:50.521 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504297+0000", 2024-09-13T15:00:50.521 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.9969999999999994e-05 2024-09-13T15:00:50.521 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.521 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:50.521 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.521 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.521 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.521 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:192062 9.7 9:e98e390c:::benchmark_data_smithi096_256909_object72654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e331)", 2024-09-13T15:00:50.521 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:50.482026+0000", 2024-09-13T15:00:50.521 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.024853654999999999, 2024-09-13T15:00:50.522 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.025063172000000002, 2024-09-13T15:00:50.522 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:50.522 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:00:50.522 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:50.522 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:50.522 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:50.522 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 192062 2024-09-13T15:00:50.522 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.522 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:50.522 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.522 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:50.522 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.482026+0000", 2024-09-13T15:00:50.522 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.523 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.523 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.523 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:50.523 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.482026+0000", 2024-09-13T15:00:50.523 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.523 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.523 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.523 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:50.523 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.482025+0000", 2024-09-13T15:00:50.523 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:00:50.523 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.523 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.523 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:50.524 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.482030+0000", 2024-09-13T15:00:50.524 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6190000000000002e-06 2024-09-13T15:00:50.524 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.524 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.524 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:50.524 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.482037+0000", 2024-09-13T15:00:50.524 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2150000000000004e-06 2024-09-13T15:00:50.524 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.525 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.525 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:50.525 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.482041+0000", 2024-09-13T15:00:50.525 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8569999999999997e-06 2024-09-13T15:00:50.525 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.525 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.525 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:50.525 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.493026+0000", 2024-09-13T15:00:50.525 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010984707 2024-09-13T15:00:50.525 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.525 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.526 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:50.526 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.493187+0000", 2024-09-13T15:00:50.526 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016097799999999999 2024-09-13T15:00:50.526 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.526 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.526 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:00:50.526 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.493356+0000", 2024-09-13T15:00:50.526 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000169767 2024-09-13T15:00:50.526 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.526 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.526 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:50.526 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.506043+0000", 2024-09-13T15:00:50.526 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012686226 2024-09-13T15:00:50.527 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.527 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:50.527 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.527 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.527 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.527 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:192067 9.f 9:f45b1d07:::benchmark_data_smithi096_256909_object72659:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e331)", 2024-09-13T15:00:50.527 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:50.483108+0000", 2024-09-13T15:00:50.527 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.023771759, 2024-09-13T15:00:50.527 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.024052852, 2024-09-13T15:00:50.527 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:50.527 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:00:50.527 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:50.527 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:50.527 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:50.528 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 192067 2024-09-13T15:00:50.528 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.528 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:50.528 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.528 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:50.528 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.483108+0000", 2024-09-13T15:00:50.528 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.528 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.528 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.528 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:50.528 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.483108+0000", 2024-09-13T15:00:50.528 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.528 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.529 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.529 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:50.529 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.483107+0000", 2024-09-13T15:00:50.529 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:00:50.529 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.529 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.529 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:50.529 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.483113+0000", 2024-09-13T15:00:50.529 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5459999999999999e-06 2024-09-13T15:00:50.529 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.529 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.529 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:50.529 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.483124+0000", 2024-09-13T15:00:50.530 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0730999999999999e-05 2024-09-13T15:00:50.530 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.530 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.530 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:50.530 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.483129+0000", 2024-09-13T15:00:50.530 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2209999999999996e-06 2024-09-13T15:00:50.530 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.530 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.530 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:50.530 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.490222+0000", 2024-09-13T15:00:50.530 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.007093294 2024-09-13T15:00:50.530 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.530 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.530 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:50.530 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.490355+0000", 2024-09-13T15:00:50.531 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013255400000000001 2024-09-13T15:00:50.531 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.531 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.531 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:00:50.531 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.490494+0000", 2024-09-13T15:00:50.531 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013915499999999999 2024-09-13T15:00:50.531 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.531 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.531 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:50.531 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.500723+0000", 2024-09-13T15:00:50.531 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010228602 2024-09-13T15:00:50.531 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.531 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:50.531 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.532 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.532 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.532 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:192068 9.f 9:fdd7067f:::benchmark_data_smithi096_256909_object72660:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e331)", 2024-09-13T15:00:50.532 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:50.483255+0000", 2024-09-13T15:00:50.532 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.023624850999999999, 2024-09-13T15:00:50.532 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.023976151000000001, 2024-09-13T15:00:50.532 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:50.532 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:00:50.532 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:50.532 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:50.532 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:50.532 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 192068 2024-09-13T15:00:50.532 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.532 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:50.533 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.533 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:50.533 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.483255+0000", 2024-09-13T15:00:50.533 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.533 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.533 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.533 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:50.533 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.483255+0000", 2024-09-13T15:00:50.533 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.533 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.533 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.533 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:50.533 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.483254+0000", 2024-09-13T15:00:50.533 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:00:50.534 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.534 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.534 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:50.534 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.483259+0000", 2024-09-13T15:00:50.534 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2939999999999999e-06 2024-09-13T15:00:50.534 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.534 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.534 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:50.534 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.483268+0000", 2024-09-13T15:00:50.534 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.2580000000000001e-06 2024-09-13T15:00:50.534 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.534 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.534 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:50.534 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.483273+0000", 2024-09-13T15:00:50.535 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6990000000000002e-06 2024-09-13T15:00:50.535 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.535 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.535 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:50.535 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.497086+0000", 2024-09-13T15:00:50.535 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.01381338 2024-09-13T15:00:50.535 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.535 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.535 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:50.535 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.497242+0000", 2024-09-13T15:00:50.535 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000155644 2024-09-13T15:00:50.535 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.535 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.536 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:00:50.536 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.497401+0000", 2024-09-13T15:00:50.536 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015928000000000001 2024-09-13T15:00:50.536 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.536 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:50.536 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.536 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.536 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.536 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:192075 9.6 9:602a988d:::benchmark_data_smithi096_256909_object72667:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e331)", 2024-09-13T15:00:50.536 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:50.499666+0000", 2024-09-13T15:00:50.536 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0072142380000000004, 2024-09-13T15:00:50.536 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0076302469999999997, 2024-09-13T15:00:50.537 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:50.537 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:00:50.537 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:50.537 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:50.537 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:50.537 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 192075 2024-09-13T15:00:50.537 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.537 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:50.537 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.537 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:50.537 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.499666+0000", 2024-09-13T15:00:50.537 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.537 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.538 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.538 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:50.538 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.499666+0000", 2024-09-13T15:00:50.538 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.538 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.538 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.538 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:50.538 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.499664+0000", 2024-09-13T15:00:50.538 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:50.538 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.538 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.538 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:50.538 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.499669+0000", 2024-09-13T15:00:50.538 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6650000000000002e-06 2024-09-13T15:00:50.539 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.539 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.539 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:50.539 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.499677+0000", 2024-09-13T15:00:50.539 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.259e-06 2024-09-13T15:00:50.539 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.539 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.539 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:50.539 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.499682+0000", 2024-09-13T15:00:50.539 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1999999999999996e-06 2024-09-13T15:00:50.539 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.539 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.540 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:50.540 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.499724+0000", 2024-09-13T15:00:50.540 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2783e-05 2024-09-13T15:00:50.540 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.540 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.540 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:50.540 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.499844+0000", 2024-09-13T15:00:50.540 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119206 2024-09-13T15:00:50.540 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.540 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.540 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:00:50.540 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.502089+0000", 2024-09-13T15:00:50.540 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0022450930000000001 2024-09-13T15:00:50.541 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.541 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.541 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:50.541 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.506248+0000", 2024-09-13T15:00:50.541 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0041591140000000002 2024-09-13T15:00:50.541 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.541 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:50.541 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.541 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.541 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.541 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.46040.0:192061 9.f e331/316 ondisk, result = 0)", 2024-09-13T15:00:50.541 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:50.501869+0000", 2024-09-13T15:00:50.541 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0050109719999999998, 2024-09-13T15:00:50.542 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0054952289999999999, 2024-09-13T15:00:50.542 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:50.542 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:00:50.542 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:50.542 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.542 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:50.542 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.501869+0000", 2024-09-13T15:00:50.542 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.542 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.542 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.542 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:50.542 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.501869+0000", 2024-09-13T15:00:50.542 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.542 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.543 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.543 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:50.543 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.501868+0000", 2024-09-13T15:00:50.543 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:00:50.543 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.543 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.543 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:50.543 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.501874+0000", 2024-09-13T15:00:50.543 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8119999999999996e-06 2024-09-13T15:00:50.543 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.543 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.543 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:50.543 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.501890+0000", 2024-09-13T15:00:50.544 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5418e-05 2024-09-13T15:00:50.544 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.544 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.544 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:50.544 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.501895+0000", 2024-09-13T15:00:50.544 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7300000000000002e-06 2024-09-13T15:00:50.544 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.544 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.544 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:50.544 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504191+0000", 2024-09-13T15:00:50.544 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002296078 2024-09-13T15:00:50.544 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.544 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.544 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:50.544 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504200+0000", 2024-09-13T15:00:50.545 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7059999999999993e-06 2024-09-13T15:00:50.545 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.545 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:50.545 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.545 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.545 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.545 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.46040.0:192056 9.7 e331/316 ondisk, result = 0)", 2024-09-13T15:00:50.545 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:50.504008+0000", 2024-09-13T15:00:50.545 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0028723479999999998, 2024-09-13T15:00:50.545 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0034059160000000002, 2024-09-13T15:00:50.545 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:50.545 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:00:50.545 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:50.545 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.546 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:50.546 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504008+0000", 2024-09-13T15:00:50.546 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.546 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.546 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.546 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:50.546 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504008+0000", 2024-09-13T15:00:50.546 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.546 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.546 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.546 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:50.546 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504007+0000", 2024-09-13T15:00:50.546 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:00:50.546 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.546 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.547 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:50.547 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504011+0000", 2024-09-13T15:00:50.547 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6610000000000002e-06 2024-09-13T15:00:50.547 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.547 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.547 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:50.547 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504023+0000", 2024-09-13T15:00:50.547 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2289999999999999e-05 2024-09-13T15:00:50.547 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.547 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.547 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:50.547 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504028+0000", 2024-09-13T15:00:50.547 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.164e-06 2024-09-13T15:00:50.547 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.547 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.548 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:50.548 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.505900+0000", 2024-09-13T15:00:50.548 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0018718929999999999 2024-09-13T15:00:50.548 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.548 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.548 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:50.548 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.505994+0000", 2024-09-13T15:00:50.548 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.3419999999999997e-05 2024-09-13T15:00:50.548 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.548 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:50.548 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.548 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.548 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.548 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.46040.0:192068 9.f e331/316)", 2024-09-13T15:00:50.549 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:50.504112+0000", 2024-09-13T15:00:50.549 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0027676570000000002, 2024-09-13T15:00:50.549 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0033499950000000001, 2024-09-13T15:00:50.549 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:50.549 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "queued for pg", 2024-09-13T15:00:50.549 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:50.549 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.549 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:50.549 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504112+0000", 2024-09-13T15:00:50.549 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.549 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.549 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.549 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:50.550 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504112+0000", 2024-09-13T15:00:50.550 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.550 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.550 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.550 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:50.550 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504112+0000", 2024-09-13T15:00:50.550 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:00:50.550 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.550 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.550 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:50.550 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504115+0000", 2024-09-13T15:00:50.550 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9749999999999999e-06 2024-09-13T15:00:50.550 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.550 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.551 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:50.551 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504125+0000", 2024-09-13T15:00:50.551 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5400000000000001e-06 2024-09-13T15:00:50.551 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.551 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.551 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:50.551 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504129+0000", 2024-09-13T15:00:50.551 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1250000000000003e-06 2024-09-13T15:00:50.551 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.551 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:50.551 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.551 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.551 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.551 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.46040.0:192076 9.5 e331/302 9:afb818c6:::benchmark_data_smithi096_256909_object72668:head v 331'12133, mlcod=331'12131)", 2024-09-13T15:00:50.552 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:50.504230+0000", 2024-09-13T15:00:50.552 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0026498440000000002, 2024-09-13T15:00:50.552 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0032776929999999999, 2024-09-13T15:00:50.552 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:50.552 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:00:50.552 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:50.552 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.552 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:50.552 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504230+0000", 2024-09-13T15:00:50.552 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.552 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.552 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.552 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:50.553 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504230+0000", 2024-09-13T15:00:50.553 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.553 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.553 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.553 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:50.553 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504230+0000", 2024-09-13T15:00:50.553 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:00:50.553 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.553 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.553 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:50.554 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504236+0000", 2024-09-13T15:00:50.554 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4710000000000004e-06 2024-09-13T15:00:50.554 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.554 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.554 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:50.554 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504247+0000", 2024-09-13T15:00:50.554 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0519999999999999e-05 2024-09-13T15:00:50.554 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.554 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.554 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:50.554 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504252+0000", 2024-09-13T15:00:50.554 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4179999999999996e-06 2024-09-13T15:00:50.555 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.555 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.555 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:50.555 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504319+0000", 2024-09-13T15:00:50.555 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7393999999999999e-05 2024-09-13T15:00:50.555 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.555 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.555 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:50.555 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.504342+0000", 2024-09-13T15:00:50.555 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2707000000000001e-05 2024-09-13T15:00:50.555 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.555 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:50.555 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.556 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.556 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.556 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.46040.0:192062 9.7 e331/316)", 2024-09-13T15:00:50.556 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:50.505591+0000", 2024-09-13T15:00:50.556 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.001289255, 2024-09-13T15:00:50.556 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0019718600000000002, 2024-09-13T15:00:50.556 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:50.556 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "queued for pg", 2024-09-13T15:00:50.556 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:50.556 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.556 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:50.556 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.505591+0000", 2024-09-13T15:00:50.557 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.557 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.557 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.557 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:50.557 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.505591+0000", 2024-09-13T15:00:50.557 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.557 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.557 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.557 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:50.557 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.505590+0000", 2024-09-13T15:00:50.557 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:00:50.557 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.557 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.558 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:50.558 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.505595+0000", 2024-09-13T15:00:50.558 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0309999999999998e-06 2024-09-13T15:00:50.558 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.558 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.558 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:50.558 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.505609+0000", 2024-09-13T15:00:50.558 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3492999999999999e-05 2024-09-13T15:00:50.558 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.558 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.558 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:50.558 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.505615+0000", 2024-09-13T15:00:50.558 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4169999999999997e-06 2024-09-13T15:00:50.559 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.559 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:50.559 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.559 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.559 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.559 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.46040.0:192077 9.3 e331/323 9:c222713c:::benchmark_data_smithi096_256909_object72669:head v 331'12029, mlcod=331'12027)", 2024-09-13T15:00:50.559 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:50.506502+0000", 2024-09-13T15:00:50.559 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.000377912, 2024-09-13T15:00:50.559 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0011043509999999999, 2024-09-13T15:00:50.559 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:50.559 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:00:50.559 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:50.560 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.560 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:50.560 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.506502+0000", 2024-09-13T15:00:50.560 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.560 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.560 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.560 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:50.560 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.506502+0000", 2024-09-13T15:00:50.560 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.560 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.560 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.560 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:50.560 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.506501+0000", 2024-09-13T15:00:50.561 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:00:50.561 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.561 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.561 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:50.561 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.506508+0000", 2024-09-13T15:00:50.561 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4500000000000001e-06 2024-09-13T15:00:50.561 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.561 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.561 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:50.561 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.506524+0000", 2024-09-13T15:00:50.561 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6390999999999999e-05 2024-09-13T15:00:50.561 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.561 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.562 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:50.562 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.506530+0000", 2024-09-13T15:00:50.562 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1709999999999999e-06 2024-09-13T15:00:50.562 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.562 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.562 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:50.562 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.506572+0000", 2024-09-13T15:00:50.562 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.172e-05 2024-09-13T15:00:50.562 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.562 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.562 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:50.562 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.506591+0000", 2024-09-13T15:00:50.562 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8660000000000001e-05 2024-09-13T15:00:50.563 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.563 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:50.563 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.563 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.563 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.563 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.46040.0:192080 9.8 e331/318)", 2024-09-13T15:00:50.563 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:50.507534+0000", 2024-09-13T15:00:50.563 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 4294967295.9993458, 2024-09-13T15:00:50.563 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012348700000000001, 2024-09-13T15:00:50.563 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:50.563 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "queued for pg", 2024-09-13T15:00:50.563 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:50.563 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.564 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:50.564 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.507534+0000", 2024-09-13T15:00:50.564 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.564 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.564 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.564 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:50.564 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.507534+0000", 2024-09-13T15:00:50.564 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:50.564 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.564 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.564 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:50.564 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.507534+0000", 2024-09-13T15:00:50.565 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:00:50.565 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.565 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.565 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:50.565 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.507540+0000", 2024-09-13T15:00:50.565 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6460000000000004e-06 2024-09-13T15:00:50.565 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.565 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.565 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:50.565 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.507550+0000", 2024-09-13T15:00:50.565 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0003000000000001e-05 2024-09-13T15:00:50.565 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.565 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.565 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:50.566 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.507555+0000", 2024-09-13T15:00:50.566 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.006e-06 2024-09-13T15:00:50.566 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:50.566 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:50.566 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:50.566 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:50.507660+0000", 2024-09-13T15:00:50.566 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000104854 2024-09-13T15:00:50.566 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.566 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:50.566 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.566 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:50.566 INFO:teuthology.orchestra.run.smithi204.stdout: ], 2024-09-13T15:00:50.566 INFO:teuthology.orchestra.run.smithi204.stdout: "num_ops": 13 2024-09-13T15:00:50.566 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:00:50.575 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:50.605 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:50.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:50.707 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:50.735 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:50.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:50.836 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:50.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.885 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:50.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:50.986 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:51.018 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:51.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:51.120 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:51.158 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:51.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:51.260 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:51.293 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:51.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:51.395 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:51.429 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:51.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:51.498 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T15:00:51.531 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:51.588 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:51.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:51.690 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:51 smithi204 ceph-mon[87071]: pgmap v1099: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 32438/145578 objects degraded (22.282%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:00:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:51.724 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:51.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:51.728 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:51.763 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:00:51.763 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:00:51.826 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:51.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.879 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:51.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:51.980 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:51.987 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:51 smithi096 ceph-mon[98901]: pgmap v1099: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 32438/145578 objects degraded (22.282%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:00:51.987 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:51.988 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:51 smithi096 ceph-mon[102928]: pgmap v1099: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 32438/145578 objects degraded (22.282%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:00:51.988 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:51.999 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:00:52.028 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 6] out_osds: [3] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2024-09-13T15:00:52.028 INFO:tasks.thrashosds.thrasher:Reviving osd 3 2024-09-13T15:00:52.029 INFO:tasks.cephadm.osd.3:Restarting osd.3 (starting--it wasn't running)... 2024-09-13T15:00:52.029 DEBUG:teuthology.orchestra.run.smithi096:osd.3> sudo journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.3.service 2024-09-13T15:00:52.031 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl start ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.3 2024-09-13T15:00:52.035 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:52.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:52.137 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:52.180 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:52.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:52.282 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:52.291 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:52 smithi096 systemd[1]: Starting Ceph osd.3 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:00:52.310 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:52.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:52.412 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:52.443 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:52.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:52.545 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:52.573 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:52.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:52.675 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:52.703 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:52.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:52.804 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:52.849 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:52.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:52.951 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:52.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.986 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:52.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:53.088 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:53.114 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:53.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:53.216 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:53.246 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:53.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:53.348 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:53.381 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:53.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:53.460 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:53 smithi096 podman[275391]: 2024-09-13 15:00:53.183110554 +0000 UTC m=+0.128090006 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:00:53.460 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:53 smithi096 podman[275391]: 2024-09-13 15:00:53.459366401 +0000 UTC m=+0.404345838 container create 8a5475a73fca24d8a1b4a541055ccb0a0ab4116e010d8d3d95c84902943b4621 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-13T15:00:53.483 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:53.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.525 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:53.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:53.626 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:53.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.665 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:53.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:53.764 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:00:53.766 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:53.772 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:53 smithi096 ceph-mon[98901]: pgmap v1100: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 32438/145578 objects degraded (22.282%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:00:53.772 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:53 smithi096 ceph-mon[102928]: pgmap v1100: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 32438/145578 objects degraded (22.282%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:00:53.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.811 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:53.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:53.875 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:53 smithi204 ceph-mon[87071]: pgmap v1100: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 32438/145578 objects degraded (22.282%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:00:53.912 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:53.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.961 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:53.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:54.062 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:54.093 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:54.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:54.195 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:54.229 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:54.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:54.323 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:54 smithi096 podman[275391]: 2024-09-13 15:00:54.064604499 +0000 UTC m=+1.009583947 container init 8a5475a73fca24d8a1b4a541055ccb0a0ab4116e010d8d3d95c84902943b4621 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:00:54.323 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:54 smithi096 podman[275391]: 2024-09-13 15:00:54.071862243 +0000 UTC m=+1.016841685 container start 8a5475a73fca24d8a1b4a541055ccb0a0ab4116e010d8d3d95c84902943b4621 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:00:54.323 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:54 smithi096 podman[275391]: 2024-09-13 15:00:54.181244406 +0000 UTC m=+1.126223858 container attach 8a5475a73fca24d8a1b4a541055ccb0a0ab4116e010d8d3d95c84902943b4621 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T15:00:54.331 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:54.348 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:54.366 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:54.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:54.467 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:54.500 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:54.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:54.566 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:00:54.566 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T15:00:54.566 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T15:00:54.566 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T15:00:54.566 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:00:54.602 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:54.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.639 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:54.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:54.653 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T15:00:54.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:54 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/107133945' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:54.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:54 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:54.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:54 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 32438/145578 objects degraded (22.282%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:54.740 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:54.773 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:54.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:54.811 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:54 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/107133945' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:54.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:54 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:54.812 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:54 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 32438/145578 objects degraded (22.282%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:54.812 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:54 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/107133945' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:54.812 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:54 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:54.812 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:54 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 32438/145578 objects degraded (22.282%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:54.875 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:54.902 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:54.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:55.004 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:55.044 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:55.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:55.147 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:55.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.185 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:55.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:55.286 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:55.311 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T15:00:55.322 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:55.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:55.424 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:55.471 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:55.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:55.573 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:55.608 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:55.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:55.648 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:55 smithi204 ceph-mon[87071]: pgmap v1101: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 801 op/s; 31775/137064 objects degraded (23.183%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T15:00:55.711 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:55.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.760 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:55.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:55.772 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:55 smithi096 ceph-mon[98901]: pgmap v1101: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 801 op/s; 31775/137064 objects degraded (23.183%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T15:00:55.772 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:55 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:55.772 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:55 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:00:55.772 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:55 smithi096 ceph-mon[98901]: osdmap e332: 8 total, 7 up, 7 in 2024-09-13T15:00:55.772 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:00:55.773 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:55 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[275442]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:55.773 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:55 smithi096 bash[275391]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:55.773 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:55 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[275442]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:55.773 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:55 smithi096 bash[275391]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:55.773 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:55 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[275442]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T15:00:55.773 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:55 smithi096 bash[275391]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T15:00:55.774 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:55 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[275442]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:55.774 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:55 smithi096 bash[275391]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:55.774 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:55 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[275442]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:55.774 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:55 smithi096 bash[275391]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:55.774 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:55 smithi096 ceph-mon[102928]: pgmap v1101: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 801 op/s; 31775/137064 objects degraded (23.183%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T15:00:55.774 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:55 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:55.774 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:55 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:00:55.774 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:55 smithi096 ceph-mon[102928]: osdmap e332: 8 total, 7 up, 7 in 2024-09-13T15:00:55.862 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:55.892 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:55.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:55.915 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:55 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:55.915 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:55 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:00:55.915 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:55 smithi204 ceph-mon[87071]: osdmap e332: 8 total, 7 up, 7 in 2024-09-13T15:00:55.995 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:56.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.022 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:56.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:56.073 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:55 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[275442]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T15:00:56.073 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:55 smithi096 bash[275391]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T15:00:56.073 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:55 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[275442]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-13T15:00:56.073 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:55 smithi096 bash[275391]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-13T15:00:56.123 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:56.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.153 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:56.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:56.255 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:56.288 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:56.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:56.389 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:56.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.422 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:56.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:56.524 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:56.565 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:56.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:56.667 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:56.693 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:56.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:56.795 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:56.829 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:56.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:56.931 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:56.973 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:56.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:57.075 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:57.109 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:57.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:57.211 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:57.245 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:57.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:57.347 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:57.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.395 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:57.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:57.497 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:57.536 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:57.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:57.637 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:57.677 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:57.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:57.779 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:57.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.814 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:57.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:57.915 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:57.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:57 smithi096 ceph-mon[98901]: pgmap v1103: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 775 op/s; 31486/133914 objects degraded (23.512%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:00:57.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:57 smithi096 ceph-mon[102928]: pgmap v1103: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 775 op/s; 31486/133914 objects degraded (23.512%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:00:57.947 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[275442]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T15:00:57.947 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:57 smithi096 bash[275391]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T15:00:57.947 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[275442]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-13T15:00:57.947 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:57 smithi096 bash[275391]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-13T15:00:57.948 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[275442]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T15:00:57.948 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:57 smithi096 bash[275391]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T15:00:57.948 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[275442]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T15:00:57.948 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:57 smithi096 bash[275391]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T15:00:57.948 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate[275442]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-13T15:00:57.948 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:57 smithi096 bash[275391]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-13T15:00:57.948 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:57 smithi096 podman[275391]: 2024-09-13 15:00:57.806048776 +0000 UTC m=+4.751028216 container died 8a5475a73fca24d8a1b4a541055ccb0a0ab4116e010d8d3d95c84902943b4621 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True) 2024-09-13T15:00:57.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.964 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:57.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:57 smithi204 ceph-mon[87071]: pgmap v1103: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 775 op/s; 31486/133914 objects degraded (23.512%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:00:58.065 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:58.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.088 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:58.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:58.189 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:58.217 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:00:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:00:57] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.51.0" 2024-09-13T15:00:58.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.221 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:58.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:58.298 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:00:58.298 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T15:00:58.298 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T15:00:58.299 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T15:00:58.299 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.299 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:51945 9.f 9:fc0db2d0:::benchmark_data_smithi096_256909_object51944:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2024-09-13T15:00:58.299 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:14.513677+0000", 2024-09-13T15:00:58.299 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 163.51622266699999, 2024-09-13T15:00:58.299 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19199633699999999, 2024-09-13T15:00:58.299 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.299 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.299 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.299 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.299 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.299 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 51945 2024-09-13T15:00:58.300 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.300 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.300 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.300 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.300 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513677+0000", 2024-09-13T15:00:58.300 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.300 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.300 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.300 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.300 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513677+0000", 2024-09-13T15:00:58.300 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.300 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.301 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.301 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.301 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513676+0000", 2024-09-13T15:00:58.301 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:58.301 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.301 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.301 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.301 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513728+0000", 2024-09-13T15:00:58.301 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2562000000000003e-05 2024-09-13T15:00:58.301 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.301 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.302 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.302 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513739+0000", 2024-09-13T15:00:58.302 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1173e-05 2024-09-13T15:00:58.302 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.302 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.302 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.302 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513745+0000", 2024-09-13T15:00:58.302 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9560000000000002e-06 2024-09-13T15:00:58.303 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.303 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.303 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.303 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513780+0000", 2024-09-13T15:00:58.303 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4693999999999997e-05 2024-09-13T15:00:58.303 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.303 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.303 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.303 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539271+0000", 2024-09-13T15:00:58.303 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.025491362 2024-09-13T15:00:58.303 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.304 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.304 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T15:00:58.304 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539413+0000", 2024-09-13T15:00:58.304 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000142058 2024-09-13T15:00:58.304 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.304 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.304 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.304 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.544116+0000", 2024-09-13T15:00:58.304 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0047030259999999999 2024-09-13T15:00:58.304 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.304 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.304 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.305 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701627+0000", 2024-09-13T15:00:58.305 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15751026100000001 2024-09-13T15:00:58.305 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.305 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.305 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.305 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705595+0000", 2024-09-13T15:00:58.305 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0039685709999999997 2024-09-13T15:00:58.305 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.305 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.305 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.305 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705639+0000", 2024-09-13T15:00:58.306 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3959999999999999e-05 2024-09-13T15:00:58.306 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.306 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.306 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.306 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705673+0000", 2024-09-13T15:00:58.306 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3964999999999998e-05 2024-09-13T15:00:58.306 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.306 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.306 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.306 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.306 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.306 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:51946 9.7 9:ecbe616b:::benchmark_data_smithi096_256909_object51945:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2024-09-13T15:00:58.306 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:14.513993+0000", 2024-09-13T15:00:58.307 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 163.515906683, 2024-09-13T15:00:58.307 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.187708018, 2024-09-13T15:00:58.307 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.307 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.307 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.307 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.307 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.307 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 51946 2024-09-13T15:00:58.307 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.307 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.307 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.307 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.307 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513993+0000", 2024-09-13T15:00:58.308 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.308 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.308 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.308 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.308 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513993+0000", 2024-09-13T15:00:58.308 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.308 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.308 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.308 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.308 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.513992+0000", 2024-09-13T15:00:58.308 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:00:58.308 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.308 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.309 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.309 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.514057+0000", 2024-09-13T15:00:58.309 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5326000000000004e-05 2024-09-13T15:00:58.309 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.309 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.309 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.309 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539161+0000", 2024-09-13T15:00:58.309 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.025103503999999999 2024-09-13T15:00:58.310 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.310 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.310 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.310 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539174+0000", 2024-09-13T15:00:58.310 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2542e-05 2024-09-13T15:00:58.310 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.310 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.310 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.310 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539542+0000", 2024-09-13T15:00:58.310 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00036863999999999999 2024-09-13T15:00:58.310 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.310 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.311 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.311 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539683+0000", 2024-09-13T15:00:58.311 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014123200000000001 2024-09-13T15:00:58.311 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.311 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.311 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T15:00:58.311 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539825+0000", 2024-09-13T15:00:58.311 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014115699999999999 2024-09-13T15:00:58.311 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.311 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.311 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.311 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.544025+0000", 2024-09-13T15:00:58.311 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0042005430000000002 2024-09-13T15:00:58.311 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.312 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.312 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.312 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.544395+0000", 2024-09-13T15:00:58.312 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000369432 2024-09-13T15:00:58.312 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.312 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.312 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.312 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701625+0000", 2024-09-13T15:00:58.312 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.157230326 2024-09-13T15:00:58.312 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.312 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.312 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.313 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701664+0000", 2024-09-13T15:00:58.313 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8760000000000002e-05 2024-09-13T15:00:58.313 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.313 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.313 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.313 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701701+0000", 2024-09-13T15:00:58.313 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7243999999999998e-05 2024-09-13T15:00:58.313 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.313 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.313 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.313 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.313 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.313 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:51947 9.f 9:f8f9d21b:::benchmark_data_smithi096_256909_object51946:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2024-09-13T15:00:58.314 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:14.539191+0000", 2024-09-13T15:00:58.314 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 163.49070817500001, 2024-09-13T15:00:58.314 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.166579427, 2024-09-13T15:00:58.314 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.314 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.314 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.314 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.314 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.314 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 51947 2024-09-13T15:00:58.314 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.314 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.314 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.315 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.315 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539191+0000", 2024-09-13T15:00:58.315 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.315 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.315 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.315 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.315 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539191+0000", 2024-09-13T15:00:58.315 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.315 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.315 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.315 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.315 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539190+0000", 2024-09-13T15:00:58.315 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:58.315 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.315 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.316 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.316 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539238+0000", 2024-09-13T15:00:58.316 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8136000000000001e-05 2024-09-13T15:00:58.316 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.316 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.316 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.316 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539249+0000", 2024-09-13T15:00:58.316 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1051e-05 2024-09-13T15:00:58.316 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.316 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.316 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.316 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539253+0000", 2024-09-13T15:00:58.316 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4120000000000003e-06 2024-09-13T15:00:58.316 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.317 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.317 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.317 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.542175+0000", 2024-09-13T15:00:58.317 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029216730000000001 2024-09-13T15:00:58.317 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.317 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.317 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.317 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.542272+0000", 2024-09-13T15:00:58.317 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6768999999999995e-05 2024-09-13T15:00:58.317 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.317 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.317 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,3", 2024-09-13T15:00:58.317 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.542363+0000", 2024-09-13T15:00:58.318 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1615999999999998e-05 2024-09-13T15:00:58.318 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.318 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.318 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.318 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.544362+0000", 2024-09-13T15:00:58.318 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0019985200000000002 2024-09-13T15:00:58.318 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.318 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.318 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.318 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.701753+0000", 2024-09-13T15:00:58.318 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15739146200000001 2024-09-13T15:00:58.318 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.318 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.319 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.319 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705705+0000", 2024-09-13T15:00:58.319 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0039517579999999997 2024-09-13T15:00:58.319 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.319 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.319 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.319 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705742+0000", 2024-09-13T15:00:58.319 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6715000000000003e-05 2024-09-13T15:00:58.319 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.319 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.319 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.319 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.705771+0000", 2024-09-13T15:00:58.319 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8858999999999999e-05 2024-09-13T15:00:58.319 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.320 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.320 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.320 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.320 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.320 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:51948 9.6 9:6c080a85:::benchmark_data_smithi096_256909_object51947:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2024-09-13T15:00:58.320 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:58:14.539259+0000", 2024-09-13T15:00:58.320 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 163.49064075199999, 2024-09-13T15:00:58.320 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16478026000000001, 2024-09-13T15:00:58.320 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.320 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.320 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.320 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.321 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.321 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 51948 2024-09-13T15:00:58.321 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.321 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.321 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.321 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.321 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539259+0000", 2024-09-13T15:00:58.321 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.321 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.321 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.321 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.321 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539259+0000", 2024-09-13T15:00:58.321 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.321 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.322 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.322 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.322 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539258+0000", 2024-09-13T15:00:58.322 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:00:58.322 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.322 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.322 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.322 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539295+0000", 2024-09-13T15:00:58.322 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7004e-05 2024-09-13T15:00:58.322 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.322 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.322 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.322 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539302+0000", 2024-09-13T15:00:58.322 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0350000000000004e-06 2024-09-13T15:00:58.322 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.323 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.323 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.323 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539306+0000", 2024-09-13T15:00:58.323 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.929e-06 2024-09-13T15:00:58.323 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.323 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.323 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.323 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539804+0000", 2024-09-13T15:00:58.323 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00049790399999999997 2024-09-13T15:00:58.323 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.323 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.323 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.323 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.539898+0000", 2024-09-13T15:00:58.324 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.4310999999999999e-05 2024-09-13T15:00:58.324 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.324 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.324 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:00:58.324 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.540041+0000", 2024-09-13T15:00:58.324 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014301200000000001 2024-09-13T15:00:58.324 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.324 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.324 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.324 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.543085+0000", 2024-09-13T15:00:58.324 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0030438230000000002 2024-09-13T15:00:58.324 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.324 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.325 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.325 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.703863+0000", 2024-09-13T15:00:58.325 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16077804600000001 2024-09-13T15:00:58.325 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.325 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.325 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.325 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.703953+0000", 2024-09-13T15:00:58.325 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.9790000000000006e-05 2024-09-13T15:00:58.325 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.325 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.325 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.325 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.704001+0000", 2024-09-13T15:00:58.326 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7555000000000001e-05 2024-09-13T15:00:58.326 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.326 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.326 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.326 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:58:14.704039+0000", 2024-09-13T15:00:58.326 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8339e-05 2024-09-13T15:00:58.326 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.326 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.326 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.326 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.326 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.326 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:122522 9.f 9:f8ed9e47:::benchmark_data_smithi096_256909_object3114:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-13T15:00:58.326 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:25.398448+0000", 2024-09-13T15:00:58.327 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 92.631451529000003, 2024-09-13T15:00:58.327 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19402323399999999, 2024-09-13T15:00:58.327 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.327 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.327 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.327 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.327 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.327 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 122522 2024-09-13T15:00:58.327 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.327 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.327 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.327 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.328 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.398448+0000", 2024-09-13T15:00:58.328 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.328 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.328 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.328 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.328 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.398448+0000", 2024-09-13T15:00:58.328 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.328 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.328 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.328 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.328 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.398447+0000", 2024-09-13T15:00:58.328 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:58.328 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.329 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.329 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.329 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.398454+0000", 2024-09-13T15:00:58.329 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9990000000000002e-06 2024-09-13T15:00:58.329 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.329 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.329 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.329 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.398466+0000", 2024-09-13T15:00:58.329 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2388999999999999e-05 2024-09-13T15:00:58.329 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.329 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.330 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.330 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.398472+0000", 2024-09-13T15:00:58.330 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1020000000000002e-06 2024-09-13T15:00:58.330 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.330 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.330 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.330 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.404885+0000", 2024-09-13T15:00:58.330 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0064125170000000004 2024-09-13T15:00:58.330 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.330 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.330 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.330 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.405047+0000", 2024-09-13T15:00:58.330 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016234999999999999 2024-09-13T15:00:58.331 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.331 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.331 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:00:58.331 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.405190+0000", 2024-09-13T15:00:58.331 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014338299999999999 2024-09-13T15:00:58.331 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.331 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.331 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.331 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.413527+0000", 2024-09-13T15:00:58.331 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0083364870000000001 2024-09-13T15:00:58.331 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.332 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.332 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.332 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.413559+0000", 2024-09-13T15:00:58.332 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1881999999999997e-05 2024-09-13T15:00:58.332 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.332 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.332 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.332 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.452008+0000", 2024-09-13T15:00:58.332 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.038449695999999998 2024-09-13T15:00:58.332 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.332 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.332 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.333 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.452108+0000", 2024-09-13T15:00:58.333 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9802000000000004e-05 2024-09-13T15:00:58.333 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.333 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.333 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.333 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.592471+0000", 2024-09-13T15:00:58.333 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.14036306700000001 2024-09-13T15:00:58.333 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.333 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.333 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.333 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.333 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.334 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:122523 9.7 9:e3764e48:::benchmark_data_smithi096_256909_object3115:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-13T15:00:58.334 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:25.402697+0000", 2024-09-13T15:00:58.334 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 92.627202162000003, 2024-09-13T15:00:58.334 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19291148, 2024-09-13T15:00:58.334 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.334 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.334 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.334 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.334 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.334 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 122523 2024-09-13T15:00:58.334 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.334 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.335 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.335 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.335 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.402697+0000", 2024-09-13T15:00:58.335 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.335 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.335 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.335 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.335 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.402697+0000", 2024-09-13T15:00:58.335 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.335 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.335 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.335 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.335 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.402696+0000", 2024-09-13T15:00:58.336 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:00:58.336 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.336 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.336 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.336 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.402702+0000", 2024-09-13T15:00:58.336 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4489999999999998e-06 2024-09-13T15:00:58.336 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.336 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.336 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.336 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.402710+0000", 2024-09-13T15:00:58.337 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3259999999999996e-06 2024-09-13T15:00:58.337 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.337 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.337 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.337 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.402715+0000", 2024-09-13T15:00:58.337 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.395e-06 2024-09-13T15:00:58.337 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.337 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.337 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.337 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.414165+0000", 2024-09-13T15:00:58.338 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011449234000000001 2024-09-13T15:00:58.338 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.338 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.338 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.338 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.414312+0000", 2024-09-13T15:00:58.338 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000146988 2024-09-13T15:00:58.338 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.338 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.338 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:00:58.338 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.414478+0000", 2024-09-13T15:00:58.338 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016611100000000001 2024-09-13T15:00:58.338 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.339 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.339 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.339 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.435522+0000", 2024-09-13T15:00:58.339 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.021044054999999999 2024-09-13T15:00:58.339 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.339 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.339 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.339 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.592523+0000", 2024-09-13T15:00:58.339 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15700096199999999 2024-09-13T15:00:58.339 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.339 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.339 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.340 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.592649+0000", 2024-09-13T15:00:58.340 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012593399999999999 2024-09-13T15:00:58.340 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.340 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.340 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.592697+0000", 2024-09-13T15:00:58.340 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8720000000000001e-05 2024-09-13T15:00:58.340 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.340 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.340 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.595609+0000", 2024-09-13T15:00:58.340 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029114879999999998 2024-09-13T15:00:58.340 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.341 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.341 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.341 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.341 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.341 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:122531 9.7 9:e98e75df:::benchmark_data_smithi096_256909_object3123:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-13T15:00:58.341 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:25.409137+0000", 2024-09-13T15:00:58.341 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 92.620762546999998, 2024-09-13T15:00:58.341 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.189021246, 2024-09-13T15:00:58.341 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.341 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.341 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.341 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.341 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.342 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 122531 2024-09-13T15:00:58.342 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.342 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.342 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.342 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.342 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.409137+0000", 2024-09-13T15:00:58.342 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.342 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.342 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.342 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.342 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.409137+0000", 2024-09-13T15:00:58.342 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.342 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.343 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.343 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.343 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.409135+0000", 2024-09-13T15:00:58.343 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:58.343 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.343 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.343 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.343 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.409142+0000", 2024-09-13T15:00:58.343 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2890000000000003e-06 2024-09-13T15:00:58.343 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.343 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.343 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.343 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.409153+0000", 2024-09-13T15:00:58.343 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0910999999999999e-05 2024-09-13T15:00:58.344 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.344 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.344 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.344 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.409158+0000", 2024-09-13T15:00:58.344 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6740000000000002e-06 2024-09-13T15:00:58.344 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.344 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.344 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.344 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.418330+0000", 2024-09-13T15:00:58.344 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0091718070000000006 2024-09-13T15:00:58.344 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.344 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.344 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.345 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.418519+0000", 2024-09-13T15:00:58.345 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018919499999999999 2024-09-13T15:00:58.345 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.345 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.345 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:00:58.345 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.418657+0000", 2024-09-13T15:00:58.345 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000138238 2024-09-13T15:00:58.345 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.345 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.345 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.345 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.435549+0000", 2024-09-13T15:00:58.345 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.016891722000000001 2024-09-13T15:00:58.345 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.592555+0000", 2024-09-13T15:00:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15700614299999999 2024-09-13T15:00:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.595621+0000", 2024-09-13T15:00:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0030659010000000002 2024-09-13T15:00:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.595675+0000", 2024-09-13T15:00:58.347 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4011999999999997e-05 2024-09-13T15:00:58.347 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.347 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.347 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.347 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.598158+0000", 2024-09-13T15:00:58.347 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002483004 2024-09-13T15:00:58.347 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.347 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.347 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.347 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.347 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.347 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:122533 9.f 9:ff4e7d5a:::benchmark_data_smithi096_256909_object3125:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-13T15:00:58.347 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:25.410223+0000", 2024-09-13T15:00:58.347 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 92.619676741999996, 2024-09-13T15:00:58.348 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.197011773, 2024-09-13T15:00:58.348 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.348 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.348 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.348 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.348 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.348 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 122533 2024-09-13T15:00:58.348 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.348 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.348 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.348 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.348 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.410223+0000", 2024-09-13T15:00:58.348 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.348 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.349 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.349 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.349 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.410223+0000", 2024-09-13T15:00:58.349 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.349 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.349 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.349 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.349 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.410222+0000", 2024-09-13T15:00:58.349 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:00:58.349 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.349 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.349 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.350 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.410227+0000", 2024-09-13T15:00:58.350 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0900000000000004e-06 2024-09-13T15:00:58.350 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.350 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.350 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.350 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.410238+0000", 2024-09-13T15:00:58.350 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1344e-05 2024-09-13T15:00:58.350 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.350 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.350 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.350 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.410244+0000", 2024-09-13T15:00:58.350 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3940000000000004e-06 2024-09-13T15:00:58.350 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.351 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.351 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.351 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.417862+0000", 2024-09-13T15:00:58.351 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0076183420000000002 2024-09-13T15:00:58.351 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.351 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.351 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.351 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.418020+0000", 2024-09-13T15:00:58.351 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000158331 2024-09-13T15:00:58.352 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.352 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.352 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:00:58.352 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.418170+0000", 2024-09-13T15:00:58.352 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000150075 2024-09-13T15:00:58.352 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.352 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.352 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.352 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.435580+0000", 2024-09-13T15:00:58.352 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.017410106000000002 2024-09-13T15:00:58.352 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.352 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.353 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.353 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.592639+0000", 2024-09-13T15:00:58.353 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.157058213 2024-09-13T15:00:58.353 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.353 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.353 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.353 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.602292+0000", 2024-09-13T15:00:58.353 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0096537589999999996 2024-09-13T15:00:58.353 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.353 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.353 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.353 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.602355+0000", 2024-09-13T15:00:58.353 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2273999999999994e-05 2024-09-13T15:00:58.354 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.354 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.354 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.354 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.607235+0000", 2024-09-13T15:00:58.354 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0048798729999999998 2024-09-13T15:00:58.354 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.354 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.354 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.354 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.354 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.354 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:122544 9.7 9:ea427f81:::benchmark_data_smithi096_256909_object3136:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-13T15:00:58.354 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:25.429196+0000", 2024-09-13T15:00:58.355 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 92.600703479000003, 2024-09-13T15:00:58.355 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18685217000000001, 2024-09-13T15:00:58.355 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.355 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.355 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.355 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.355 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.355 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 122544 2024-09-13T15:00:58.355 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.355 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.355 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.355 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.355 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.429196+0000", 2024-09-13T15:00:58.355 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.356 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.356 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.356 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.356 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.429196+0000", 2024-09-13T15:00:58.356 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.356 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.356 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.356 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.356 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.429194+0000", 2024-09-13T15:00:58.356 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:00:58.356 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.357 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.357 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.357 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.429202+0000", 2024-09-13T15:00:58.357 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.542e-06 2024-09-13T15:00:58.357 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.357 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.357 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.357 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.429222+0000", 2024-09-13T15:00:58.357 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9981999999999999e-05 2024-09-13T15:00:58.357 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.357 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.357 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.357 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.429227+0000", 2024-09-13T15:00:58.358 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6829999999999996e-06 2024-09-13T15:00:58.358 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.358 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.358 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.358 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.598771+0000", 2024-09-13T15:00:58.358 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16954316799999999 2024-09-13T15:00:58.358 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.358 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.358 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.358 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.598967+0000", 2024-09-13T15:00:58.358 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00019599299999999999 2024-09-13T15:00:58.358 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.358 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.359 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:00:58.359 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.599098+0000", 2024-09-13T15:00:58.359 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000131583 2024-09-13T15:00:58.359 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.359 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.359 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.359 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.611257+0000", 2024-09-13T15:00:58.359 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012159021000000001 2024-09-13T15:00:58.359 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.359 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.359 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.359 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.611298+0000", 2024-09-13T15:00:58.359 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0697e-05 2024-09-13T15:00:58.360 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.360 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.360 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.360 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.611360+0000", 2024-09-13T15:00:58.360 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2127999999999999e-05 2024-09-13T15:00:58.360 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.360 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.360 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.360 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.611492+0000", 2024-09-13T15:00:58.360 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013154799999999999 2024-09-13T15:00:58.360 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.360 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.361 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.361 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.616048+0000", 2024-09-13T15:00:58.361 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0045565850000000001 2024-09-13T15:00:58.361 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.361 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.361 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.361 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.361 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.361 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.46040.0:122522 9.f e321/316 ondisk, result = 0)", 2024-09-13T15:00:58.361 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:25.429278+0000", 2024-09-13T15:00:58.361 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 92.600621145999995, 2024-09-13T15:00:58.361 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.163201708, 2024-09-13T15:00:58.361 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.361 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:00:58.362 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.362 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.362 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.362 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.429278+0000", 2024-09-13T15:00:58.362 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.362 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.362 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.362 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.362 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.429278+0000", 2024-09-13T15:00:58.362 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.362 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.362 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.362 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.363 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.429278+0000", 2024-09-13T15:00:58.363 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967296 2024-09-13T15:00:58.363 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.363 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.363 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.363 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.429281+0000", 2024-09-13T15:00:58.363 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.396e-06 2024-09-13T15:00:58.363 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.363 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.363 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.363 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.429285+0000", 2024-09-13T15:00:58.363 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4880000000000001e-06 2024-09-13T15:00:58.363 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.364 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.364 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.364 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.429288+0000", 2024-09-13T15:00:58.364 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.616e-06 2024-09-13T15:00:58.364 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.364 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.364 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.364 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.451994+0000", 2024-09-13T15:00:58.364 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.022706015 2024-09-13T15:00:58.364 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.364 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.364 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.364 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.452001+0000", 2024-09-13T15:00:58.365 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7430000000000002e-06 2024-09-13T15:00:58.365 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.365 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.365 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.365 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.592480+0000", 2024-09-13T15:00:58.365 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.14047860100000001 2024-09-13T15:00:58.365 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.365 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.365 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.365 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.365 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.365 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:122551 9.f 9:f1c9fe4e:::benchmark_data_smithi096_256909_object3143:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-13T15:00:58.365 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:25.434988+0000", 2024-09-13T15:00:58.365 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 92.594911948999993, 2024-09-13T15:00:58.366 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.185074497, 2024-09-13T15:00:58.366 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.366 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.366 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.366 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.366 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.366 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 122551 2024-09-13T15:00:58.366 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.366 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.366 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.366 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.366 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.434988+0000", 2024-09-13T15:00:58.366 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.366 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.367 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.367 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.367 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.434988+0000", 2024-09-13T15:00:58.367 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.367 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.367 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.367 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.367 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.434986+0000", 2024-09-13T15:00:58.367 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:58.367 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.367 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.367 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.367 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.434993+0000", 2024-09-13T15:00:58.368 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.3019999999999997e-06 2024-09-13T15:00:58.368 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.368 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.368 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.368 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.435010+0000", 2024-09-13T15:00:58.368 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6521999999999999e-05 2024-09-13T15:00:58.368 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.368 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.368 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.368 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.435016+0000", 2024-09-13T15:00:58.368 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.387e-06 2024-09-13T15:00:58.369 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.369 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.369 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.369 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.598201+0000", 2024-09-13T15:00:58.369 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16318431999999999 2024-09-13T15:00:58.369 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.369 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.369 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.369 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.598370+0000", 2024-09-13T15:00:58.369 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016979500000000001 2024-09-13T15:00:58.369 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.369 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.369 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:00:58.370 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.598536+0000", 2024-09-13T15:00:58.370 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016509399999999999 2024-09-13T15:00:58.370 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.370 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.370 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.370 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.611249+0000", 2024-09-13T15:00:58.370 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012713255999999999 2024-09-13T15:00:58.370 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.370 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.370 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.370 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.611333+0000", 2024-09-13T15:00:58.370 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3772999999999999e-05 2024-09-13T15:00:58.370 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.371 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.371 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.371 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.614358+0000", 2024-09-13T15:00:58.371 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0030257840000000001 2024-09-13T15:00:58.371 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.371 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.371 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.371 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.614415+0000", 2024-09-13T15:00:58.371 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6641000000000002e-05 2024-09-13T15:00:58.371 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.371 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.371 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.372 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.620062+0000", 2024-09-13T15:00:58.372 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0056470330000000001 2024-09-13T15:00:58.372 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.372 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.372 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.372 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.372 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.372 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:122558 9.7 9:ea5241ac:::benchmark_data_smithi096_256909_object3150:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-13T15:00:58.372 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:25.443237+0000", 2024-09-13T15:00:58.372 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 92.586662685999997, 2024-09-13T15:00:58.372 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18301764500000001, 2024-09-13T15:00:58.372 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.372 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.373 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.373 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.373 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.373 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 122558 2024-09-13T15:00:58.373 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.373 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.373 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.373 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.373 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.443237+0000", 2024-09-13T15:00:58.373 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.373 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.373 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.374 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.374 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.443237+0000", 2024-09-13T15:00:58.374 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.374 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.374 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.374 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.374 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.443236+0000", 2024-09-13T15:00:58.374 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:00:58.374 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.374 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.374 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.374 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.443241+0000", 2024-09-13T15:00:58.374 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4089999999999996e-06 2024-09-13T15:00:58.375 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.375 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.375 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.375 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.443249+0000", 2024-09-13T15:00:58.375 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9670000000000001e-06 2024-09-13T15:00:58.375 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.375 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.375 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.375 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.443252+0000", 2024-09-13T15:00:58.375 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.963e-06 2024-09-13T15:00:58.375 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.375 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.375 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.375 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.602531+0000", 2024-09-13T15:00:58.376 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.159278422 2024-09-13T15:00:58.376 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.376 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.376 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.376 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.602666+0000", 2024-09-13T15:00:58.376 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013526099999999999 2024-09-13T15:00:58.376 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.376 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.376 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:00:58.376 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.602831+0000", 2024-09-13T15:00:58.376 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016460699999999999 2024-09-13T15:00:58.376 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.376 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.376 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.377 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.611264+0000", 2024-09-13T15:00:58.377 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0084330929999999991 2024-09-13T15:00:58.377 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.377 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.377 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.377 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.616092+0000", 2024-09-13T15:00:58.377 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0048281959999999999 2024-09-13T15:00:58.377 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.377 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.377 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.377 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.621026+0000", 2024-09-13T15:00:58.377 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0049335009999999999 2024-09-13T15:00:58.377 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.377 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.378 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.378 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.621111+0000", 2024-09-13T15:00:58.378 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5006000000000006e-05 2024-09-13T15:00:58.378 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.378 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.378 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.378 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.626254+0000", 2024-09-13T15:00:58.378 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0051439229999999999 2024-09-13T15:00:58.378 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.378 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.378 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.378 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.378 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.378 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:122564 9.f 9:f02f853e:::benchmark_data_smithi096_256909_object3156:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-13T15:00:58.378 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:25.450031+0000", 2024-09-13T15:00:58.379 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 92.579868586000003, 2024-09-13T15:00:58.379 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.17624504699999999, 2024-09-13T15:00:58.379 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.379 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.379 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.379 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.379 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.379 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 122564 2024-09-13T15:00:58.379 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.379 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.379 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.379 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.379 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.450031+0000", 2024-09-13T15:00:58.380 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.380 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.380 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.380 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.380 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.450031+0000", 2024-09-13T15:00:58.380 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.380 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.380 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.380 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.380 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.450029+0000", 2024-09-13T15:00:58.380 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:00:58.380 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.380 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.381 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.381 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.450035+0000", 2024-09-13T15:00:58.381 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9030000000000001e-06 2024-09-13T15:00:58.381 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.381 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.381 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.381 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.450045+0000", 2024-09-13T15:00:58.381 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0020000000000001e-05 2024-09-13T15:00:58.381 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.381 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.381 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.381 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.450049+0000", 2024-09-13T15:00:58.381 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2640000000000002e-06 2024-09-13T15:00:58.382 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.382 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.382 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.382 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.607268+0000", 2024-09-13T15:00:58.382 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15721901499999999 2024-09-13T15:00:58.382 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.382 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.382 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.382 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.607411+0000", 2024-09-13T15:00:58.382 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000142965 2024-09-13T15:00:58.382 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.382 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.382 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:00:58.383 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.607551+0000", 2024-09-13T15:00:58.383 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000140087 2024-09-13T15:00:58.383 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.383 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.383 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.383 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.620155+0000", 2024-09-13T15:00:58.383 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012603840999999999 2024-09-13T15:00:58.383 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.383 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.383 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.383 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.620991+0000", 2024-09-13T15:00:58.383 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00083570900000000002 2024-09-13T15:00:58.383 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.383 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.384 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.384 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.621713+0000", 2024-09-13T15:00:58.384 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00072207 2024-09-13T15:00:58.384 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.384 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.384 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.384 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.621761+0000", 2024-09-13T15:00:58.384 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8025000000000002e-05 2024-09-13T15:00:58.384 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.384 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.384 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.384 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.626276+0000", 2024-09-13T15:00:58.384 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0045150600000000004 2024-09-13T15:00:58.385 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.385 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.385 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.385 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.385 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.385 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:122567 9.f 9:f972ef4b:::benchmark_data_smithi096_256909_object3159:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-13T15:00:58.385 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:25.450835+0000", 2024-09-13T15:00:58.385 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 92.579064243000005, 2024-09-13T15:00:58.386 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.19485133399999999, 2024-09-13T15:00:58.386 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.386 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.386 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.386 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.386 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.386 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 122567 2024-09-13T15:00:58.386 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.386 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.386 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.386 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.387 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.450835+0000", 2024-09-13T15:00:58.387 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.387 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.387 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.387 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.387 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.450835+0000", 2024-09-13T15:00:58.387 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.387 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.387 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.387 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.387 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.450835+0000", 2024-09-13T15:00:58.387 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:00:58.387 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.387 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.387 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.388 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.450838+0000", 2024-09-13T15:00:58.388 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7009999999999998e-06 2024-09-13T15:00:58.388 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.388 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.388 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.388 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.450847+0000", 2024-09-13T15:00:58.388 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.619e-06 2024-09-13T15:00:58.388 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.388 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.388 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.388 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.450852+0000", 2024-09-13T15:00:58.388 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6979999999999997e-06 2024-09-13T15:00:58.388 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.388 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.389 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.389 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.626313+0000", 2024-09-13T15:00:58.389 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.17546094800000001 2024-09-13T15:00:58.389 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.389 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.389 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.389 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.626533+0000", 2024-09-13T15:00:58.389 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00022061100000000001 2024-09-13T15:00:58.389 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.390 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.390 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:00:58.390 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.626920+0000", 2024-09-13T15:00:58.390 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000386399 2024-09-13T15:00:58.390 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.390 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.390 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.390 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.639219+0000", 2024-09-13T15:00:58.390 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012299765000000001 2024-09-13T15:00:58.390 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.390 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.390 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.391 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.639286+0000", 2024-09-13T15:00:58.391 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6901000000000002e-05 2024-09-13T15:00:58.391 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.391 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.391 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.391 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.640649+0000", 2024-09-13T15:00:58.391 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0013625219999999999 2024-09-13T15:00:58.391 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.391 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.391 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.391 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.640698+0000", 2024-09-13T15:00:58.391 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8902e-05 2024-09-13T15:00:58.392 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.392 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.392 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.392 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.645687+0000", 2024-09-13T15:00:58.392 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0049889410000000002 2024-09-13T15:00:58.392 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.392 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.392 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.392 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.392 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.393 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:122570 9.f 9:f63b108c:::benchmark_data_smithi096_256909_object3162:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-13T15:00:58.393 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:25.456452+0000", 2024-09-13T15:00:58.393 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 92.573447555000001, 2024-09-13T15:00:58.393 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.21767521200000001, 2024-09-13T15:00:58.393 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.393 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.393 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.393 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.393 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.393 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 122570 2024-09-13T15:00:58.393 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.393 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.394 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.394 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.394 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.456452+0000", 2024-09-13T15:00:58.394 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.394 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.394 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.394 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.394 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.456452+0000", 2024-09-13T15:00:58.394 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.394 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.394 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.394 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.395 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.456450+0000", 2024-09-13T15:00:58.395 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:58.395 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.395 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.395 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.395 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.456459+0000", 2024-09-13T15:00:58.395 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2109999999999998e-06 2024-09-13T15:00:58.395 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.395 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.395 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.395 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.592370+0000", 2024-09-13T15:00:58.395 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.135911592 2024-09-13T15:00:58.396 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.396 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.396 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.396 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.592381+0000", 2024-09-13T15:00:58.396 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0883000000000001e-05 2024-09-13T15:00:58.396 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.396 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.396 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.396 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.630598+0000", 2024-09-13T15:00:58.396 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.038217274000000002 2024-09-13T15:00:58.396 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.396 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.397 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.397 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.630758+0000", 2024-09-13T15:00:58.397 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015957800000000001 2024-09-13T15:00:58.397 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.397 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.397 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:00:58.397 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.630923+0000", 2024-09-13T15:00:58.397 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000164847 2024-09-13T15:00:58.397 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.397 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.397 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.397 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.639257+0000", 2024-09-13T15:00:58.397 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0083341609999999997 2024-09-13T15:00:58.398 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.398 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.398 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.398 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.645696+0000", 2024-09-13T15:00:58.398 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0064392920000000001 2024-09-13T15:00:58.398 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.398 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.398 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.398 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.669023+0000", 2024-09-13T15:00:58.398 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.023326815000000001 2024-09-13T15:00:58.398 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.398 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.398 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.398 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.669068+0000", 2024-09-13T15:00:58.399 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4888999999999997e-05 2024-09-13T15:00:58.399 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.399 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.399 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.399 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:25.674127+0000", 2024-09-13T15:00:58.399 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0050592459999999999 2024-09-13T15:00:58.399 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.399 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.399 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.399 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.399 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.399 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:140548 9.7 9:ebc7ba91:::benchmark_data_smithi096_256909_object21140:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:00:58.400 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:47.261272+0000", 2024-09-13T15:00:58.400 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 70.768627370000004, 2024-09-13T15:00:58.400 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18830650700000001, 2024-09-13T15:00:58.400 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.400 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.400 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.400 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.400 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.400 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 140548 2024-09-13T15:00:58.400 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.400 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.400 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.401 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.401 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.261272+0000", 2024-09-13T15:00:58.401 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.401 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.401 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.401 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.401 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.261272+0000", 2024-09-13T15:00:58.401 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.401 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.401 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.401 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.401 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.261272+0000", 2024-09-13T15:00:58.401 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:00:58.402 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.402 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.402 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.402 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.261275+0000", 2024-09-13T15:00:58.402 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0769999999999999e-06 2024-09-13T15:00:58.402 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.402 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.403 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.403 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.261280+0000", 2024-09-13T15:00:58.403 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3040000000000004e-06 2024-09-13T15:00:58.403 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.403 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.403 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.403 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.261283+0000", 2024-09-13T15:00:58.403 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8940000000000002e-06 2024-09-13T15:00:58.403 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.403 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.403 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.403 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.264412+0000", 2024-09-13T15:00:58.403 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0031288969999999998 2024-09-13T15:00:58.404 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.404 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.404 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.404 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.264542+0000", 2024-09-13T15:00:58.404 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000130259 2024-09-13T15:00:58.404 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.404 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.404 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:00:58.404 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.264683+0000", 2024-09-13T15:00:58.404 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014074500000000001 2024-09-13T15:00:58.404 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.404 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.404 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.405 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.306040+0000", 2024-09-13T15:00:58.405 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.041356720999999999 2024-09-13T15:00:58.405 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.405 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.405 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.405 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.306072+0000", 2024-09-13T15:00:58.405 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2283999999999998e-05 2024-09-13T15:00:58.405 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.405 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.405 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.405 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.445565+0000", 2024-09-13T15:00:58.405 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.13949298299999999 2024-09-13T15:00:58.406 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.406 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.406 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.406 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.445632+0000", 2024-09-13T15:00:58.406 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6587000000000005e-05 2024-09-13T15:00:58.406 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.406 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.406 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.406 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.449579+0000", 2024-09-13T15:00:58.406 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0039470529999999998 2024-09-13T15:00:58.406 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.407 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.407 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.407 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.407 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.407 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:140552 9.7 9:e6fbeed4:::benchmark_data_smithi096_256909_object21144:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:00:58.407 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:47.266697+0000", 2024-09-13T15:00:58.407 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 70.763202566000004, 2024-09-13T15:00:58.407 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18682443800000001, 2024-09-13T15:00:58.407 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.407 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.407 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.407 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.407 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.407 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 140552 2024-09-13T15:00:58.408 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.408 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.408 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.408 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.408 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.266697+0000", 2024-09-13T15:00:58.408 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.408 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.408 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.408 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.408 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.266697+0000", 2024-09-13T15:00:58.408 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.408 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.408 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.409 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.409 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.266695+0000", 2024-09-13T15:00:58.409 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:58.409 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.409 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.409 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.409 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.266702+0000", 2024-09-13T15:00:58.409 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0920000000000003e-06 2024-09-13T15:00:58.409 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.409 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.409 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.409 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.266712+0000", 2024-09-13T15:00:58.410 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9830000000000008e-06 2024-09-13T15:00:58.410 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.410 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.410 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.410 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.266717+0000", 2024-09-13T15:00:58.410 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2349999999999999e-06 2024-09-13T15:00:58.410 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.410 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.410 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.410 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.270823+0000", 2024-09-13T15:00:58.410 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0041062900000000003 2024-09-13T15:00:58.410 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.410 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.410 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.411 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.270918+0000", 2024-09-13T15:00:58.411 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5464000000000006e-05 2024-09-13T15:00:58.411 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.411 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.411 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:00:58.411 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.271071+0000", 2024-09-13T15:00:58.411 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015221899999999999 2024-09-13T15:00:58.411 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.411 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.411 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.411 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.306078+0000", 2024-09-13T15:00:58.411 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.035006949000000002 2024-09-13T15:00:58.411 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.411 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.412 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.412 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.306108+0000", 2024-09-13T15:00:58.412 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0283000000000001e-05 2024-09-13T15:00:58.412 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.412 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.412 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.412 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.449662+0000", 2024-09-13T15:00:58.412 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.14355425699999999 2024-09-13T15:00:58.412 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.412 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.412 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.412 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.449713+0000", 2024-09-13T15:00:58.412 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0467000000000003e-05 2024-09-13T15:00:58.412 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.412 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.413 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.413 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.453521+0000", 2024-09-13T15:00:58.413 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0038087189999999999 2024-09-13T15:00:58.413 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.413 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.413 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.413 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.413 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.413 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:140555 9.7 9:eec11426:::benchmark_data_smithi096_256909_object21147:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:00:58.413 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:47.270962+0000", 2024-09-13T15:00:58.413 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 70.758937962000005, 2024-09-13T15:00:58.413 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.18684115900000001, 2024-09-13T15:00:58.414 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.414 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.414 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.414 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.414 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.414 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 140555 2024-09-13T15:00:58.414 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.414 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.414 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.414 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.414 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.270962+0000", 2024-09-13T15:00:58.414 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.414 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.415 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.415 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.415 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.270962+0000", 2024-09-13T15:00:58.415 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.415 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.415 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.415 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.415 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.270960+0000", 2024-09-13T15:00:58.415 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:58.415 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.415 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.415 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.415 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.270967+0000", 2024-09-13T15:00:58.415 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7619999999999998e-06 2024-09-13T15:00:58.416 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.416 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.416 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.416 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.270978+0000", 2024-09-13T15:00:58.416 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1246e-05 2024-09-13T15:00:58.416 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.416 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.416 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.416 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.270984+0000", 2024-09-13T15:00:58.416 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5300000000000004e-06 2024-09-13T15:00:58.416 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.416 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.416 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.416 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.306131+0000", 2024-09-13T15:00:58.417 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.035147240000000003 2024-09-13T15:00:58.417 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.417 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.417 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.417 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.306282+0000", 2024-09-13T15:00:58.417 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015111700000000001 2024-09-13T15:00:58.417 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.417 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.417 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:00:58.417 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.306434+0000", 2024-09-13T15:00:58.417 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015182800000000001 2024-09-13T15:00:58.417 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.417 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.417 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.418 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.309395+0000", 2024-09-13T15:00:58.418 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029607069999999999 2024-09-13T15:00:58.418 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.418 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.418 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.418 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.319250+0000", 2024-09-13T15:00:58.418 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0098552429999999996 2024-09-13T15:00:58.418 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.418 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.418 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.418 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.453676+0000", 2024-09-13T15:00:58.418 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.13442626499999999 2024-09-13T15:00:58.418 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.418 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.418 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.419 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.453808+0000", 2024-09-13T15:00:58.419 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013212599999999999 2024-09-13T15:00:58.419 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.419 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.419 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.419 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:47.457803+0000", 2024-09-13T15:00:58.419 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0039945249999999996 2024-09-13T15:00:58.419 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.419 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.419 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.419 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.419 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.419 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:182526 9.7 9:ec68ee8b:::benchmark_data_smithi096_256909_object63118:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-09-13T15:00:58.419 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:38.156414+0000", 2024-09-13T15:00:58.420 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 19.873485109000001, 2024-09-13T15:00:58.420 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.172650995, 2024-09-13T15:00:58.420 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.420 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.420 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.420 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.420 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.420 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 182526 2024-09-13T15:00:58.420 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.420 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.420 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.421 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.421 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.156414+0000", 2024-09-13T15:00:58.421 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.421 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.421 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.421 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.421 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.156414+0000", 2024-09-13T15:00:58.421 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.421 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.424 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.424 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.425 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.156413+0000", 2024-09-13T15:00:58.425 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:58.425 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.425 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.425 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.425 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.156418+0000", 2024-09-13T15:00:58.425 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0529999999999997e-06 2024-09-13T15:00:58.425 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.425 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.425 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.425 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.156429+0000", 2024-09-13T15:00:58.425 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0794e-05 2024-09-13T15:00:58.425 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.425 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.426 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.426 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.156434+0000", 2024-09-13T15:00:58.426 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4600000000000002e-06 2024-09-13T15:00:58.426 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.426 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.426 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.426 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.174344+0000", 2024-09-13T15:00:58.426 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.017909581000000001 2024-09-13T15:00:58.426 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.426 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.426 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.427 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.174490+0000", 2024-09-13T15:00:58.427 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014556500000000001 2024-09-13T15:00:58.427 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.427 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.427 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:00:58.427 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.174633+0000", 2024-09-13T15:00:58.427 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000143769 2024-09-13T15:00:58.427 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.427 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.427 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.427 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.179409+0000", 2024-09-13T15:00:58.427 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0047756029999999998 2024-09-13T15:00:58.427 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.428 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.428 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.428 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.179416+0000", 2024-09-13T15:00:58.428 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0639999999999996e-06 2024-09-13T15:00:58.428 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.428 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.428 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.428 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.218097+0000", 2024-09-13T15:00:58.428 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.038680702999999997 2024-09-13T15:00:58.428 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.428 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.428 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.429 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.218175+0000", 2024-09-13T15:00:58.429 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7888999999999999e-05 2024-09-13T15:00:58.429 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.429 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.429 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.429 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.329065+0000", 2024-09-13T15:00:58.429 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.110890798 2024-09-13T15:00:58.429 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.429 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.429 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.429 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.429 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.429 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:182576 9.7 9:e7f8727e:::benchmark_data_smithi096_256909_object63168:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-09-13T15:00:58.430 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:38.220124+0000", 2024-09-13T15:00:58.430 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 19.809775922, 2024-09-13T15:00:58.430 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.16695649800000001, 2024-09-13T15:00:58.430 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:00:58.430 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:00:58.430 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:00:58.430 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:00:58.430 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:00:58.430 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 182576 2024-09-13T15:00:58.430 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.430 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:00:58.431 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.431 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:00:58.431 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.220124+0000", 2024-09-13T15:00:58.431 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.431 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.431 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.431 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:00:58.431 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.220124+0000", 2024-09-13T15:00:58.431 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:00:58.431 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.431 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.431 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:00:58.432 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.220122+0000", 2024-09-13T15:00:58.432 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:58.432 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.432 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.432 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:00:58.432 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.220131+0000", 2024-09-13T15:00:58.432 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1649999999999995e-06 2024-09-13T15:00:58.432 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.432 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.432 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:00:58.432 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.323747+0000", 2024-09-13T15:00:58.433 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.103615602 2024-09-13T15:00:58.433 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.433 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.433 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:00:58.433 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.323759+0000", 2024-09-13T15:00:58.433 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2454e-05 2024-09-13T15:00:58.433 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.433 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.433 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:00:58.433 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.355716+0000", 2024-09-13T15:00:58.433 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.031957122999999997 2024-09-13T15:00:58.433 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.434 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.434 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:00:58.434 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.355872+0000", 2024-09-13T15:00:58.434 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000155508 2024-09-13T15:00:58.434 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.434 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.434 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:00:58.434 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.356039+0000", 2024-09-13T15:00:58.434 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016663499999999999 2024-09-13T15:00:58.434 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.434 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.434 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:00:58.434 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.369662+0000", 2024-09-13T15:00:58.435 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013623788 2024-09-13T15:00:58.435 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.435 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.435 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.435 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.369795+0000", 2024-09-13T15:00:58.435 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013297800000000001 2024-09-13T15:00:58.435 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.435 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.435 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:58.435 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.383239+0000", 2024-09-13T15:00:58.435 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013444049 2024-09-13T15:00:58.435 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.436 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.436 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:00:58.436 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.383329+0000", 2024-09-13T15:00:58.436 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.9190999999999994e-05 2024-09-13T15:00:58.436 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:00:58.436 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:00:58.436 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:00:58.436 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:38.387080+0000", 2024-09-13T15:00:58.436 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0037515780000000002 2024-09-13T15:00:58.436 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.436 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.436 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.436 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:00:58.436 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:00:58.437 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:00:58.437 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:58.469 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:58.571 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:58.601 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:58.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:58.703 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:58.746 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:58.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:58.848 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:58.886 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:58.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:58.959 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:58 smithi096 podman[275824]: 2024-09-13 15:00:58.638095158 +0000 UTC m=+0.925688668 container remove 8a5475a73fca24d8a1b4a541055ccb0a0ab4116e010d8d3d95c84902943b4621 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-activate, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:00:58.988 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:59.041 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:59.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:59.074 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:00:59.142 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:59.181 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:59.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:59.247 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:00:59.247 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:00:59.283 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:59.285 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:00:59.285 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:00:59.314 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:59.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:59.316 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:00:59.316 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:00:59.317 INFO:teuthology.orchestra.run.smithi096.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:00:59.347 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:00:59.348 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:00:59.381 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:00:59.381 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:00:59.410 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:00:59.411 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:00:59.417 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:59.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.445 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:59.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:59.465 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:00:59.466 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:00:59.466 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:59 smithi096 podman[275952]: 2024-09-13 15:00:59.266878776 +0000 UTC m=+0.270284401 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:00:59.466 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:59 smithi096 podman[275952]: 2024-09-13 15:00:59.385205633 +0000 UTC m=+0.388611244 container create 3dd8d5d5bb70f025ff9bff825c610f89affe973e5a1451657f1fa9b30194bf22 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-13T15:00:59.547 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:59.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.615 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:00:59.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:59.716 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:59.751 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:59.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:59.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:59 smithi096 ceph-mon[98901]: pgmap v1104: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 794 op/s; 31041/126681 objects degraded (24.503%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T15:00:59.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:59 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3791094349' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:59.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:00:59 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 31041/126681 objects degraded (24.503%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:59.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:59 smithi096 ceph-mon[102928]: pgmap v1104: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 794 op/s; 31041/126681 objects degraded (24.503%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T15:00:59.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:59 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3791094349' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:59.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:00:59 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 31041/126681 objects degraded (24.503%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:59.853 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:59.893 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:00:59.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:59.958 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T15:00:59.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:59 smithi204 ceph-mon[87071]: pgmap v1104: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 794 op/s; 31041/126681 objects degraded (24.503%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T15:00:59.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:59 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3791094349' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:59.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:00:59 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 31041/126681 objects degraded (24.503%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:59.994 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:59.995 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:00.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.031 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:00.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:00.127 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:59 smithi096 podman[275952]: 2024-09-13 15:00:59.836748635 +0000 UTC m=+0.840154249 container init 3dd8d5d5bb70f025ff9bff825c610f89affe973e5a1451657f1fa9b30194bf22 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-13T15:01:00.128 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:59 smithi096 podman[275952]: 2024-09-13 15:00:59.842319397 +0000 UTC m=+0.845725012 container start 3dd8d5d5bb70f025ff9bff825c610f89affe973e5a1451657f1fa9b30194bf22 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-13T15:01:00.128 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:59 smithi096 ceph-osd[276124]: -- 172.21.15.96:0/3691166498 <== mon.2 v2:172.21.15.96:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557f902e5a40 con 0x557f910b4000 2024-09-13T15:01:00.128 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:59 smithi096 bash[275952]: 3dd8d5d5bb70f025ff9bff825c610f89affe973e5a1451657f1fa9b30194bf22 2024-09-13T15:01:00.128 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:00:59 smithi096 systemd[1]: Started Ceph osd.3 for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T15:01:00.132 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:00.163 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:00.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:00.265 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:00.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.301 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:00.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:00.402 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:00.435 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:00.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:01:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:01:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:01:00.537 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:00.570 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:00.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:00.672 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:00.709 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:00.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:00.811 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:00.844 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:00.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:00.855 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:01:00.855 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:01:00.948 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:00.982 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:00.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:01.084 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:01.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.106 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:01.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:01.207 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:01.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.229 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:01.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:01.330 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:01.352 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:01.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:01.372 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:01.454 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:01.475 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:01.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:01.577 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:01.612 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:01.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:01.714 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:01.757 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:01.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:01.814 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:01 smithi096 ceph-mon[98901]: pgmap v1105: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 30516/121008 objects degraded (25.218%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T15:01:01.814 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:01 smithi096 ceph-mon[102928]: pgmap v1105: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 30516/121008 objects degraded (25.218%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T15:01:01.815 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:01:01 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[276103]: 2024-09-13T15:01:01.544+0000 7fbe24f7e640 -1 Falling back to public interface 2024-09-13T15:01:01.858 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.898 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:01.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:01.917 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:01 smithi204 ceph-mon[87071]: pgmap v1105: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 30516/121008 objects degraded (25.218%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T15:01:02.000 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:02.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.040 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:02.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:02.141 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:02.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.176 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:02.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:02.277 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:02.304 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:02.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:02.406 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:02.440 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:02.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:02.542 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:02.575 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:02.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:02.677 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:02.715 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:02.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:02.817 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:02.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.852 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:02.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:02.856 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:01:02.951 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T15:01:02.953 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:03.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.043 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:03.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:03.145 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:03.206 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:03.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:03.308 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:03.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.362 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:03.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:03.411 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:03.463 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:03.495 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:03.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:03.596 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:03.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.634 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:03.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:03.735 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:03.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.770 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:03.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:03.814 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:03 smithi096 ceph-mon[98901]: pgmap v1106: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 30516/121008 objects degraded (25.218%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T15:01:03.814 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:03 smithi096 ceph-mon[102928]: pgmap v1106: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 30516/121008 objects degraded (25.218%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T15:01:03.864 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:03 smithi204 ceph-mon[87071]: pgmap v1106: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 30516/121008 objects degraded (25.218%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T15:01:03.871 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:03.918 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:03.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:04.019 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:04.056 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:04.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:04.117 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:04.159 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:04.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.196 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:04.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:04.297 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:04.331 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:04.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:04.433 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:04.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.474 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:04.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:04.575 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:04.602 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:04.683 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T15:01:04.705 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:04.732 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:04.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:04.834 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:04.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.869 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:04.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:04.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:04 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1127845169' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:04.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:04 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:04.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:04 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 30516/121008 objects degraded (25.218%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:04.969 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:04.991 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:04.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:04 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1127845169' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:04 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:05.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:04 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 30516/121008 objects degraded (25.218%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:05.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:04 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1127845169' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:05.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:04 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:05.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:04 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 30516/121008 objects degraded (25.218%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:05.094 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:05.124 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:05.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:05.226 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:05.258 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:05.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:05.360 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:05.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.384 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:05.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:05.485 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:05.513 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:05.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:05.615 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:05.648 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:05.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:05.717 INFO:teuthology.orchestra.run.smithi096.stderr:no valid command found; 10 closest matches: 2024-09-13T15:01:05.717 INFO:teuthology.orchestra.run.smithi096.stderr:0 2024-09-13T15:01:05.717 INFO:teuthology.orchestra.run.smithi096.stderr:1 2024-09-13T15:01:05.717 INFO:teuthology.orchestra.run.smithi096.stderr:2 2024-09-13T15:01:05.717 INFO:teuthology.orchestra.run.smithi096.stderr:abort 2024-09-13T15:01:05.717 INFO:teuthology.orchestra.run.smithi096.stderr:assert 2024-09-13T15:01:05.717 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs debug_inject_read_zeros 2024-09-13T15:01:05.717 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs files list 2024-09-13T15:01:05.717 INFO:teuthology.orchestra.run.smithi096.stderr:bluefs stats 2024-09-13T15:01:05.717 INFO:teuthology.orchestra.run.smithi096.stderr:bluestore bluefs device info [] 2024-09-13T15:01:05.718 INFO:teuthology.orchestra.run.smithi096.stderr:config diff 2024-09-13T15:01:05.718 INFO:teuthology.orchestra.run.smithi096.stderr:admin_socket: invalid command 2024-09-13T15:01:05.750 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:05.783 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:05.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:05.886 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:05.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.924 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:05.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:05 smithi204 ceph-mon[87071]: pgmap v1107: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 821 op/s; 29748/112398 objects degraded (26.467%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T15:01:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:05 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:01:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:05 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:05 smithi204 ceph-mon[87071]: osdmap e333: 8 total, 7 up, 7 in 2024-09-13T15:01:06.026 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:06.062 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:06.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:06.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:05 smithi096 ceph-mon[98901]: pgmap v1107: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 821 op/s; 29748/112398 objects degraded (26.467%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T15:01:06.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:05 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:01:06.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:05 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:06.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:05 smithi096 ceph-mon[98901]: osdmap e333: 8 total, 7 up, 7 in 2024-09-13T15:01:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:05 smithi096 ceph-mon[102928]: pgmap v1107: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 821 op/s; 29748/112398 objects degraded (26.467%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T15:01:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:05 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:01:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:05 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:05 smithi096 ceph-mon[102928]: osdmap e333: 8 total, 7 up, 7 in 2024-09-13T15:01:06.164 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:06.219 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:06.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:06.321 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:06.356 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:06.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:06.458 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:06.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.494 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:06.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:06.595 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:06.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.634 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:06.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:06.735 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:06.770 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:06.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:06.798 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:01:06.799 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:01:06.872 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:06.908 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:06.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:06.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:01:06.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:07.010 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:07.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.051 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:07.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:07.152 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:07.194 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:07.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:07.196 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:01:07.196 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:07.196 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:01:07.196 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:07.296 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:07.328 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:07.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:07.430 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:07.475 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:07.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:07.576 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:07.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T15:01:07.616 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:07.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:07.717 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:07.766 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T15:01:07.766 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-3, ['dump_ops_in_flight'] 2024-09-13T15:01:07.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.767 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:07.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:07.868 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:07.937 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:07.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:08.040 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:08.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:07 smithi096 ceph-mon[98901]: pgmap v1109: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 29476/109485 objects degraded (26.922%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T15:01:08.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:01:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:01:07] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.51.0" 2024-09-13T15:01:08.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:07 smithi096 ceph-mon[102928]: pgmap v1109: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 29476/109485 objects degraded (26.922%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T15:01:08.078 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:08.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.096 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:08.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:08.197 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:08.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:07 smithi204 ceph-mon[87071]: pgmap v1109: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 29476/109485 objects degraded (26.922%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T15:01:08.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.232 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:08.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:08.333 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:08.364 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:08.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:08.466 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:08.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.507 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:08.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:08.608 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:08.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.643 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:08.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:08.744 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:08.755 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:01:08.755 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:01:08.776 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:08.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:08.787 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:01:08.787 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:01:08.817 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:01:08.817 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:01:08.817 INFO:teuthology.orchestra.run.smithi096.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:01:08.848 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:01:08.848 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:01:08.878 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:08.898 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:01:08.898 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:01:08.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.913 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:08.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:08.934 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:01:08.934 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:01:08.967 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:01:08.968 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:01:08.968 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:08 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4007678632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:08.968 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:08 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4007678632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:09.014 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:09.048 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:09.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:09.150 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:09.189 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:09.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:09.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:08 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4007678632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:09.290 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:09.324 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:09.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:09.426 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:09.459 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:09.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:09.561 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:09.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.600 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:09.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:09.701 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:09.738 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:09.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:09.840 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:09.859 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:01:09.860 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:01:09.879 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:09.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:09.980 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:10.015 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:10.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:10.117 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:10.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.153 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:10.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:10.157 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:09 smithi204 ceph-mon[87071]: pgmap v1110: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 803 op/s; 29010/102573 objects degraded (28.282%); 2.9 MiB/s, 47 objects/s recovering 2024-09-13T15:01:10.157 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:09 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 29010/102573 objects degraded (28.282%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:10.203 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:01:10.209 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:01:10.209 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T15:01:10.209 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T15:01:10.209 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T15:01:10.209 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:01:10.236 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:09 smithi096 ceph-mon[102928]: pgmap v1110: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 803 op/s; 29010/102573 objects degraded (28.282%); 2.9 MiB/s, 47 objects/s recovering 2024-09-13T15:01:10.237 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:09 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 29010/102573 objects degraded (28.282%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:10.237 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:09 smithi096 ceph-mon[98901]: pgmap v1110: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 803 op/s; 29010/102573 objects degraded (28.282%); 2.9 MiB/s, 47 objects/s recovering 2024-09-13T15:01:10.237 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:09 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 29010/102573 objects degraded (28.282%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:10.255 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:10.312 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:10.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:10.414 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:10.452 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:01:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:01:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:01:10.554 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:10.610 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:10.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:10.630 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:10.712 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:10.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.745 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:10.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:10.846 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:10.875 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:10.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:10.976 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:10.978 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T15:01:11.006 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:11.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:11.109 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:11.140 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:11.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:11.242 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:11.301 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:11.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:11.322 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:01:11 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[276103]: 2024-09-13T15:01:11.208+0000 7fbe24f7e640 -1 osd.3 317 log_to_monitors true 2024-09-13T15:01:11.403 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:11.416 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:11.459 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:11.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:11.561 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:11.598 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:11.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:11.700 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:11.749 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:11.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:11.851 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:11.861 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:01:11.879 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:11.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:11.981 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:12.026 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:12.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:12.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:11 smithi096 ceph-mon[98901]: pgmap v1111: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 786 op/s; 28502/97401 objects degraded (29.263%); 2.9 MiB/s, 47 objects/s recovering 2024-09-13T15:01:12.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:11 smithi096 ceph-mon[98901]: from='osd.3 [v2:172.21.15.96:6826/3715117729,v1:172.21.15.96:6827/3715117729]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:01:12.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:11 smithi096 ceph-mon[98901]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:01:12.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:11 smithi096 ceph-mon[102928]: pgmap v1111: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 786 op/s; 28502/97401 objects degraded (29.263%); 2.9 MiB/s, 47 objects/s recovering 2024-09-13T15:01:12.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:11 smithi096 ceph-mon[102928]: from='osd.3 [v2:172.21.15.96:6826/3715117729,v1:172.21.15.96:6827/3715117729]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:01:12.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:11 smithi096 ceph-mon[102928]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:01:12.074 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:01:12 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[276103]: 2024-09-13T15:01:12.005+0000 7fbe18d20640 -1 osd.3 334 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T15:01:12.127 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:12.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.161 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:12.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:12.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:11 smithi204 ceph-mon[87071]: pgmap v1111: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 786 op/s; 28502/97401 objects degraded (29.263%); 2.9 MiB/s, 47 objects/s recovering 2024-09-13T15:01:12.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:11 smithi204 ceph-mon[87071]: from='osd.3 [v2:172.21.15.96:6826/3715117729,v1:172.21.15.96:6827/3715117729]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:01:12.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:11 smithi204 ceph-mon[87071]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:01:12.262 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:12.289 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:12.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:12.391 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:12.420 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:12.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:12.522 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:12.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.563 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:12.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:12.664 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:12.690 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:12.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:12.768 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T15:01:12.792 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:12.835 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:12.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:12.937 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:12.976 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:12.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:13.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:12 smithi096 ceph-mon[98901]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T15:01:13.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:12 smithi096 ceph-mon[98901]: osdmap e334: 8 total, 7 up, 7 in 2024-09-13T15:01:13.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:12 smithi096 ceph-mon[98901]: from='osd.3 [v2:172.21.15.96:6826/3715117729,v1:172.21.15.96:6827/3715117729]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T15:01:13.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:12 smithi096 ceph-mon[98901]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T15:01:13.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:12 smithi096 ceph-mon[102928]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T15:01:13.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:12 smithi096 ceph-mon[102928]: osdmap e334: 8 total, 7 up, 7 in 2024-09-13T15:01:13.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:12 smithi096 ceph-mon[102928]: from='osd.3 [v2:172.21.15.96:6826/3715117729,v1:172.21.15.96:6827/3715117729]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T15:01:13.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:12 smithi096 ceph-mon[102928]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T15:01:13.078 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:13.104 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:13.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:13.206 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:13.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:12 smithi204 ceph-mon[87071]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T15:01:13.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:12 smithi204 ceph-mon[87071]: osdmap e334: 8 total, 7 up, 7 in 2024-09-13T15:01:13.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:12 smithi204 ceph-mon[87071]: from='osd.3 [v2:172.21.15.96:6826/3715117729,v1:172.21.15.96:6827/3715117729]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T15:01:13.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:12 smithi204 ceph-mon[87071]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-13T15:01:13.243 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:13.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:13.345 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:13.371 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:13.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:13.473 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:13.521 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:13.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:13.624 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:13.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.659 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:13.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:13.760 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:13.793 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:13.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:13.805 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:01:13.805 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T15:01:13.805 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T15:01:13.805 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T15:01:13.805 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.805 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193672 7.c 7:3c3448d3:::benchmark_data_smithi096_196313_object73541:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-09-13T15:01:13.806 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.177979+0000", 2024-09-13T15:01:13.806 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 568.62310531100002, 2024-09-13T15:01:13.806 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.032648639, 2024-09-13T15:01:13.806 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.806 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T15:01:13.806 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.806 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:01:13.806 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:01:13.806 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193672 2024-09-13T15:01:13.806 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.806 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.807 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.807 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.807 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.177979+0000", 2024-09-13T15:01:13.807 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.807 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.807 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.807 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.807 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.177979+0000", 2024-09-13T15:01:13.807 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.807 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.808 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.808 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.808 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.177977+0000", 2024-09-13T15:01:13.808 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:01:13.808 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.808 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.808 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.808 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.177984+0000", 2024-09-13T15:01:13.808 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9689999999999997e-06 2024-09-13T15:01:13.808 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.808 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.808 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.809 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.177997+0000", 2024-09-13T15:01:13.809 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3138e-05 2024-09-13T15:01:13.809 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.809 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.809 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.809 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.178003+0000", 2024-09-13T15:01:13.809 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.84e-06 2024-09-13T15:01:13.809 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.810 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.810 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.810 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.178051+0000", 2024-09-13T15:01:13.810 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8118000000000002e-05 2024-09-13T15:01:13.810 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.810 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.810 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.810 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.178189+0000", 2024-09-13T15:01:13.810 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000137632 2024-09-13T15:01:13.810 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.810 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.810 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T15:01:13.810 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.178324+0000", 2024-09-13T15:01:13.811 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013502499999999999 2024-09-13T15:01:13.811 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.811 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.811 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.811 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.186082+0000", 2024-09-13T15:01:13.811 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0077580089999999997 2024-09-13T15:01:13.811 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.811 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.811 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.811 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.186093+0000", 2024-09-13T15:01:13.811 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1022000000000001e-05 2024-09-13T15:01:13.811 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.812 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.812 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.812 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.189796+0000", 2024-09-13T15:01:13.812 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0037034949999999998 2024-09-13T15:01:13.812 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.812 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.812 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:13.812 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.189805+0000", 2024-09-13T15:01:13.812 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.037e-06 2024-09-13T15:01:13.812 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.812 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.812 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.812 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.210554+0000", 2024-09-13T15:01:13.813 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0207485650000001 2024-09-13T15:01:13.813 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.813 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.813 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.813 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.210627+0000", 2024-09-13T15:01:13.813 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.3478000000000004e-05 2024-09-13T15:01:13.813 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.813 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.813 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.813 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.813 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.813 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193675 7.c 7:3f91f1c4:::benchmark_data_smithi096_196313_object73544:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-09-13T15:01:13.813 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.181545+0000", 2024-09-13T15:01:13.814 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 568.61953859300002, 2024-09-13T15:01:13.814 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.029564718, 2024-09-13T15:01:13.814 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.814 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T15:01:13.814 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.814 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:01:13.814 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:01:13.814 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193675 2024-09-13T15:01:13.814 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.814 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.814 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.814 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.814 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.181545+0000", 2024-09-13T15:01:13.815 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.815 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.815 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.815 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.815 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.181545+0000", 2024-09-13T15:01:13.815 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.815 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.815 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.815 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.815 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.181544+0000", 2024-09-13T15:01:13.815 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:13.815 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.815 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.816 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.816 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.181550+0000", 2024-09-13T15:01:13.816 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4370000000000004e-06 2024-09-13T15:01:13.816 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.816 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.816 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.816 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.181564+0000", 2024-09-13T15:01:13.816 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3356999999999999e-05 2024-09-13T15:01:13.816 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.817 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.817 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.817 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.181569+0000", 2024-09-13T15:01:13.817 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.446e-06 2024-09-13T15:01:13.817 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.817 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.817 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.817 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.182087+0000", 2024-09-13T15:01:13.817 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00051824300000000005 2024-09-13T15:01:13.817 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.817 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.817 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.817 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.182198+0000", 2024-09-13T15:01:13.818 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000111106 2024-09-13T15:01:13.818 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.818 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.818 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,3,7", 2024-09-13T15:01:13.818 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.182297+0000", 2024-09-13T15:01:13.818 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9020999999999999e-05 2024-09-13T15:01:13.818 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.818 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.818 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.818 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.187301+0000", 2024-09-13T15:01:13.818 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0050036910000000002 2024-09-13T15:01:13.818 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.818 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.819 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.819 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.189901+0000", 2024-09-13T15:01:13.819 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0025996700000000001 2024-09-13T15:01:13.819 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.819 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.819 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:13.819 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.189908+0000", 2024-09-13T15:01:13.819 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5830000000000001e-06 2024-09-13T15:01:13.819 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.819 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.819 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.819 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.211040+0000", 2024-09-13T15:01:13.819 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0211311439999999 2024-09-13T15:01:13.820 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.820 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.820 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.820 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.211110+0000", 2024-09-13T15:01:13.820 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0531999999999995e-05 2024-09-13T15:01:13.820 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.820 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.820 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.820 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.820 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.820 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193694 7.c 7:375370fa:::benchmark_data_smithi096_196313_object73563:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2024-09-13T15:01:13.820 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.212908+0000", 2024-09-13T15:01:13.820 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 568.58817562199999, 2024-09-13T15:01:13.821 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0150302760000001, 2024-09-13T15:01:13.821 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.821 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.821 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.821 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:01:13.822 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:01:13.822 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193694 2024-09-13T15:01:13.822 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.822 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.822 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.822 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.822 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.212908+0000", 2024-09-13T15:01:13.822 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.822 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.822 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.823 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.823 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.212908+0000", 2024-09-13T15:01:13.823 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.823 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.823 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.823 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.823 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.212907+0000", 2024-09-13T15:01:13.823 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:13.823 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.823 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.823 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.823 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.212914+0000", 2024-09-13T15:01:13.823 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7660000000000001e-06 2024-09-13T15:01:13.824 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.824 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.824 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.824 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.212945+0000", 2024-09-13T15:01:13.824 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1584000000000001e-05 2024-09-13T15:01:13.824 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.824 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.824 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.824 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.212952+0000", 2024-09-13T15:01:13.824 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6139999999999997e-06 2024-09-13T15:01:13.824 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.824 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.824 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.825 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.213009+0000", 2024-09-13T15:01:13.825 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7293999999999997e-05 2024-09-13T15:01:13.825 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.825 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.825 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:13.825 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.213018+0000", 2024-09-13T15:01:13.825 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5369999999999997e-06 2024-09-13T15:01:13.825 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.825 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.825 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.825 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.211146+0000", 2024-09-13T15:01:13.825 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99812826099999996 2024-09-13T15:01:13.825 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.826 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.826 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.826 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.211234+0000", 2024-09-13T15:01:13.826 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7675000000000003e-05 2024-09-13T15:01:13.826 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.826 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.826 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T15:01:13.826 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.211304+0000", 2024-09-13T15:01:13.826 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0656999999999998e-05 2024-09-13T15:01:13.826 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.826 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.826 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.827 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.223283+0000", 2024-09-13T15:01:13.827 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011978277000000001 2024-09-13T15:01:13.827 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.827 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.827 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.827 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.223334+0000", 2024-09-13T15:01:13.827 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0895e-05 2024-09-13T15:01:13.827 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.827 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.827 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.827 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.227842+0000", 2024-09-13T15:01:13.827 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0045089420000000002 2024-09-13T15:01:13.827 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.827 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.828 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.828 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.227891+0000", 2024-09-13T15:01:13.828 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8998000000000001e-05 2024-09-13T15:01:13.828 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.828 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.828 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.828 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.227939+0000", 2024-09-13T15:01:13.828 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7102999999999999e-05 2024-09-13T15:01:13.828 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.828 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.828 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.828 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.828 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.828 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193715 7.c 7:311e1db2:::benchmark_data_smithi096_196313_object73584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2024-09-13T15:01:13.829 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.238753+0000", 2024-09-13T15:01:13.829 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 568.56233057999998, 2024-09-13T15:01:13.829 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99428842900000003, 2024-09-13T15:01:13.829 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.829 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.829 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.829 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:01:13.829 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:01:13.829 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193715 2024-09-13T15:01:13.829 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.829 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.829 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.829 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.830 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238753+0000", 2024-09-13T15:01:13.830 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.830 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.830 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.830 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.830 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238753+0000", 2024-09-13T15:01:13.830 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.830 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.830 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.830 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.830 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238752+0000", 2024-09-13T15:01:13.830 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:13.830 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.830 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.831 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.831 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238760+0000", 2024-09-13T15:01:13.831 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0830000000000004e-06 2024-09-13T15:01:13.831 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.831 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.831 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.831 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238777+0000", 2024-09-13T15:01:13.831 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6670000000000001e-05 2024-09-13T15:01:13.831 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.831 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.831 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.831 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238785+0000", 2024-09-13T15:01:13.831 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7689999999999996e-06 2024-09-13T15:01:13.831 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.832 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.832 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.832 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238828+0000", 2024-09-13T15:01:13.832 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3096999999999998e-05 2024-09-13T15:01:13.832 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.832 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.832 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:13.832 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.238836+0000", 2024-09-13T15:01:13.832 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9459999999999998e-06 2024-09-13T15:01:13.832 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.832 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.832 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.832 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.215529+0000", 2024-09-13T15:01:13.833 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97669314500000004 2024-09-13T15:01:13.833 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.833 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.833 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.833 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.215644+0000", 2024-09-13T15:01:13.833 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000115549 2024-09-13T15:01:13.833 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.833 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.833 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T15:01:13.833 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.215770+0000", 2024-09-13T15:01:13.833 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012607300000000001 2024-09-13T15:01:13.833 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.833 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.833 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.834 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.227949+0000", 2024-09-13T15:01:13.834 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012179024 2024-09-13T15:01:13.834 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.834 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.834 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.834 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.232885+0000", 2024-09-13T15:01:13.834 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0049357089999999999 2024-09-13T15:01:13.834 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.834 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.834 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.834 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.232945+0000", 2024-09-13T15:01:13.834 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0398e-05 2024-09-13T15:01:13.835 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.835 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.835 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.835 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.232996+0000", 2024-09-13T15:01:13.835 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0985000000000001e-05 2024-09-13T15:01:13.835 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.835 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.835 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.835 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233042+0000", 2024-09-13T15:01:13.835 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5321000000000002e-05 2024-09-13T15:01:13.835 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.835 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.835 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.835 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.836 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.836 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193716 7.c 7:3f878838:::benchmark_data_smithi096_196313_object73585:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2024-09-13T15:01:13.836 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.239711+0000", 2024-09-13T15:01:13.836 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 568.56137284900001, 2024-09-13T15:01:13.836 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99352322699999995, 2024-09-13T15:01:13.836 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.836 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.836 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.836 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:01:13.836 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:01:13.836 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193716 2024-09-13T15:01:13.836 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.836 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.837 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.837 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.837 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239711+0000", 2024-09-13T15:01:13.837 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.837 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.837 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.837 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.837 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239711+0000", 2024-09-13T15:01:13.837 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.837 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.837 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.837 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.837 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239710+0000", 2024-09-13T15:01:13.837 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:13.838 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.838 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.838 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.838 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239715+0000", 2024-09-13T15:01:13.838 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4009999999999999e-06 2024-09-13T15:01:13.838 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.838 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.838 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.838 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239723+0000", 2024-09-13T15:01:13.838 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5990000000000002e-06 2024-09-13T15:01:13.838 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.838 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.838 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.838 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239729+0000", 2024-09-13T15:01:13.839 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1780000000000002e-06 2024-09-13T15:01:13.839 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.839 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.839 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.839 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239755+0000", 2024-09-13T15:01:13.839 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.6869e-05 2024-09-13T15:01:13.839 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.839 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.839 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:13.839 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.239761+0000", 2024-09-13T15:01:13.839 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3530000000000002e-06 2024-09-13T15:01:13.839 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.840 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.840 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.840 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.219460+0000", 2024-09-13T15:01:13.840 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97969935600000002 2024-09-13T15:01:13.840 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.840 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.840 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.840 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.219555+0000", 2024-09-13T15:01:13.840 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.4474000000000004e-05 2024-09-13T15:01:13.840 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.840 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.840 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T15:01:13.840 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.219662+0000", 2024-09-13T15:01:13.840 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000107624 2024-09-13T15:01:13.841 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.841 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.841 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.841 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233063+0000", 2024-09-13T15:01:13.841 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013401135999999999 2024-09-13T15:01:13.841 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.841 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.841 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.841 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233107+0000", 2024-09-13T15:01:13.841 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3887999999999997e-05 2024-09-13T15:01:13.841 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.841 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.841 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.842 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233151+0000", 2024-09-13T15:01:13.842 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3993999999999999e-05 2024-09-13T15:01:13.842 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.842 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.842 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.842 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233197+0000", 2024-09-13T15:01:13.842 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5535999999999999e-05 2024-09-13T15:01:13.842 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.842 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.842 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.842 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233234+0000", 2024-09-13T15:01:13.842 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7459999999999997e-05 2024-09-13T15:01:13.843 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.843 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.843 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.843 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.843 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.843 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193744 7.c 7:3cb38b24:::benchmark_data_smithi096_196313_object73613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2024-09-13T15:01:13.843 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.277160+0000", 2024-09-13T15:01:13.843 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 568.52392438899994, 2024-09-13T15:01:13.843 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95624868799999996, 2024-09-13T15:01:13.843 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.843 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.843 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.843 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:01:13.844 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:01:13.844 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193744 2024-09-13T15:01:13.844 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.844 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.844 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.844 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.844 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277160+0000", 2024-09-13T15:01:13.844 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.844 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.844 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.844 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.844 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277160+0000", 2024-09-13T15:01:13.844 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.845 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.845 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.845 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.845 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277158+0000", 2024-09-13T15:01:13.845 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:01:13.845 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.845 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.845 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.845 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277164+0000", 2024-09-13T15:01:13.846 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5260000000000001e-06 2024-09-13T15:01:13.846 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.846 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.846 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.846 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277176+0000", 2024-09-13T15:01:13.846 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2353e-05 2024-09-13T15:01:13.846 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.846 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.846 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.846 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277182+0000", 2024-09-13T15:01:13.846 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.818e-06 2024-09-13T15:01:13.846 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.847 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.847 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.847 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277218+0000", 2024-09-13T15:01:13.847 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.578e-05 2024-09-13T15:01:13.847 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.847 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.847 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:13.847 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.277225+0000", 2024-09-13T15:01:13.847 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.4869999999999996e-06 2024-09-13T15:01:13.847 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.847 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.847 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.847 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.223362+0000", 2024-09-13T15:01:13.848 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.94613646500000004 2024-09-13T15:01:13.848 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.848 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.848 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.848 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.223463+0000", 2024-09-13T15:01:13.848 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000101635 2024-09-13T15:01:13.848 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.848 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.848 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T15:01:13.848 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.223569+0000", 2024-09-13T15:01:13.848 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000106063 2024-09-13T15:01:13.848 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.849 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.849 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.849 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233078+0000", 2024-09-13T15:01:13.849 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0095081460000000003 2024-09-13T15:01:13.849 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.849 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.849 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.849 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233289+0000", 2024-09-13T15:01:13.849 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00021173899999999999 2024-09-13T15:01:13.849 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.850 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.850 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.850 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233326+0000", 2024-09-13T15:01:13.850 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6884999999999997e-05 2024-09-13T15:01:13.850 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.850 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.850 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.850 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233373+0000", 2024-09-13T15:01:13.850 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6882999999999998e-05 2024-09-13T15:01:13.850 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.850 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.850 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.851 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233408+0000", 2024-09-13T15:01:13.851 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5098999999999997e-05 2024-09-13T15:01:13.851 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.851 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.851 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.851 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.851 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.851 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193768 7.c 7:3b3e0e52:::benchmark_data_smithi096_196313_object73637:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2024-09-13T15:01:13.851 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.313978+0000", 2024-09-13T15:01:13.851 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 568.487106036, 2024-09-13T15:01:13.851 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93176736800000004, 2024-09-13T15:01:13.851 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.852 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.852 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.852 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:01:13.852 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:01:13.852 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193768 2024-09-13T15:01:13.852 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.852 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.852 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.852 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.852 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.313978+0000", 2024-09-13T15:01:13.852 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.852 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.852 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.853 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.853 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.313978+0000", 2024-09-13T15:01:13.853 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.853 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.853 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.853 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.853 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.313976+0000", 2024-09-13T15:01:13.853 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:13.853 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.853 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.853 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.853 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.313983+0000", 2024-09-13T15:01:13.853 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6590000000000001e-06 2024-09-13T15:01:13.854 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.854 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.854 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.854 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.314003+0000", 2024-09-13T15:01:13.854 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9842000000000001e-05 2024-09-13T15:01:13.854 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.854 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.854 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.854 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.314010+0000", 2024-09-13T15:01:13.854 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0260000000000002e-06 2024-09-13T15:01:13.854 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.854 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.854 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.855 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.314057+0000", 2024-09-13T15:01:13.855 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7092000000000002e-05 2024-09-13T15:01:13.855 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.855 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.855 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:13.855 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.314065+0000", 2024-09-13T15:01:13.855 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9869999999999999e-06 2024-09-13T15:01:13.855 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.855 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.855 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.855 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.227993+0000", 2024-09-13T15:01:13.855 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.91392780200000001 2024-09-13T15:01:13.856 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.856 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.856 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.856 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.228101+0000", 2024-09-13T15:01:13.856 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000108186 2024-09-13T15:01:13.856 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.856 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.856 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T15:01:13.856 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.228225+0000", 2024-09-13T15:01:13.856 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012376100000000001 2024-09-13T15:01:13.856 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.856 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.856 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.857 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233244+0000", 2024-09-13T15:01:13.857 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0050197490000000004 2024-09-13T15:01:13.857 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.857 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.857 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.857 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233450+0000", 2024-09-13T15:01:13.857 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00020564499999999999 2024-09-13T15:01:13.857 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.857 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.857 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.857 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.245654+0000", 2024-09-13T15:01:13.857 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012204281000000001 2024-09-13T15:01:13.857 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.858 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.858 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.858 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.245706+0000", 2024-09-13T15:01:13.858 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1480000000000002e-05 2024-09-13T15:01:13.858 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.858 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.858 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.858 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.245745+0000", 2024-09-13T15:01:13.858 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9524999999999999e-05 2024-09-13T15:01:13.858 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.858 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.858 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.858 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.859 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.859 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193782 7.c 7:3bff5279:::benchmark_data_smithi096_196313_object73651:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2024-09-13T15:01:13.859 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.329589+0000", 2024-09-13T15:01:13.859 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 568.47149539700001, 2024-09-13T15:01:13.859 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.92642909600000001, 2024-09-13T15:01:13.859 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.859 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.859 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.859 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:01:13.859 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:01:13.859 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193782 2024-09-13T15:01:13.859 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.859 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.859 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.860 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.860 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329589+0000", 2024-09-13T15:01:13.860 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.860 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.860 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.860 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.860 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329589+0000", 2024-09-13T15:01:13.860 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.860 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.860 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.860 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.860 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329587+0000", 2024-09-13T15:01:13.861 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:13.861 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.861 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.861 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.861 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329594+0000", 2024-09-13T15:01:13.861 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9299999999999997e-06 2024-09-13T15:01:13.861 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.861 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.861 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.861 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329604+0000", 2024-09-13T15:01:13.861 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0658e-05 2024-09-13T15:01:13.861 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.861 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.861 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.862 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329609+0000", 2024-09-13T15:01:13.862 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3019999999999996e-06 2024-09-13T15:01:13.862 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.862 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.862 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.862 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329653+0000", 2024-09-13T15:01:13.862 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4045999999999998e-05 2024-09-13T15:01:13.862 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.862 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.862 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:13.862 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.329662+0000", 2024-09-13T15:01:13.862 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.3740000000000004e-06 2024-09-13T15:01:13.862 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.862 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.863 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.863 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233483+0000", 2024-09-13T15:01:13.863 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.90382105800000001 2024-09-13T15:01:13.863 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.863 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.863 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.863 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233646+0000", 2024-09-13T15:01:13.863 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016245300000000001 2024-09-13T15:01:13.863 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.863 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.863 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T15:01:13.863 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.233764+0000", 2024-09-13T15:01:13.863 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000118293 2024-09-13T15:01:13.863 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.864 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.864 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.864 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.245755+0000", 2024-09-13T15:01:13.864 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011990806 2024-09-13T15:01:13.864 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.864 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.864 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.864 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.245856+0000", 2024-09-13T15:01:13.864 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000100739 2024-09-13T15:01:13.864 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.864 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.864 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.864 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.255938+0000", 2024-09-13T15:01:13.865 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010082559 2024-09-13T15:01:13.865 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.865 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.865 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.865 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.255978+0000", 2024-09-13T15:01:13.865 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9461e-05 2024-09-13T15:01:13.865 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.865 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.865 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.865 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.256018+0000", 2024-09-13T15:01:13.865 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0059999999999999e-05 2024-09-13T15:01:13.865 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.865 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.865 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.866 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.866 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.866 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:193788 7.c 7:37a34649:::benchmark_data_smithi096_196313_object73657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2024-09-13T15:01:13.866 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:51:45.334453+0000", 2024-09-13T15:01:13.867 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 568.46663058900003, 2024-09-13T15:01:13.867 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.92167261300000003, 2024-09-13T15:01:13.867 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.867 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.867 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.867 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:01:13.867 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:01:13.867 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 193788 2024-09-13T15:01:13.867 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.868 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.868 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.868 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.868 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334453+0000", 2024-09-13T15:01:13.868 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.868 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.868 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.868 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.868 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334453+0000", 2024-09-13T15:01:13.868 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.868 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.868 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.869 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.869 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334452+0000", 2024-09-13T15:01:13.869 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:13.869 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.869 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.869 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.869 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334459+0000", 2024-09-13T15:01:13.869 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1539999999999996e-06 2024-09-13T15:01:13.869 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.869 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.869 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.869 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334475+0000", 2024-09-13T15:01:13.869 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5577e-05 2024-09-13T15:01:13.869 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.870 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.870 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.870 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334482+0000", 2024-09-13T15:01:13.870 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9879999999999998e-06 2024-09-13T15:01:13.870 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.870 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.870 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.870 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334526+0000", 2024-09-13T15:01:13.870 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4279000000000001e-05 2024-09-13T15:01:13.870 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.870 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.870 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:13.870 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:45.334535+0000", 2024-09-13T15:01:13.870 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.3430000000000002e-06 2024-09-13T15:01:13.871 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.871 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.871 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.871 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.237838+0000", 2024-09-13T15:01:13.871 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.90330297299999995 2024-09-13T15:01:13.871 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.871 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.871 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.871 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.237958+0000", 2024-09-13T15:01:13.871 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119401 2024-09-13T15:01:13.871 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.871 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.871 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 3,7", 2024-09-13T15:01:13.872 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.238068+0000", 2024-09-13T15:01:13.872 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000110418 2024-09-13T15:01:13.872 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.872 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.872 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.872 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.245761+0000", 2024-09-13T15:01:13.872 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0076925450000000003 2024-09-13T15:01:13.872 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.872 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.872 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.872 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.255890+0000", 2024-09-13T15:01:13.872 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010129712000000001 2024-09-13T15:01:13.872 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.872 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.873 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.873 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.256071+0000", 2024-09-13T15:01:13.873 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000180512 2024-09-13T15:01:13.873 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.873 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.873 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.873 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.256095+0000", 2024-09-13T15:01:13.873 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4224e-05 2024-09-13T15:01:13.873 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.873 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.873 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.873 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:51:46.256126+0000", 2024-09-13T15:01:13.873 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0917999999999998e-05 2024-09-13T15:01:13.873 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.874 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.874 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.874 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.874 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.874 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:226690 7.7 7:ee5af816:::benchmark_data_smithi096_196313_object106559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2024-09-13T15:01:13.874 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:52:31.424483+0000", 2024-09-13T15:01:13.874 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 522.376601381, 2024-09-13T15:01:13.874 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0532642480000001, 2024-09-13T15:01:13.874 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.874 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T15:01:13.874 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.874 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:01:13.874 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:01:13.874 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 226690 2024-09-13T15:01:13.875 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.875 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.875 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.875 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.875 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424483+0000", 2024-09-13T15:01:13.875 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.875 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.875 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.875 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.875 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424483+0000", 2024-09-13T15:01:13.875 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.875 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.875 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.875 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.876 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424482+0000", 2024-09-13T15:01:13.876 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:13.876 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.876 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.876 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.876 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424485+0000", 2024-09-13T15:01:13.876 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.3690000000000001e-06 2024-09-13T15:01:13.876 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.876 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.876 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.876 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424491+0000", 2024-09-13T15:01:13.876 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.251e-06 2024-09-13T15:01:13.876 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.876 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.877 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.877 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424495+0000", 2024-09-13T15:01:13.877 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8940000000000003e-06 2024-09-13T15:01:13.877 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.877 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.877 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.877 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424537+0000", 2024-09-13T15:01:13.877 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1993999999999998e-05 2024-09-13T15:01:13.877 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.877 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.877 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.877 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424658+0000", 2024-09-13T15:01:13.877 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000120875 2024-09-13T15:01:13.877 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.878 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.878 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,1,3", 2024-09-13T15:01:13.878 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.424769+0000", 2024-09-13T15:01:13.878 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000111619 2024-09-13T15:01:13.878 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.878 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.878 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.878 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.430664+0000", 2024-09-13T15:01:13.878 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0058946800000000002 2024-09-13T15:01:13.878 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.878 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.878 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.878 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.439757+0000", 2024-09-13T15:01:13.878 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0090930149999999994 2024-09-13T15:01:13.879 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.879 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.879 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.879 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.441422+0000", 2024-09-13T15:01:13.879 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0016653869999999999 2024-09-13T15:01:13.879 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.879 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.879 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.879 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.445501+0000", 2024-09-13T15:01:13.880 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0040786319999999996 2024-09-13T15:01:13.880 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.880 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.880 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:13.880 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.445511+0000", 2024-09-13T15:01:13.880 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0197000000000001e-05 2024-09-13T15:01:13.880 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.880 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.880 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.880 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.477698+0000", 2024-09-13T15:01:13.880 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0321867280000001 2024-09-13T15:01:13.880 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.880 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.880 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.881 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.477747+0000", 2024-09-13T15:01:13.881 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8998000000000001e-05 2024-09-13T15:01:13.881 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.881 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.881 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.881 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.881 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.881 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:226698 7.7 7:ed1c0bd5:::benchmark_data_smithi096_196313_object106567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2024-09-13T15:01:13.881 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:52:31.443183+0000", 2024-09-13T15:01:13.881 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 522.35790103800002, 2024-09-13T15:01:13.881 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0511919729999999, 2024-09-13T15:01:13.881 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.881 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.882 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.882 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:01:13.882 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:01:13.882 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 226698 2024-09-13T15:01:13.882 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.882 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.882 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.882 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.882 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.443183+0000", 2024-09-13T15:01:13.882 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.882 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.882 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.882 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.882 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.443183+0000", 2024-09-13T15:01:13.883 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.883 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.883 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.883 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.883 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.443182+0000", 2024-09-13T15:01:13.883 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:13.883 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.883 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.883 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.883 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.443185+0000", 2024-09-13T15:01:13.883 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4839999999999998e-06 2024-09-13T15:01:13.883 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.883 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.884 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.884 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.443190+0000", 2024-09-13T15:01:13.884 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5079999999999996e-06 2024-09-13T15:01:13.884 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.884 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.884 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.884 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.443194+0000", 2024-09-13T15:01:13.884 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6389999999999998e-06 2024-09-13T15:01:13.884 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.884 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.884 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.884 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.445536+0000", 2024-09-13T15:01:13.884 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0023414709999999999 2024-09-13T15:01:13.885 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.885 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.885 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:13.885 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.445544+0000", 2024-09-13T15:01:13.885 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3000000000000002e-06 2024-09-13T15:01:13.885 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.885 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.885 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.885 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.478073+0000", 2024-09-13T15:01:13.885 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0325288770000001 2024-09-13T15:01:13.885 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.885 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.885 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.886 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.478143+0000", 2024-09-13T15:01:13.886 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9918000000000004e-05 2024-09-13T15:01:13.886 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.886 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.886 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:01:13.886 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.478219+0000", 2024-09-13T15:01:13.886 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6754000000000004e-05 2024-09-13T15:01:13.886 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.886 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.886 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.886 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.487596+0000", 2024-09-13T15:01:13.886 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0093767369999999996 2024-09-13T15:01:13.886 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.886 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.887 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.887 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.487601+0000", 2024-09-13T15:01:13.887 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4780000000000002e-06 2024-09-13T15:01:13.887 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.887 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.887 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.887 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494280+0000", 2024-09-13T15:01:13.887 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0066789900000000001 2024-09-13T15:01:13.887 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.887 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.887 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.887 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494334+0000", 2024-09-13T15:01:13.887 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4627999999999999e-05 2024-09-13T15:01:13.887 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.888 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.888 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.888 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494375+0000", 2024-09-13T15:01:13.888 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0593999999999998e-05 2024-09-13T15:01:13.888 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.888 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.888 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.888 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.888 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.888 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:226705 7.7 7:e639073c:::benchmark_data_smithi096_196313_object106574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-13T15:01:13.888 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:52:31.458774+0000", 2024-09-13T15:01:13.888 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 522.34230950799997, 2024-09-13T15:01:13.889 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0357834050000001, 2024-09-13T15:01:13.889 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.889 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.889 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.889 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:01:13.889 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:01:13.889 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 226705 2024-09-13T15:01:13.889 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.889 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.890 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.890 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.890 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458774+0000", 2024-09-13T15:01:13.890 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.890 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.890 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.890 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.890 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458774+0000", 2024-09-13T15:01:13.890 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.890 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.890 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.890 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.890 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458773+0000", 2024-09-13T15:01:13.890 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:13.891 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.891 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.891 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.891 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458780+0000", 2024-09-13T15:01:13.891 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7290000000000003e-06 2024-09-13T15:01:13.891 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.891 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.891 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.891 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458791+0000", 2024-09-13T15:01:13.891 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1192999999999999e-05 2024-09-13T15:01:13.891 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.891 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.891 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.892 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458797+0000", 2024-09-13T15:01:13.892 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9800000000000003e-06 2024-09-13T15:01:13.892 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.892 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.892 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.892 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458838+0000", 2024-09-13T15:01:13.892 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1474000000000003e-05 2024-09-13T15:01:13.892 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.892 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.892 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:13.892 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.458845+0000", 2024-09-13T15:01:13.892 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7780000000000002e-06 2024-09-13T15:01:13.892 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.892 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.893 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.893 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.481450+0000", 2024-09-13T15:01:13.893 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.022604946 2024-09-13T15:01:13.893 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.893 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.893 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.893 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.481527+0000", 2024-09-13T15:01:13.893 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6643000000000005e-05 2024-09-13T15:01:13.893 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.893 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.893 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:01:13.893 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.481607+0000", 2024-09-13T15:01:13.893 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9943000000000004e-05 2024-09-13T15:01:13.894 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.894 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.894 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.894 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.487604+0000", 2024-09-13T15:01:13.894 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0059977119999999997 2024-09-13T15:01:13.894 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.894 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.894 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.894 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.487652+0000", 2024-09-13T15:01:13.894 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7268999999999998e-05 2024-09-13T15:01:13.894 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.894 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.894 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.894 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494471+0000", 2024-09-13T15:01:13.895 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0068191359999999999 2024-09-13T15:01:13.895 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.895 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.895 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.895 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494519+0000", 2024-09-13T15:01:13.895 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8155999999999998e-05 2024-09-13T15:01:13.895 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.895 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.895 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.895 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494558+0000", 2024-09-13T15:01:13.895 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8822999999999997e-05 2024-09-13T15:01:13.895 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.896 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.896 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.896 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.896 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.896 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:226760 7.7 7:ead9d959:::benchmark_data_smithi096_196313_object106629:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-13T15:01:13.896 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:52:31.515678+0000", 2024-09-13T15:01:13.896 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 522.28540562700005, 2024-09-13T15:01:13.896 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99782514, 2024-09-13T15:01:13.896 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.896 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.896 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.896 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:01:13.896 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:01:13.897 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 226760 2024-09-13T15:01:13.897 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.897 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.897 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.897 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.897 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515678+0000", 2024-09-13T15:01:13.897 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.897 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.897 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.897 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.897 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515678+0000", 2024-09-13T15:01:13.897 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.897 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.897 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.898 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.898 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515677+0000", 2024-09-13T15:01:13.898 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:13.898 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.898 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.898 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.898 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515684+0000", 2024-09-13T15:01:13.898 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.866e-06 2024-09-13T15:01:13.898 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.898 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.898 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.898 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515699+0000", 2024-09-13T15:01:13.899 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5417000000000002e-05 2024-09-13T15:01:13.899 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.899 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.899 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.899 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515706+0000", 2024-09-13T15:01:13.899 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4520000000000002e-06 2024-09-13T15:01:13.899 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.899 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.899 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.899 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515754+0000", 2024-09-13T15:01:13.899 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8773999999999998e-05 2024-09-13T15:01:13.899 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.899 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.899 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:13.900 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.515765+0000", 2024-09-13T15:01:13.900 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0103e-05 2024-09-13T15:01:13.900 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.900 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.900 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.900 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.484331+0000", 2024-09-13T15:01:13.900 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.968566869 2024-09-13T15:01:13.900 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.900 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.900 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.900 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.484438+0000", 2024-09-13T15:01:13.900 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010631700000000001 2024-09-13T15:01:13.900 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.901 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.901 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:01:13.901 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.484551+0000", 2024-09-13T15:01:13.901 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000113021 2024-09-13T15:01:13.901 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.901 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.901 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.901 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494224+0000", 2024-09-13T15:01:13.901 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0096733240000000005 2024-09-13T15:01:13.901 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.901 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.901 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.901 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.494388+0000", 2024-09-13T15:01:13.902 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000163726 2024-09-13T15:01:13.902 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.902 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.902 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.902 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.513421+0000", 2024-09-13T15:01:13.902 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.019033042 2024-09-13T15:01:13.902 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.902 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.902 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.902 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.513463+0000", 2024-09-13T15:01:13.902 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2015999999999999e-05 2024-09-13T15:01:13.902 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.902 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.903 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.903 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.513503+0000", 2024-09-13T15:01:13.903 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0602000000000002e-05 2024-09-13T15:01:13.903 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.903 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.903 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.903 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.903 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.903 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.35607.0:226835 7.7 7:eb0e6d5e:::benchmark_data_smithi096_196313_object106704:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-13T15:01:13.903 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:52:31.597667+0000", 2024-09-13T15:01:13.903 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 522.20341712100003, 2024-09-13T15:01:13.903 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.92256422400000004, 2024-09-13T15:01:13.903 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.903 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.904 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.904 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.35607", 2024-09-13T15:01:13.904 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/4263959079", 2024-09-13T15:01:13.904 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 226835 2024-09-13T15:01:13.904 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.904 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.904 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.904 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.904 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597667+0000", 2024-09-13T15:01:13.904 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.904 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.904 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.904 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.905 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597667+0000", 2024-09-13T15:01:13.905 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.905 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.905 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.905 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.905 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597666+0000", 2024-09-13T15:01:13.905 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:13.905 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.905 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.905 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.905 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597671+0000", 2024-09-13T15:01:13.905 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3989999999999997e-06 2024-09-13T15:01:13.905 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.905 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.906 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.906 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597679+0000", 2024-09-13T15:01:13.906 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8910000000000001e-06 2024-09-13T15:01:13.906 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.906 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.906 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.906 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597683+0000", 2024-09-13T15:01:13.906 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4740000000000001e-06 2024-09-13T15:01:13.906 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.906 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.906 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.906 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597725+0000", 2024-09-13T15:01:13.906 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1904999999999999e-05 2024-09-13T15:01:13.907 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.907 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.907 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:13.907 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:31.597732+0000", 2024-09-13T15:01:13.907 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.4869999999999996e-06 2024-09-13T15:01:13.907 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.907 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.907 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.907 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.487698+0000", 2024-09-13T15:01:13.907 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.88996593099999999 2024-09-13T15:01:13.907 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.907 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.907 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.907 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.487827+0000", 2024-09-13T15:01:13.908 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000129014 2024-09-13T15:01:13.908 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.908 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.908 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:01:13.908 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.487893+0000", 2024-09-13T15:01:13.908 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5758000000000003e-05 2024-09-13T15:01:13.908 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.908 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.908 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.908 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.497981+0000", 2024-09-13T15:01:13.908 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010087646 2024-09-13T15:01:13.908 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.908 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.908 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.909 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.500910+0000", 2024-09-13T15:01:13.909 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029293510000000002 2024-09-13T15:01:13.909 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.909 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.909 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.909 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.520141+0000", 2024-09-13T15:01:13.909 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.019230550999999999 2024-09-13T15:01:13.909 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.909 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.909 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.909 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.520189+0000", 2024-09-13T15:01:13.909 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8591999999999998e-05 2024-09-13T15:01:13.909 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.910 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.910 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.910 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:52:32.520231+0000", 2024-09-13T15:01:13.910 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1937000000000002e-05 2024-09-13T15:01:13.910 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.910 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.910 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.910 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.910 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.910 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:207980 9.1 9:8aa0b8cb:::benchmark_data_smithi096_256909_object88572:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-09-13T15:01:13.910 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:10.079727+0000", 2024-09-13T15:01:13.910 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 3.7213573210000002, 2024-09-13T15:01:13.910 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.044769164, 2024-09-13T15:01:13.910 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.911 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.911 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.911 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:13.911 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:13.911 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 207980 2024-09-13T15:01:13.911 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.911 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.911 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.911 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.911 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.079727+0000", 2024-09-13T15:01:13.911 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.911 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.911 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.911 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.912 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.079727+0000", 2024-09-13T15:01:13.912 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.912 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.912 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.912 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.912 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.079724+0000", 2024-09-13T15:01:13.912 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999976 2024-09-13T15:01:13.912 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.912 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.912 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.912 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.079733+0000", 2024-09-13T15:01:13.912 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4670000000000004e-06 2024-09-13T15:01:13.912 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.912 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.913 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.913 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.079748+0000", 2024-09-13T15:01:13.913 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5023e-05 2024-09-13T15:01:13.913 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.913 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.913 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.913 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.079754+0000", 2024-09-13T15:01:13.913 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9000000000000003e-06 2024-09-13T15:01:13.913 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.913 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.913 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.913 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.086561+0000", 2024-09-13T15:01:13.913 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.006807568 2024-09-13T15:01:13.914 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.914 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.914 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.914 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.086812+0000", 2024-09-13T15:01:13.914 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00025082899999999999 2024-09-13T15:01:13.914 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.914 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.914 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:01:13.914 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.087209+0000", 2024-09-13T15:01:13.914 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00039689100000000002 2024-09-13T15:01:13.914 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.914 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.914 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.915 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.098642+0000", 2024-09-13T15:01:13.915 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.01143305 2024-09-13T15:01:13.915 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.915 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.915 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.915 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.104133+0000", 2024-09-13T15:01:13.915 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0054910430000000001 2024-09-13T15:01:13.915 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.915 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.915 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.915 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.122318+0000", 2024-09-13T15:01:13.915 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.018185013999999999 2024-09-13T15:01:13.915 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.915 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.916 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.916 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.122462+0000", 2024-09-13T15:01:13.916 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000143953 2024-09-13T15:01:13.916 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.916 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.916 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.916 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.124496+0000", 2024-09-13T15:01:13.916 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0020336519999999999 2024-09-13T15:01:13.916 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.916 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.916 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.916 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.916 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.917 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:207982 9.1 9:8cd54a71:::benchmark_data_smithi096_256909_object88574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-09-13T15:01:13.917 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:10.081273+0000", 2024-09-13T15:01:13.917 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 3.719811296, 2024-09-13T15:01:13.917 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.045594856000000003, 2024-09-13T15:01:13.917 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.917 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.917 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.917 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:13.917 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:13.917 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 207982 2024-09-13T15:01:13.917 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.917 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.917 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.918 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.918 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.081273+0000", 2024-09-13T15:01:13.918 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.918 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.918 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.918 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.918 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.081273+0000", 2024-09-13T15:01:13.918 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.918 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.918 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.918 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.918 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.081271+0000", 2024-09-13T15:01:13.918 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:01:13.919 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.919 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.919 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.919 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.081278+0000", 2024-09-13T15:01:13.919 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6690000000000006e-06 2024-09-13T15:01:13.919 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.919 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.919 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.919 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.081294+0000", 2024-09-13T15:01:13.919 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5255e-05 2024-09-13T15:01:13.919 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.919 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.919 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.920 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.081300+0000", 2024-09-13T15:01:13.920 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2550000000000003e-06 2024-09-13T15:01:13.920 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.920 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.920 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.920 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.091010+0000", 2024-09-13T15:01:13.920 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0097096209999999999 2024-09-13T15:01:13.920 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.920 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.920 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.920 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.091241+0000", 2024-09-13T15:01:13.920 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00023108199999999999 2024-09-13T15:01:13.920 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.921 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.921 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:01:13.921 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.091642+0000", 2024-09-13T15:01:13.921 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00040116400000000002 2024-09-13T15:01:13.921 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.921 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.921 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.921 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.103209+0000", 2024-09-13T15:01:13.921 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011567630000000001 2024-09-13T15:01:13.921 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.921 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.921 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.921 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.104689+0000", 2024-09-13T15:01:13.921 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0014796360000000001 2024-09-13T15:01:13.922 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.922 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.922 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.922 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.124679+0000", 2024-09-13T15:01:13.922 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.019989843 2024-09-13T15:01:13.922 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.922 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.922 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.922 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.124870+0000", 2024-09-13T15:01:13.922 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00019143299999999999 2024-09-13T15:01:13.922 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.922 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.922 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.922 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:10.126867+0000", 2024-09-13T15:01:13.923 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001997146 2024-09-13T15:01:13.923 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.923 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.923 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.923 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.923 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.923 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:208861 9.1 9:83b75cf1:::benchmark_data_smithi096_256909_object89453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-09-13T15:01:13.923 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:11.177963+0000", 2024-09-13T15:01:13.923 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 2.6231209680000003, 2024-09-13T15:01:13.923 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.050241386999999998, 2024-09-13T15:01:13.923 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.923 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.923 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.924 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:13.924 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:13.924 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 208861 2024-09-13T15:01:13.924 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.924 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.924 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.924 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.924 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:13.925 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.177963+0000", 2024-09-13T15:01:13.926 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.926 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.926 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.926 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.926 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.177963+0000", 2024-09-13T15:01:13.926 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.926 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.926 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.926 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.926 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.177961+0000", 2024-09-13T15:01:13.926 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999976 2024-09-13T15:01:13.926 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.926 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.927 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.927 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.177973+0000", 2024-09-13T15:01:13.927 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2339e-05 2024-09-13T15:01:13.927 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.927 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.927 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.927 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.177993+0000", 2024-09-13T15:01:13.927 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0267000000000001e-05 2024-09-13T15:01:13.927 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.927 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.927 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.927 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.178001+0000", 2024-09-13T15:01:13.927 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9389999999999997e-06 2024-09-13T15:01:13.927 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.928 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.928 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.928 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.191069+0000", 2024-09-13T15:01:13.928 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013067336000000001 2024-09-13T15:01:13.928 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.928 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.928 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.928 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.191179+0000", 2024-09-13T15:01:13.928 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0001102 2024-09-13T15:01:13.928 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.928 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.928 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:01:13.928 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.191301+0000", 2024-09-13T15:01:13.929 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012251800000000001 2024-09-13T15:01:13.929 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.929 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.929 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.929 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.205497+0000", 2024-09-13T15:01:13.929 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.014195598 2024-09-13T15:01:13.929 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.929 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.929 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.929 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.207922+0000", 2024-09-13T15:01:13.929 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0024252380000000001 2024-09-13T15:01:13.929 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.929 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.930 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.930 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.226386+0000", 2024-09-13T15:01:13.930 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.018463547 2024-09-13T15:01:13.930 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.930 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.930 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.930 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.226447+0000", 2024-09-13T15:01:13.930 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1527999999999998e-05 2024-09-13T15:01:13.930 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.930 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.930 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.930 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.228204+0000", 2024-09-13T15:01:13.930 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001757074 2024-09-13T15:01:13.930 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.931 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.931 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.931 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.931 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.931 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:208869 9.1 9:8a8527a0:::benchmark_data_smithi096_256909_object89461:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-09-13T15:01:13.931 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:11.186378+0000", 2024-09-13T15:01:13.931 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 2.6147057949999999, 2024-09-13T15:01:13.931 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.043863454000000003, 2024-09-13T15:01:13.931 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.931 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.931 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.931 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:13.931 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:13.931 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 208869 2024-09-13T15:01:13.932 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.932 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.932 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.932 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.932 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.186378+0000", 2024-09-13T15:01:13.932 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.932 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.932 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.932 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.932 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.186378+0000", 2024-09-13T15:01:13.932 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.932 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.932 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.933 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.933 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.186376+0000", 2024-09-13T15:01:13.933 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:01:13.933 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.933 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.933 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.933 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.186383+0000", 2024-09-13T15:01:13.933 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0979999999999996e-06 2024-09-13T15:01:13.933 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.933 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.933 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.933 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.186393+0000", 2024-09-13T15:01:13.933 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7119999999999995e-06 2024-09-13T15:01:13.933 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.934 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.934 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.934 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.186399+0000", 2024-09-13T15:01:13.934 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7780000000000004e-06 2024-09-13T15:01:13.934 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.934 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.934 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.934 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.194352+0000", 2024-09-13T15:01:13.934 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0079533050000000008 2024-09-13T15:01:13.934 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.934 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.934 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.934 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.194450+0000", 2024-09-13T15:01:13.935 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7683000000000006e-05 2024-09-13T15:01:13.935 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.935 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.935 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:01:13.935 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.194590+0000", 2024-09-13T15:01:13.935 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013985899999999999 2024-09-13T15:01:13.935 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.935 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.935 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.935 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.205503+0000", 2024-09-13T15:01:13.935 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010913661 2024-09-13T15:01:13.935 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.935 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.936 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.936 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.207963+0000", 2024-09-13T15:01:13.936 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0024597709999999999 2024-09-13T15:01:13.936 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.936 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.936 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.936 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.228352+0000", 2024-09-13T15:01:13.936 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.020388729000000001 2024-09-13T15:01:13.936 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.936 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.936 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.936 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.228487+0000", 2024-09-13T15:01:13.936 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000135217 2024-09-13T15:01:13.936 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.937 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.937 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.937 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.230242+0000", 2024-09-13T15:01:13.937 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00175449 2024-09-13T15:01:13.937 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.937 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.937 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.937 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.937 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.937 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:208871 9.1 9:847bcf70:::benchmark_data_smithi096_256909_object89463:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-09-13T15:01:13.937 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:11.187741+0000", 2024-09-13T15:01:13.937 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 2.6133429120000002, 2024-09-13T15:01:13.937 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.046425150999999998, 2024-09-13T15:01:13.938 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.938 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.938 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.938 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:13.938 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:13.938 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 208871 2024-09-13T15:01:13.938 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.938 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.938 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.938 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.938 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.187741+0000", 2024-09-13T15:01:13.938 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.938 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.938 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.938 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.939 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.187741+0000", 2024-09-13T15:01:13.939 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.939 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.939 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.939 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.939 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.187739+0000", 2024-09-13T15:01:13.939 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:13.939 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.939 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.939 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.939 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.187746+0000", 2024-09-13T15:01:13.939 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3969999999999999e-06 2024-09-13T15:01:13.939 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.940 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.940 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.940 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.187763+0000", 2024-09-13T15:01:13.940 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7025e-05 2024-09-13T15:01:13.940 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.940 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.940 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.940 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.187770+0000", 2024-09-13T15:01:13.940 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9920000000000001e-06 2024-09-13T15:01:13.940 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.940 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.940 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.940 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.197472+0000", 2024-09-13T15:01:13.940 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0097020679999999995 2024-09-13T15:01:13.941 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.941 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.941 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.941 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.197694+0000", 2024-09-13T15:01:13.941 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000221771 2024-09-13T15:01:13.941 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.941 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.941 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:01:13.941 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.198102+0000", 2024-09-13T15:01:13.941 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00040840499999999998 2024-09-13T15:01:13.941 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.941 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.942 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.942 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.206085+0000", 2024-09-13T15:01:13.942 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0079831929999999995 2024-09-13T15:01:13.942 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.942 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.942 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.942 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.211245+0000", 2024-09-13T15:01:13.942 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0051600350000000003 2024-09-13T15:01:13.942 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.942 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.942 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.942 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.233179+0000", 2024-09-13T15:01:13.942 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.021933976000000001 2024-09-13T15:01:13.942 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.943 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.943 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.943 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.233243+0000", 2024-09-13T15:01:13.943 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4029999999999995e-05 2024-09-13T15:01:13.943 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.943 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.943 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.943 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.234166+0000", 2024-09-13T15:01:13.943 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00092285299999999995 2024-09-13T15:01:13.943 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.943 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.943 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.943 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.944 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.944 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:208877 9.1 9:8717a194:::benchmark_data_smithi096_256909_object89469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-09-13T15:01:13.944 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:11.191811+0000", 2024-09-13T15:01:13.944 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 2.6092729800000001, 2024-09-13T15:01:13.944 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.053105755999999997, 2024-09-13T15:01:13.944 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:13.944 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:13.944 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:13.944 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:13.944 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:13.944 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 208877 2024-09-13T15:01:13.944 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.944 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:13.945 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.945 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:13.945 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.191811+0000", 2024-09-13T15:01:13.945 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.945 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.945 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.945 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:13.945 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.191811+0000", 2024-09-13T15:01:13.945 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:13.945 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.945 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.945 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:13.945 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.191809+0000", 2024-09-13T15:01:13.946 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:13.946 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.946 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.946 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:13.946 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.191817+0000", 2024-09-13T15:01:13.946 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0679999999999999e-06 2024-09-13T15:01:13.946 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.946 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.946 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:13.946 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.191829+0000", 2024-09-13T15:01:13.946 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2317000000000001e-05 2024-09-13T15:01:13.946 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.946 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.947 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:13.947 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.191836+0000", 2024-09-13T15:01:13.947 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.674e-06 2024-09-13T15:01:13.947 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.947 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.947 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:13.947 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.201701+0000", 2024-09-13T15:01:13.947 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0098656230000000004 2024-09-13T15:01:13.947 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.947 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.947 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:13.947 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.201923+0000", 2024-09-13T15:01:13.947 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000221881 2024-09-13T15:01:13.947 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.948 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.948 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:01:13.948 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.202300+0000", 2024-09-13T15:01:13.948 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000376924 2024-09-13T15:01:13.948 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.948 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.948 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:13.948 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.207708+0000", 2024-09-13T15:01:13.948 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0054076840000000003 2024-09-13T15:01:13.948 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.948 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.948 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.948 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.213221+0000", 2024-09-13T15:01:13.949 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0055131870000000001 2024-09-13T15:01:13.949 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.949 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.949 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:13.949 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.242605+0000", 2024-09-13T15:01:13.949 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.029384246999999999 2024-09-13T15:01:13.949 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.949 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.949 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:13.950 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.242802+0000", 2024-09-13T15:01:13.950 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000196787 2024-09-13T15:01:13.950 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:13.950 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:13.950 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:13.950 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:11.244917+0000", 2024-09-13T15:01:13.950 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0021148730000000002 2024-09-13T15:01:13.950 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.950 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.950 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.950 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:13.950 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:13.950 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:01:13.984 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:13.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:14.086 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:14.107 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:14.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:14.209 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:14.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:13 smithi204 ceph-mon[87071]: pgmap v1113: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 28502/97401 objects degraded (29.263%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T15:01:14.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:13 smithi204 ceph-mon[87071]: osd.3 [v2:172.21.15.96:6826/3715117729,v1:172.21.15.96:6827/3715117729] boot 2024-09-13T15:01:14.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:13 smithi204 ceph-mon[87071]: osdmap e335: 8 total, 8 up, 7 in 2024-09-13T15:01:14.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:13 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T15:01:14.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.233 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:14.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:14.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:13 smithi096 ceph-mon[98901]: pgmap v1113: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 28502/97401 objects degraded (29.263%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T15:01:14.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:13 smithi096 ceph-mon[98901]: osd.3 [v2:172.21.15.96:6826/3715117729,v1:172.21.15.96:6827/3715117729] boot 2024-09-13T15:01:14.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:13 smithi096 ceph-mon[98901]: osdmap e335: 8 total, 8 up, 7 in 2024-09-13T15:01:14.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:13 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T15:01:14.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:13 smithi096 ceph-mon[102928]: pgmap v1113: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 28502/97401 objects degraded (29.263%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T15:01:14.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:13 smithi096 ceph-mon[102928]: osd.3 [v2:172.21.15.96:6826/3715117729,v1:172.21.15.96:6827/3715117729] boot 2024-09-13T15:01:14.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:13 smithi096 ceph-mon[102928]: osdmap e335: 8 total, 8 up, 7 in 2024-09-13T15:01:14.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:13 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T15:01:14.334 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:14.356 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:14.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:14.458 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:14.470 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:14.479 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:14.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:14.581 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:14.606 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:14.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:14.708 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:14.712 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:14.730 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:14.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:14.832 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:14.854 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:14.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:14.946 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T15:01:14.955 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:14.977 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:14.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:14.980 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:14 smithi096 ceph-mon[98901]: osdmap e336: 8 total, 8 up, 7 in 2024-09-13T15:01:14.980 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:14 smithi096 ceph-mon[98901]: pgmap v1116: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 799 op/s; 27741/88179 objects degraded (31.460%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T15:01:14.980 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:14 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 28502/97401 objects degraded (29.263%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:14.980 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:14 smithi096 ceph-mon[98901]: osdmap e337: 8 total, 8 up, 7 in 2024-09-13T15:01:14.981 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:14 smithi096 ceph-mon[102928]: osdmap e336: 8 total, 8 up, 7 in 2024-09-13T15:01:14.981 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:14 smithi096 ceph-mon[102928]: pgmap v1116: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 799 op/s; 27741/88179 objects degraded (31.460%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T15:01:14.981 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:14 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 28502/97401 objects degraded (29.263%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:14.981 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:14 smithi096 ceph-mon[102928]: osdmap e337: 8 total, 8 up, 7 in 2024-09-13T15:01:15.080 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:15.103 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:15.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:15.205 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:15.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:14 smithi204 ceph-mon[87071]: osdmap e336: 8 total, 8 up, 7 in 2024-09-13T15:01:15.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:14 smithi204 ceph-mon[87071]: pgmap v1116: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 799 op/s; 27741/88179 objects degraded (31.460%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T15:01:15.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:14 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 28502/97401 objects degraded (29.263%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:15.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:14 smithi204 ceph-mon[87071]: osdmap e337: 8 total, 8 up, 7 in 2024-09-13T15:01:15.229 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:15.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:15.234 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:15.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:15.352 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:15.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:15.454 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:15.483 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:15.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:15.585 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:15.620 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:15.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:15.721 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:15.753 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:15.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:15.856 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:15.890 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:15.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:15.992 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:16.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.025 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:16.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:16.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:15 smithi096 ceph-mon[98901]: 9.8 continuing backfill to osd.7 from (317'6970,335'13362] 9:101bedb3:::benchmark_data_smithi096_256909_object15566:head to 335'13362 2024-09-13T15:01:16.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:15 smithi096 ceph-mon[98901]: 9.2 continuing backfill to osd.7 from (321'7799,335'13212] MIN to 335'13212 2024-09-13T15:01:16.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:15 smithi096 ceph-mon[98901]: osdmap e338: 8 total, 8 up, 7 in 2024-09-13T15:01:16.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:15 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2503369426' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:01:16.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:15 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:01:16.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:15 smithi096 ceph-mon[102928]: 9.8 continuing backfill to osd.7 from (317'6970,335'13362] 9:101bedb3:::benchmark_data_smithi096_256909_object15566:head to 335'13362 2024-09-13T15:01:16.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:15 smithi096 ceph-mon[102928]: 9.2 continuing backfill to osd.7 from (321'7799,335'13212] MIN to 335'13212 2024-09-13T15:01:16.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:15 smithi096 ceph-mon[102928]: osdmap e338: 8 total, 8 up, 7 in 2024-09-13T15:01:16.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:15 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2503369426' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:01:16.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:15 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:01:16.127 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:16.158 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:16.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:16.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:15 smithi204 ceph-mon[87071]: 9.8 continuing backfill to osd.7 from (317'6970,335'13362] 9:101bedb3:::benchmark_data_smithi096_256909_object15566:head to 335'13362 2024-09-13T15:01:16.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:15 smithi204 ceph-mon[87071]: 9.2 continuing backfill to osd.7 from (321'7799,335'13212] MIN to 335'13212 2024-09-13T15:01:16.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:15 smithi204 ceph-mon[87071]: osdmap e338: 8 total, 8 up, 7 in 2024-09-13T15:01:16.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:15 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2503369426' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:01:16.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:15 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:01:16.260 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:16.297 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:16.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:16.336 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T15:01:16.399 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:16.440 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:16.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:16.542 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:16.579 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:16.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:16.681 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:16.712 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:16.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:16.813 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:16.861 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:16.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:16.963 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:17.006 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:17.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:17.108 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:17.158 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:17.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:17.183 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:16 smithi096 ceph-mon[98901]: 9.c continuing backfill to osd.0 from (320'7471,335'13503] MIN to 335'13503 2024-09-13T15:01:17.183 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:16 smithi096 ceph-mon[98901]: pgmap v1119: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 25794/85260 objects degraded (30.253%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T15:01:17.183 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:16 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:17.183 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:16 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:01:17.183 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:16 smithi096 ceph-mon[98901]: osdmap e339: 8 total, 8 up, 7 in 2024-09-13T15:01:17.183 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:16 smithi096 ceph-mon[102928]: 9.c continuing backfill to osd.0 from (320'7471,335'13503] MIN to 335'13503 2024-09-13T15:01:17.184 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:16 smithi096 ceph-mon[102928]: pgmap v1119: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 25794/85260 objects degraded (30.253%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T15:01:17.184 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:16 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:17.184 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:16 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:01:17.184 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:16 smithi096 ceph-mon[102928]: osdmap e339: 8 total, 8 up, 7 in 2024-09-13T15:01:17.192 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:01:17.192 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T15:01:17.192 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.192 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:212812 9.e 9:794820cd:::benchmark_data_smithi096_256909_object93404:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2024-09-13T15:01:17.192 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:17.162085+0000", 2024-09-13T15:01:17.192 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.017916813, 2024-09-13T15:01:17.192 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.017949983999999999, 2024-09-13T15:01:17.192 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:17.192 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:01:17.192 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:17.192 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:17.193 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:17.193 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 212812 2024-09-13T15:01:17.193 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.193 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:17.193 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.193 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:17.193 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162085+0000", 2024-09-13T15:01:17.193 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.193 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.193 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.193 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:17.194 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162085+0000", 2024-09-13T15:01:17.194 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.194 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.194 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.194 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:17.194 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162085+0000", 2024-09-13T15:01:17.194 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:17.194 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.194 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.194 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:17.194 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162087+0000", 2024-09-13T15:01:17.195 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.057e-06 2024-09-13T15:01:17.195 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.195 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.195 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:17.195 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162092+0000", 2024-09-13T15:01:17.195 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.933e-06 2024-09-13T15:01:17.195 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.195 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.195 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:17.195 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162095+0000", 2024-09-13T15:01:17.195 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2710000000000001e-06 2024-09-13T15:01:17.195 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.196 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.196 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:17.196 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.166707+0000", 2024-09-13T15:01:17.196 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0046119059999999998 2024-09-13T15:01:17.196 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.196 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.196 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:17.196 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.166816+0000", 2024-09-13T15:01:17.196 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000109188 2024-09-13T15:01:17.196 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.196 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.196 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:01:17.196 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.166918+0000", 2024-09-13T15:01:17.196 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010219 2024-09-13T15:01:17.197 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.197 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.197 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:17.197 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.174135+0000", 2024-09-13T15:01:17.197 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0072163849999999996 2024-09-13T15:01:17.197 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.197 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.197 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:17.197 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177046+0000", 2024-09-13T15:01:17.197 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029116620000000002 2024-09-13T15:01:17.197 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.197 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:17.197 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.198 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.198 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.198 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.46040.0:212819 9.8 e339/336 9:1095bfc8:::benchmark_data_smithi096_256909_object93411:head v 339'13468, mlcod=317'7126)", 2024-09-13T15:01:17.198 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:17.175104+0000", 2024-09-13T15:01:17.198 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0048981349999999996, 2024-09-13T15:01:17.198 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0050164889999999998, 2024-09-13T15:01:17.198 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:17.198 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:01:17.198 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:17.198 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.198 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:17.198 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.175104+0000", 2024-09-13T15:01:17.199 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.199 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.199 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.199 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:17.199 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.175104+0000", 2024-09-13T15:01:17.199 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.199 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.199 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.199 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:17.199 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.175103+0000", 2024-09-13T15:01:17.199 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:01:17.199 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.199 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.200 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:17.200 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.175111+0000", 2024-09-13T15:01:17.200 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.79e-06 2024-09-13T15:01:17.200 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.200 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.200 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:17.200 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.175125+0000", 2024-09-13T15:01:17.200 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.465e-05 2024-09-13T15:01:17.200 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.200 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.200 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:17.200 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.175131+0000", 2024-09-13T15:01:17.201 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5679999999999999e-06 2024-09-13T15:01:17.201 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.201 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.201 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:17.201 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.176881+0000", 2024-09-13T15:01:17.201 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0017496160000000001 2024-09-13T15:01:17.201 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.201 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.201 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:17.201 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.176948+0000", 2024-09-13T15:01:17.201 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7250999999999997e-05 2024-09-13T15:01:17.201 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.201 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:17.202 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.202 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.202 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.202 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:212820 9.e 9:7a36a4d3:::benchmark_data_smithi096_256909_object93412:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2024-09-13T15:01:17.202 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:17.176076+0000", 2024-09-13T15:01:17.202 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0039258870000000003, 2024-09-13T15:01:17.202 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0041013250000000003, 2024-09-13T15:01:17.202 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:17.202 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:01:17.202 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:17.202 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:17.202 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:17.202 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 212820 2024-09-13T15:01:17.203 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.203 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:17.203 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.203 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:17.203 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.176076+0000", 2024-09-13T15:01:17.203 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.203 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.203 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.203 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:17.203 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.176076+0000", 2024-09-13T15:01:17.203 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.203 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.203 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.204 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:17.204 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.176075+0000", 2024-09-13T15:01:17.204 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:17.204 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.204 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.204 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:17.204 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.176080+0000", 2024-09-13T15:01:17.204 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2310000000000004e-06 2024-09-13T15:01:17.204 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.204 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.204 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:17.204 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.176091+0000", 2024-09-13T15:01:17.204 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0538e-05 2024-09-13T15:01:17.205 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.205 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.205 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:17.205 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.176095+0000", 2024-09-13T15:01:17.205 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8169999999999998e-06 2024-09-13T15:01:17.205 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.205 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.205 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:17.205 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177064+0000", 2024-09-13T15:01:17.205 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00096865299999999998 2024-09-13T15:01:17.205 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.205 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.205 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:17.206 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177173+0000", 2024-09-13T15:01:17.206 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010849399999999999 2024-09-13T15:01:17.206 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.206 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.206 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:01:17.206 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177279+0000", 2024-09-13T15:01:17.206 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000105982 2024-09-13T15:01:17.206 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.206 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:17.206 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.206 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.206 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.206 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.46040.0:212821 9.8 e339/336 9:11df21d1:::benchmark_data_smithi096_256909_object93413:head v 339'13469, mlcod=317'7126)", 2024-09-13T15:01:17.207 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:17.177545+0000", 2024-09-13T15:01:17.207 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0024567249999999999, 2024-09-13T15:01:17.207 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0026964929999999999, 2024-09-13T15:01:17.207 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:17.207 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:01:17.207 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:17.207 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.207 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:17.207 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177545+0000", 2024-09-13T15:01:17.207 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.207 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.207 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.207 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:17.208 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177545+0000", 2024-09-13T15:01:17.208 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.208 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.208 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.208 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:17.208 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177545+0000", 2024-09-13T15:01:17.208 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:17.208 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.208 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.208 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:17.208 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177554+0000", 2024-09-13T15:01:17.208 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.9090000000000006e-06 2024-09-13T15:01:17.208 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.209 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.209 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:17.209 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177562+0000", 2024-09-13T15:01:17.209 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7430000000000008e-06 2024-09-13T15:01:17.209 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.209 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.209 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:17.209 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177567+0000", 2024-09-13T15:01:17.209 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6e-06 2024-09-13T15:01:17.209 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.209 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.209 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:17.210 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.180134+0000", 2024-09-13T15:01:17.210 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0025668760000000001 2024-09-13T15:01:17.210 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.210 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.210 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:17.210 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.180163+0000", 2024-09-13T15:01:17.210 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9502e-05 2024-09-13T15:01:17.210 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.210 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:17.210 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.210 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.210 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.210 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.46040.0:212812 9.e e339/302)", 2024-09-13T15:01:17.211 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:17.177896+0000", 2024-09-13T15:01:17.211 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0021059099999999999, 2024-09-13T15:01:17.211 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0023961450000000001, 2024-09-13T15:01:17.211 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:17.211 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "queued for pg", 2024-09-13T15:01:17.211 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:17.211 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.211 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:17.211 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177896+0000", 2024-09-13T15:01:17.211 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.211 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.211 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.211 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:17.211 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177896+0000", 2024-09-13T15:01:17.212 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.212 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.212 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.212 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:17.212 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177896+0000", 2024-09-13T15:01:17.212 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:17.212 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.212 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.212 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:17.212 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177903+0000", 2024-09-13T15:01:17.212 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7549999999999994e-06 2024-09-13T15:01:17.212 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.212 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.213 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:17.213 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177913+0000", 2024-09-13T15:01:17.213 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9660000000000008e-06 2024-09-13T15:01:17.213 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.213 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.213 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:17.213 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177919+0000", 2024-09-13T15:01:17.213 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.339e-06 2024-09-13T15:01:17.213 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.213 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:17.213 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.213 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.213 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.213 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:212823 9.e 9.c800dc7e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e339)", 2024-09-13T15:01:17.214 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:17.177964+0000", 2024-09-13T15:01:17.214 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.002038509, 2024-09-13T15:01:17.214 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002370867, 2024-09-13T15:01:17.214 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:17.214 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "queued for pg", 2024-09-13T15:01:17.214 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:17.214 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:17.214 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:17.214 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 212823 2024-09-13T15:01:17.214 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.214 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:17.214 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.214 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:17.214 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177964+0000", 2024-09-13T15:01:17.215 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.215 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.215 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.215 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:17.215 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177964+0000", 2024-09-13T15:01:17.215 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.215 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.215 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.215 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:17.215 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177963+0000", 2024-09-13T15:01:17.215 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:01:17.215 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.215 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.215 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:17.216 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177967+0000", 2024-09-13T15:01:17.216 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9439999999999998e-06 2024-09-13T15:01:17.216 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.216 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.216 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:17.216 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177980+0000", 2024-09-13T15:01:17.216 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3040000000000001e-05 2024-09-13T15:01:17.216 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.216 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.216 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:17.216 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.177984+0000", 2024-09-13T15:01:17.216 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2259999999999999e-06 2024-09-13T15:01:17.216 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.216 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:17.217 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.217 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.217 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.217 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.46040.0:212825 9.5 e339/302 9:ad87bbe6:::benchmark_data_smithi096_256909_object93417:head v 339'13417, mlcod=339'13415)", 2024-09-13T15:01:17.217 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:17.179684+0000", 2024-09-13T15:01:17.217 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.00031823799999999998, 2024-09-13T15:01:17.217 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00070048500000000004, 2024-09-13T15:01:17.217 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:17.217 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:01:17.217 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:17.217 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.217 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:17.217 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.179684+0000", 2024-09-13T15:01:17.218 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.218 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.218 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.218 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:17.218 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.179684+0000", 2024-09-13T15:01:17.218 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.218 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.218 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.218 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:17.218 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.179683+0000", 2024-09-13T15:01:17.218 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:01:17.218 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.218 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.218 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:17.218 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.179690+0000", 2024-09-13T15:01:17.219 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.878e-06 2024-09-13T15:01:17.219 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.219 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.219 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:17.219 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.179700+0000", 2024-09-13T15:01:17.219 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0322999999999999e-05 2024-09-13T15:01:17.219 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.219 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.219 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:17.219 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.179706+0000", 2024-09-13T15:01:17.219 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0129999999999999e-06 2024-09-13T15:01:17.219 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.219 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.220 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:17.220 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.179761+0000", 2024-09-13T15:01:17.220 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4568000000000003e-05 2024-09-13T15:01:17.220 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.220 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.220 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:17.220 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.180019+0000", 2024-09-13T15:01:17.220 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00025857300000000001 2024-09-13T15:01:17.220 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.220 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:17.220 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.220 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.220 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.220 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:212807 9.3 9:c8dbce8a:::benchmark_data_smithi096_256909_object93399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2024-09-13T15:01:17.221 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:17.162006+0000", 2024-09-13T15:01:17.221 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.017996469000000001, 2024-09-13T15:01:17.221 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.018432631000000001, 2024-09-13T15:01:17.221 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:17.221 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:01:17.221 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:17.221 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:17.221 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:17.221 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 212807 2024-09-13T15:01:17.221 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.221 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:17.221 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.221 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:17.221 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162006+0000", 2024-09-13T15:01:17.222 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.222 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.222 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.222 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:17.222 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162006+0000", 2024-09-13T15:01:17.222 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.222 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.222 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.222 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:17.222 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162004+0000", 2024-09-13T15:01:17.222 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:17.222 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.222 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.222 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:17.223 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162013+0000", 2024-09-13T15:01:17.223 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.3400000000000004e-06 2024-09-13T15:01:17.223 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.223 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.223 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:17.223 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162030+0000", 2024-09-13T15:01:17.223 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6748e-05 2024-09-13T15:01:17.223 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.223 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.223 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:17.223 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162036+0000", 2024-09-13T15:01:17.223 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9349999999999999e-06 2024-09-13T15:01:17.223 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.224 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.224 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:17.224 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162169+0000", 2024-09-13T15:01:17.224 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013301400000000001 2024-09-13T15:01:17.224 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.224 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.224 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:17.224 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162273+0000", 2024-09-13T15:01:17.224 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000103948 2024-09-13T15:01:17.224 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.224 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.224 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:17.224 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162386+0000", 2024-09-13T15:01:17.225 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000112601 2024-09-13T15:01:17.225 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.225 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.225 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:17.225 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.173972+0000", 2024-09-13T15:01:17.225 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011586587000000001 2024-09-13T15:01:17.225 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.225 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.225 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:17.225 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.174186+0000", 2024-09-13T15:01:17.225 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00021355199999999999 2024-09-13T15:01:17.225 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.225 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:17.226 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.226 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.226 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.226 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:212810 9.3 9:cbff8e37:::benchmark_data_smithi096_256909_object93402:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2024-09-13T15:01:17.226 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:17.162051+0000", 2024-09-13T15:01:17.226 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.017951198000000002, 2024-09-13T15:01:17.226 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.018464023, 2024-09-13T15:01:17.226 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:17.226 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:01:17.226 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:17.226 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:17.226 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:17.226 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 212810 2024-09-13T15:01:17.227 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.227 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:17.227 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.227 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:17.227 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162051+0000", 2024-09-13T15:01:17.227 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.227 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.227 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.227 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:17.227 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162051+0000", 2024-09-13T15:01:17.227 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:17.227 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.228 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.228 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:17.228 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162050+0000", 2024-09-13T15:01:17.228 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:17.228 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.228 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.228 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:17.228 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162053+0000", 2024-09-13T15:01:17.228 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8140000000000001e-06 2024-09-13T15:01:17.228 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.228 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.228 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:17.228 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162059+0000", 2024-09-13T15:01:17.228 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1460000000000001e-06 2024-09-13T15:01:17.229 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.229 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.229 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:17.229 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.162063+0000", 2024-09-13T15:01:17.229 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6349999999999999e-06 2024-09-13T15:01:17.229 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.229 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.229 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:17.229 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.170198+0000", 2024-09-13T15:01:17.229 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0081348340000000005 2024-09-13T15:01:17.229 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.229 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.229 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:17.229 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.170306+0000", 2024-09-13T15:01:17.230 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000108084 2024-09-13T15:01:17.230 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.230 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.230 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:17.230 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.170414+0000", 2024-09-13T15:01:17.230 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010848700000000001 2024-09-13T15:01:17.230 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.230 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.230 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:17.230 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.174190+0000", 2024-09-13T15:01:17.230 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0037752770000000001 2024-09-13T15:01:17.230 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:17.230 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:17.231 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:17.231 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:17.178094+0000", 2024-09-13T15:01:17.231 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0039037389999999998 2024-09-13T15:01:17.231 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.231 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:17.231 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.231 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:17.231 INFO:teuthology.orchestra.run.smithi204.stdout: ], 2024-09-13T15:01:17.231 INFO:teuthology.orchestra.run.smithi204.stdout: "num_ops": 9 2024-09-13T15:01:17.231 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:01:17.232 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:16 smithi204 ceph-mon[87071]: 9.c continuing backfill to osd.0 from (320'7471,335'13503] MIN to 335'13503 2024-09-13T15:01:17.232 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:16 smithi204 ceph-mon[87071]: pgmap v1119: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 25794/85260 objects degraded (30.253%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T15:01:17.232 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:16 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:17.232 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:16 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:01:17.232 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:16 smithi204 ceph-mon[87071]: osdmap e339: 8 total, 8 up, 7 in 2024-09-13T15:01:17.260 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:17.300 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:17.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:17.401 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:17.439 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:17.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:17.541 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:17.567 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:17.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:17.650 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:01:17.651 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:01:17.668 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:17.687 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:01:17.687 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:01:17.701 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:17.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:17.730 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:01:17.730 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:01:17.762 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:01:17.762 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-09-13T15:01:17.787 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:01:17.787 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:01:17.803 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:17.820 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:01:17.820 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:01:17.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.837 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:17.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:17.846 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:01:17.847 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:01:17.874 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:01:17.875 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:01:17.937 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:17.971 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:17.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:18.071 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:17 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3791040087' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:18.071 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:01:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:01:17] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.51.0" 2024-09-13T15:01:18.072 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:17 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3791040087' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:18.072 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:18.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.105 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:18.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:18.206 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:18.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:17 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3791040087' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:18.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.242 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:18.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:18.343 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:18.381 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:18.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:18.484 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:18.516 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:18.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:18.618 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:18.639 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:18.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:18.741 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:18.782 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:18.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:18.884 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:18.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.923 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:18.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:19.024 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:19.036 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T15:01:19.065 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:19.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:19.168 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:19.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.210 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:19.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:19.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:18 smithi204 ceph-mon[87071]: pgmap v1121: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.6 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 24640/79956 objects degraded (30.817%); 490/79956 objects misplaced (0.613%); 6.8 MiB/s, 109 objects/s recovering 2024-09-13T15:01:19.311 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:19.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:18 smithi096 ceph-mon[98901]: pgmap v1121: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.6 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 24640/79956 objects degraded (30.817%); 490/79956 objects misplaced (0.613%); 6.8 MiB/s, 109 objects/s recovering 2024-09-13T15:01:19.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:18 smithi096 ceph-mon[102928]: pgmap v1121: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.6 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 24640/79956 objects degraded (30.817%); 490/79956 objects misplaced (0.613%); 6.8 MiB/s, 109 objects/s recovering 2024-09-13T15:01:19.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.354 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:19.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:19.455 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:19.486 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:19.492 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:19.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:19.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:19.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.638 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:19.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:19.739 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:19.783 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:19.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:19.885 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:19.908 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:19.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:20.009 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:20.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.031 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:20.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:20.132 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:20.154 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:20.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:20.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:19 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 24640/79956 objects degraded (30.817%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:20.249 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:01:20.256 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:20.265 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:19 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 24640/79956 objects degraded (30.817%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:20.266 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:19 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 24640/79956 objects degraded (30.817%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:20.290 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:20.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:20.392 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:20.421 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:20.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:01:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:01:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:01:20.523 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:20.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.548 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:20.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:20.649 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:20.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.688 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:20.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:20.721 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:01:20.721 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:01:20.789 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:20.809 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:20.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:20.911 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:20.932 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:20.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:21.034 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:21.057 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:21.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:21.160 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:21.198 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:21.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:21.300 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:21.332 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:21.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:21.434 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:21.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.470 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:21.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:21.529 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T15:01:21.564 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:21.572 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:21.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.609 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:21.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:21.623 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:01:21.710 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:21.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.779 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:21.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:21.881 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:21.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.925 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:21.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:22.027 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:22.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.058 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:22.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:22.122 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:21 smithi204 ceph-mon[87071]: pgmap v1122: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 687 op/s; 25953/75792 objects degraded (34.242%); 490/75792 objects misplaced (0.647%); 4.8 MiB/s, 76 objects/s recovering 2024-09-13T15:01:22.347 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:01:22.347 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:22.348 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:22.349 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:21 smithi096 ceph-mon[98901]: pgmap v1122: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 687 op/s; 25953/75792 objects degraded (34.242%); 490/75792 objects misplaced (0.647%); 4.8 MiB/s, 76 objects/s recovering 2024-09-13T15:01:22.349 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:01:22.349 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:22.349 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:21 smithi096 ceph-mon[102928]: pgmap v1122: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 687 op/s; 25953/75792 objects degraded (34.242%); 490/75792 objects misplaced (0.647%); 4.8 MiB/s, 76 objects/s recovering 2024-09-13T15:01:22.350 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:01:22.350 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:22.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.381 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:22.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:22.482 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:22.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.520 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:22.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:22.621 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:22.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.655 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:22.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:22.756 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:22.781 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:22.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:22.883 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:22.920 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:22.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:22.960 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:23.004 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:23.022 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:23.053 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:23.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:23.155 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:23.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.187 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:23.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:23.288 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:23.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.336 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:23.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:23.437 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:23.471 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:23.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:23.573 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:23.602 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:23.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:23.704 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:23.740 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:23.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:23.842 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:23.873 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:23.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:23.974 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:24.004 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:23 smithi096 ceph-mon[98901]: pgmap v1123: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 403 op/s; 25953/75792 objects degraded (34.242%); 490/75792 objects misplaced (0.647%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T15:01:24.005 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:23 smithi096 ceph-mon[102928]: pgmap v1123: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 403 op/s; 25953/75792 objects degraded (34.242%); 490/75792 objects misplaced (0.647%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T15:01:24.006 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:24.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:24.108 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:24.139 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:24.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:24.215 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:23 smithi204 ceph-mon[87071]: pgmap v1123: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 403 op/s; 25953/75792 objects degraded (34.242%); 490/75792 objects misplaced (0.647%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T15:01:24.241 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:24.267 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:24.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:24.369 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:24.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.397 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:24.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:24.498 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:24.528 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:24.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:24.630 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:24.657 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:24.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:24.759 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:24.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.790 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:24.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:24.883 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T15:01:24.892 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:24.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.930 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:24.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:25.000 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:24 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1198445108' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:25.000 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:24 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 25953/75792 objects degraded (34.242%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:25.001 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:24 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1198445108' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:25.001 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:24 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 25953/75792 objects degraded (34.242%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:25.030 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:25.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.071 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:25.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:25.145 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:24 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1198445108' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:25.145 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:24 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 25953/75792 objects degraded (34.242%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:25.172 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:25.203 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:25.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:25.305 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:25.333 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:25.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:25.435 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:25.474 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:25.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:25.575 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:25.611 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:25.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:25.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:25.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.753 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:25.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:25.856 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:25.887 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:25.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:25.989 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:26.023 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:26.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:26.125 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:26.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.159 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:26.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:26.195 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:25 smithi096 ceph-mon[98901]: pgmap v1124: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.4 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 25498/68154 objects degraded (37.412%); 153/68154 objects misplaced (0.224%); 5.7 MiB/s, 90 objects/s recovering 2024-09-13T15:01:26.196 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:25 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:26.196 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:25 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1198445108' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:26.196 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:25 smithi096 ceph-mon[98901]: osdmap e340: 8 total, 8 up, 7 in 2024-09-13T15:01:26.196 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:25 smithi096 ceph-mon[102928]: pgmap v1124: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.4 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 25498/68154 objects degraded (37.412%); 153/68154 objects misplaced (0.224%); 5.7 MiB/s, 90 objects/s recovering 2024-09-13T15:01:26.196 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:25 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:26.196 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:25 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1198445108' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:26.196 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:25 smithi096 ceph-mon[102928]: osdmap e340: 8 total, 8 up, 7 in 2024-09-13T15:01:26.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:25 smithi204 ceph-mon[87071]: pgmap v1124: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.4 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 25498/68154 objects degraded (37.412%); 153/68154 objects misplaced (0.224%); 5.7 MiB/s, 90 objects/s recovering 2024-09-13T15:01:26.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:25 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:26.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:25 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1198445108' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:26.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:25 smithi204 ceph-mon[87071]: osdmap e340: 8 total, 8 up, 7 in 2024-09-13T15:01:26.261 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:26.295 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:26.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:26.350 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:01:26.350 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:01:26.397 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:26.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.430 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:26.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:26.532 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:26.555 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:26.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:26.657 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:26.691 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:26.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:26.793 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:26.818 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:26.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:26.898 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T15:01:26.919 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:26.943 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:26.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:27.045 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:27.076 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:27.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:27.178 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:27.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.212 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:27.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:27.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:26 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1905885159' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:01:27.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:26 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:26 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1905885159' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:26 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:26 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1905885159' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:26 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:01:27.314 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:27.366 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:27.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:27.468 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:27.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.502 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:27.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:27.603 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:27.638 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:27.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:27.740 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:27.775 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:27.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:27.877 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:27.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.914 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:27.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:28.015 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:28.040 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:27 smithi204 ceph-mon[87071]: pgmap v1126: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 25283/65442 objects degraded (38.634%); 153/65442 objects misplaced (0.234%); 5.1 MiB/s, 81 objects/s recovering 2024-09-13T15:01:28.041 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:27 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:01:28.041 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:27 smithi204 ceph-mon[87071]: osdmap e341: 8 total, 8 up, 7 in 2024-09-13T15:01:28.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.051 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:28.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:28.084 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T15:01:28.145 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:27 smithi096 ceph-mon[98901]: pgmap v1126: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 25283/65442 objects degraded (38.634%); 153/65442 objects misplaced (0.234%); 5.1 MiB/s, 81 objects/s recovering 2024-09-13T15:01:28.145 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:27 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:01:28.145 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:27 smithi096 ceph-mon[98901]: osdmap e341: 8 total, 8 up, 7 in 2024-09-13T15:01:28.146 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:01:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:01:27] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.51.0" 2024-09-13T15:01:28.146 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:27 smithi096 ceph-mon[102928]: pgmap v1126: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 25283/65442 objects degraded (38.634%); 153/65442 objects misplaced (0.234%); 5.1 MiB/s, 81 objects/s recovering 2024-09-13T15:01:28.146 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:27 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:01:28.146 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:27 smithi096 ceph-mon[102928]: osdmap e341: 8 total, 8 up, 7 in 2024-09-13T15:01:28.148 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T15:01:28.152 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:28.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.221 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:28.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:28.322 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:28.360 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:28.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:28.461 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:28.515 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:28.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:28.585 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:28.617 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:28.655 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:28.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:28.692 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:01:28.693 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:01:28.715 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:28.733 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:01:28.733 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:01:28.756 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:28.765 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:01:28.765 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:01:28.792 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:01:28.793 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:01:28.794 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:28.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:28.823 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:01:28.823 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:01:28.853 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:01:28.853 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:01:28.879 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:01:28.879 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:01:28.893 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:01:28.895 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:28.912 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:01:28.912 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:01:28.962 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:28.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:29.036 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:28 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4176398172' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:29.036 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:28 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4176398172' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:29.053 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:28 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4176398172' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:29.063 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:29.098 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:29.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:29.200 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:29.231 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:29.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:29.333 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:29.367 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:29.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:29.469 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:29.503 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:29.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:29.605 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:29.644 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:29.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:29.746 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:29.783 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:29.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:29.886 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:29.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.925 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:29.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:29.992 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:29 smithi096 ceph-mon[98901]: pgmap v1128: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.2 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 24773/58767 objects degraded (42.155%); 3.8 MiB/s, 97 objects/s recovering 2024-09-13T15:01:29.992 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:29 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 24773/58767 objects degraded (42.155%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:29.992 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:29 smithi096 ceph-mon[102928]: pgmap v1128: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.2 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 24773/58767 objects degraded (42.155%); 3.8 MiB/s, 97 objects/s recovering 2024-09-13T15:01:29.992 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:29 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 24773/58767 objects degraded (42.155%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:30.026 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:30.065 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:30.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:30.107 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:29 smithi204 ceph-mon[87071]: pgmap v1128: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.2 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 24773/58767 objects degraded (42.155%); 3.8 MiB/s, 97 objects/s recovering 2024-09-13T15:01:30.107 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:29 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 24773/58767 objects degraded (42.155%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:30.167 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:30.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.202 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:30.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:30.303 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:30.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.341 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:30.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:30.415 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:01:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:01:30.215Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:01:30.441 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:30.471 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:30.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:30.573 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:30.575 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:01:30.575 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T15:01:30.575 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T15:01:30.575 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T15:01:30.575 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.575 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:113296 9.3 9:c73004b8:::benchmark_data_smithi096_256909_object113295:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-09-13T15:01:30.575 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:11.915762+0000", 2024-09-13T15:01:30.575 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 138.653021526, 2024-09-13T15:01:30.575 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.98619730999999999, 2024-09-13T15:01:30.575 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.576 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.576 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.576 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.576 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.576 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 113296 2024-09-13T15:01:30.576 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.576 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.576 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.576 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.576 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915762+0000", 2024-09-13T15:01:30.576 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.576 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.577 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.577 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.577 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915762+0000", 2024-09-13T15:01:30.577 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.577 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.577 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.577 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.577 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915760+0000", 2024-09-13T15:01:30.577 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:30.577 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.577 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.577 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.577 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915786+0000", 2024-09-13T15:01:30.578 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5636e-05 2024-09-13T15:01:30.578 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.578 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.578 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.578 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915801+0000", 2024-09-13T15:01:30.578 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5174000000000001e-05 2024-09-13T15:01:30.578 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.578 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.578 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.578 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915808+0000", 2024-09-13T15:01:30.578 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8379999999999996e-06 2024-09-13T15:01:30.578 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.579 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.579 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.579 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915849+0000", 2024-09-13T15:01:30.579 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1340000000000001e-05 2024-09-13T15:01:30.579 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.579 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.579 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.579 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915856+0000", 2024-09-13T15:01:30.579 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0770000000000002e-06 2024-09-13T15:01:30.580 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.580 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.580 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.580 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.888274+0000", 2024-09-13T15:01:30.580 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.972418381 2024-09-13T15:01:30.580 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.580 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.580 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.581 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.888439+0000", 2024-09-13T15:01:30.581 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000164365 2024-09-13T15:01:30.581 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.581 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.581 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.581 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.888633+0000", 2024-09-13T15:01:30.581 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000194286 2024-09-13T15:01:30.581 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.581 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.581 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.581 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.897978+0000", 2024-09-13T15:01:30.581 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0093450700000000005 2024-09-13T15:01:30.581 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.582 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.582 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.582 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.898169+0000", 2024-09-13T15:01:30.582 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00019047199999999999 2024-09-13T15:01:30.582 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.582 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.582 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.582 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.901811+0000", 2024-09-13T15:01:30.582 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003641886 2024-09-13T15:01:30.582 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.582 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.582 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.582 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.901858+0000", 2024-09-13T15:01:30.582 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7889999999999997e-05 2024-09-13T15:01:30.583 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.583 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.583 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.583 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.901959+0000", 2024-09-13T15:01:30.583 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010039099999999999 2024-09-13T15:01:30.583 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.583 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.583 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.583 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.583 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.583 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:113301 9.3 9:c3749cfb:::benchmark_data_smithi096_256909_object113300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-09-13T15:01:30.583 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:11.920727+0000", 2024-09-13T15:01:30.583 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 138.64805576399999, 2024-09-13T15:01:30.584 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99153909699999998, 2024-09-13T15:01:30.584 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.584 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.584 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.584 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.584 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.584 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 113301 2024-09-13T15:01:30.584 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.584 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.584 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.584 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.584 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920727+0000", 2024-09-13T15:01:30.584 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.584 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.585 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.585 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.585 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920727+0000", 2024-09-13T15:01:30.585 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.585 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.585 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.585 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.585 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920726+0000", 2024-09-13T15:01:30.585 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:01:30.585 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.585 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.585 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.585 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920767+0000", 2024-09-13T15:01:30.585 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1270000000000003e-05 2024-09-13T15:01:30.586 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.586 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.586 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.586 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920783+0000", 2024-09-13T15:01:30.586 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6081000000000001e-05 2024-09-13T15:01:30.586 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.586 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.586 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.586 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920790+0000", 2024-09-13T15:01:30.586 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1910000000000003e-06 2024-09-13T15:01:30.586 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.586 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.586 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.586 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920839+0000", 2024-09-13T15:01:30.587 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9172999999999998e-05 2024-09-13T15:01:30.587 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.587 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.587 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.587 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920853+0000", 2024-09-13T15:01:30.587 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4098e-05 2024-09-13T15:01:30.587 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.587 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.587 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.587 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.892743+0000", 2024-09-13T15:01:30.587 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97188978699999995 2024-09-13T15:01:30.587 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.587 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.588 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.588 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.892878+0000", 2024-09-13T15:01:30.588 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000134657 2024-09-13T15:01:30.588 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.588 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.588 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.588 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.893051+0000", 2024-09-13T15:01:30.588 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00017329199999999999 2024-09-13T15:01:30.588 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.588 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.588 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.589 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.898204+0000", 2024-09-13T15:01:30.589 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0051531320000000004 2024-09-13T15:01:30.589 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.589 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.589 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.589 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.901779+0000", 2024-09-13T15:01:30.589 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035747069999999999 2024-09-13T15:01:30.589 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.589 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.589 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.589 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.912119+0000", 2024-09-13T15:01:30.589 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010340293 2024-09-13T15:01:30.590 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.590 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.590 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.590 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.912170+0000", 2024-09-13T15:01:30.590 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0917000000000001e-05 2024-09-13T15:01:30.590 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.590 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.590 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.590 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.912266+0000", 2024-09-13T15:01:30.590 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6180000000000004e-05 2024-09-13T15:01:30.590 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.590 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.590 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.590 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.591 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.591 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142789 9.3 9:c5ee02be:::benchmark_data_smithi096_256909_object23381:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:01:30.591 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.187568+0000", 2024-09-13T15:01:30.591 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 100.381215567, 2024-09-13T15:01:30.591 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.458558638, 2024-09-13T15:01:30.591 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.591 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T15:01:30.591 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.591 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.591 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.591 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142789 2024-09-13T15:01:30.591 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.592 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.592 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.592 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.592 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.187568+0000", 2024-09-13T15:01:30.592 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.592 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.592 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.592 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.592 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.187568+0000", 2024-09-13T15:01:30.592 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.592 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.592 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.592 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.593 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.187566+0000", 2024-09-13T15:01:30.593 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:01:30.593 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.593 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.593 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.593 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.187571+0000", 2024-09-13T15:01:30.593 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0270000000000003e-06 2024-09-13T15:01:30.593 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.593 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.593 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.593 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.187583+0000", 2024-09-13T15:01:30.593 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1603999999999999e-05 2024-09-13T15:01:30.594 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.594 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.594 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.594 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.187589+0000", 2024-09-13T15:01:30.594 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3169999999999998e-06 2024-09-13T15:01:30.594 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.594 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.594 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.594 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.187747+0000", 2024-09-13T15:01:30.594 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000157529 2024-09-13T15:01:30.594 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.594 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.595 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.595 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.188591+0000", 2024-09-13T15:01:30.595 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00084404 2024-09-13T15:01:30.595 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.595 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.595 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.595 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.188709+0000", 2024-09-13T15:01:30.595 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011758500000000001 2024-09-13T15:01:30.595 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.595 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.595 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.595 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.198395+0000", 2024-09-13T15:01:30.595 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0096860690000000003 2024-09-13T15:01:30.596 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.596 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.596 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.596 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.198401+0000", 2024-09-13T15:01:30.596 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9210000000000003e-06 2024-09-13T15:01:30.596 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.596 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.596 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.596 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.603902+0000", 2024-09-13T15:01:30.596 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4055006640000001 2024-09-13T15:01:30.596 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.596 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.596 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for missing object", 2024-09-13T15:01:30.596 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.604030+0000", 2024-09-13T15:01:30.597 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012781500000000001 2024-09-13T15:01:30.597 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.597 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.597 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.597 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646040+0000", 2024-09-13T15:01:30.597 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.042009791999999997 2024-09-13T15:01:30.597 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.597 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.597 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.597 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646126+0000", 2024-09-13T15:01:30.597 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.6382000000000001e-05 2024-09-13T15:01:30.597 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.597 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.597 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.598 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.598 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.598 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142798 9.3 9:c534ab8f:::benchmark_data_smithi096_256909_object23390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:01:30.598 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.201181+0000", 2024-09-13T15:01:30.598 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 100.367601689, 2024-09-13T15:01:30.598 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4451804939999999, 2024-09-13T15:01:30.598 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.598 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.598 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.598 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.598 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.598 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142798 2024-09-13T15:01:30.598 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.598 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.599 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.599 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.599 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201181+0000", 2024-09-13T15:01:30.599 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.599 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.599 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.599 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.599 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201181+0000", 2024-09-13T15:01:30.599 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.599 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.599 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.599 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.599 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201179+0000", 2024-09-13T15:01:30.600 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:01:30.600 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.600 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.600 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.600 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201186+0000", 2024-09-13T15:01:30.600 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9009999999999997e-06 2024-09-13T15:01:30.600 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.600 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.600 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.600 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201204+0000", 2024-09-13T15:01:30.600 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8158000000000001e-05 2024-09-13T15:01:30.600 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.600 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.601 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.601 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201210+0000", 2024-09-13T15:01:30.601 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1210000000000003e-06 2024-09-13T15:01:30.601 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.601 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.601 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.601 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201461+0000", 2024-09-13T15:01:30.601 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00025028499999999999 2024-09-13T15:01:30.601 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.601 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.601 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.601 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201469+0000", 2024-09-13T15:01:30.601 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8669999999999994e-06 2024-09-13T15:01:30.601 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.602 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.602 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.602 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.604066+0000", 2024-09-13T15:01:30.602 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.402596897 2024-09-13T15:01:30.602 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.602 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.602 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.602 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.604204+0000", 2024-09-13T15:01:30.602 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013892900000000001 2024-09-13T15:01:30.602 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.602 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.602 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.603 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.604352+0000", 2024-09-13T15:01:30.603 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000147119 2024-09-13T15:01:30.603 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.603 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.603 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.603 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.616334+0000", 2024-09-13T15:01:30.603 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011981921 2024-09-13T15:01:30.603 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.603 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.603 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.603 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.645720+0000", 2024-09-13T15:01:30.603 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.029386374 2024-09-13T15:01:30.603 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.604 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.604 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.604 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646210+0000", 2024-09-13T15:01:30.604 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000489684 2024-09-13T15:01:30.604 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.604 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.604 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.604 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646260+0000", 2024-09-13T15:01:30.604 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9963999999999998e-05 2024-09-13T15:01:30.604 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.604 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.604 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.604 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646362+0000", 2024-09-13T15:01:30.606 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000102352 2024-09-13T15:01:30.606 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.606 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.606 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.606 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.606 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.606 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142804 9.3 9:c741bee0:::benchmark_data_smithi096_256909_object23396:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:01:30.607 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.203140+0000", 2024-09-13T15:01:30.607 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 100.365643439, 2024-09-13T15:01:30.607 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.443340676, 2024-09-13T15:01:30.607 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.607 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.607 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.607 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.607 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.607 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142804 2024-09-13T15:01:30.607 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.607 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.607 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.608 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.608 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203140+0000", 2024-09-13T15:01:30.608 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.608 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.608 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.608 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.608 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203140+0000", 2024-09-13T15:01:30.608 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.608 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.608 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.608 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.608 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203139+0000", 2024-09-13T15:01:30.608 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:01:30.608 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.609 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.609 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.609 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203143+0000", 2024-09-13T15:01:30.609 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.642e-06 2024-09-13T15:01:30.609 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.609 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.609 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.609 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203153+0000", 2024-09-13T15:01:30.609 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0023999999999999e-05 2024-09-13T15:01:30.609 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.609 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.609 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.610 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203157+0000", 2024-09-13T15:01:30.610 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1740000000000002e-06 2024-09-13T15:01:30.610 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.610 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.610 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.610 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203184+0000", 2024-09-13T15:01:30.610 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7155999999999999e-05 2024-09-13T15:01:30.611 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.611 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.611 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.611 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203188+0000", 2024-09-13T15:01:30.611 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6070000000000001e-06 2024-09-13T15:01:30.611 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.611 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.611 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.611 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.607722+0000", 2024-09-13T15:01:30.611 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.404533799 2024-09-13T15:01:30.611 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.611 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.611 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.612 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.607849+0000", 2024-09-13T15:01:30.612 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012717499999999999 2024-09-13T15:01:30.612 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.612 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.612 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.612 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.608000+0000", 2024-09-13T15:01:30.612 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015092 2024-09-13T15:01:30.612 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.612 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.612 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.612 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.621314+0000", 2024-09-13T15:01:30.612 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013313503000000001 2024-09-13T15:01:30.612 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.613 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.613 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.613 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646164+0000", 2024-09-13T15:01:30.613 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.024850395000000001 2024-09-13T15:01:30.613 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.613 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.613 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.613 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646400+0000", 2024-09-13T15:01:30.613 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00023617199999999999 2024-09-13T15:01:30.613 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.613 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.613 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.613 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646429+0000", 2024-09-13T15:01:30.614 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8323e-05 2024-09-13T15:01:30.614 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.614 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.614 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.614 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646480+0000", 2024-09-13T15:01:30.614 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1530999999999998e-05 2024-09-13T15:01:30.614 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.614 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.614 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.614 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.614 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.614 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142822 9.3 9:c7fbd599:::benchmark_data_smithi096_256909_object23414:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:01:30.614 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.232068+0000", 2024-09-13T15:01:30.614 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 100.336715345, 2024-09-13T15:01:30.615 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.414564594, 2024-09-13T15:01:30.615 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.615 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.615 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.615 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.615 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.615 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142822 2024-09-13T15:01:30.615 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.615 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.615 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.615 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.615 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232068+0000", 2024-09-13T15:01:30.615 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.616 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.616 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.616 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.616 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232068+0000", 2024-09-13T15:01:30.616 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.616 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.616 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.616 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.616 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232067+0000", 2024-09-13T15:01:30.616 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:30.616 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.616 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.617 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.617 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232071+0000", 2024-09-13T15:01:30.617 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4159999999999997e-06 2024-09-13T15:01:30.617 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.617 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.617 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.617 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232087+0000", 2024-09-13T15:01:30.617 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5184000000000001e-05 2024-09-13T15:01:30.617 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.617 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.617 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.617 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232092+0000", 2024-09-13T15:01:30.617 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6899999999999997e-06 2024-09-13T15:01:30.618 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.618 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.618 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.618 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232659+0000", 2024-09-13T15:01:30.618 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00056658499999999998 2024-09-13T15:01:30.618 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.618 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.618 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.618 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232667+0000", 2024-09-13T15:01:30.618 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6799999999999993e-06 2024-09-13T15:01:30.618 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.618 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.618 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.619 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.611147+0000", 2024-09-13T15:01:30.619 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3784807080000001 2024-09-13T15:01:30.619 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.619 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.619 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.619 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.611228+0000", 2024-09-13T15:01:30.619 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1085e-05 2024-09-13T15:01:30.619 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.619 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.619 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.619 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.611303+0000", 2024-09-13T15:01:30.619 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.4412000000000005e-05 2024-09-13T15:01:30.619 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.620 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.620 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.620 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.621322+0000", 2024-09-13T15:01:30.620 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010018872 2024-09-13T15:01:30.620 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.620 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.620 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.620 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646521+0000", 2024-09-13T15:01:30.620 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.025199784999999999 2024-09-13T15:01:30.620 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.620 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.620 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.620 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646538+0000", 2024-09-13T15:01:30.621 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6949999999999999e-05 2024-09-13T15:01:30.621 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.621 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.621 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.621 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646564+0000", 2024-09-13T15:01:30.621 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5083999999999999e-05 2024-09-13T15:01:30.621 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.621 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.621 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.621 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646632+0000", 2024-09-13T15:01:30.621 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8813000000000007e-05 2024-09-13T15:01:30.621 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.622 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.622 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.622 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.622 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.622 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142836 9.3 9:c3b86387:::benchmark_data_smithi096_256909_object23428:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:01:30.622 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.245149+0000", 2024-09-13T15:01:30.622 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 100.32363372899999, 2024-09-13T15:01:30.622 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4016361989999999, 2024-09-13T15:01:30.622 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.622 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.622 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.622 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.622 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.622 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142836 2024-09-13T15:01:30.623 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.623 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.623 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.623 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.623 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245149+0000", 2024-09-13T15:01:30.623 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.623 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.623 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.623 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.623 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245149+0000", 2024-09-13T15:01:30.623 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.623 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.623 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.623 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.624 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245148+0000", 2024-09-13T15:01:30.624 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:30.624 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.624 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.624 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.624 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245155+0000", 2024-09-13T15:01:30.624 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.187e-06 2024-09-13T15:01:30.624 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.624 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.624 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.624 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245167+0000", 2024-09-13T15:01:30.624 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2513e-05 2024-09-13T15:01:30.624 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.625 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.625 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.625 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245173+0000", 2024-09-13T15:01:30.625 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5890000000000002e-06 2024-09-13T15:01:30.625 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.625 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.625 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.625 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245241+0000", 2024-09-13T15:01:30.625 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8044000000000001e-05 2024-09-13T15:01:30.625 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.625 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.625 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.625 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245248+0000", 2024-09-13T15:01:30.626 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1439999999999997e-06 2024-09-13T15:01:30.626 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.626 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.626 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.626 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.613589+0000", 2024-09-13T15:01:30.626 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.36834093 2024-09-13T15:01:30.626 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.626 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.626 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.626 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.613714+0000", 2024-09-13T15:01:30.626 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012501299999999999 2024-09-13T15:01:30.626 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.626 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.626 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.627 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.613852+0000", 2024-09-13T15:01:30.627 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013756200000000001 2024-09-13T15:01:30.627 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.627 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.627 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.627 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.626398+0000", 2024-09-13T15:01:30.627 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012546639 2024-09-13T15:01:30.627 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.627 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.627 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.627 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646665+0000", 2024-09-13T15:01:30.627 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.020266998000000001 2024-09-13T15:01:30.627 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.628 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.628 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.628 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646688+0000", 2024-09-13T15:01:30.628 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2529999999999999e-05 2024-09-13T15:01:30.628 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.628 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.628 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.628 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646713+0000", 2024-09-13T15:01:30.628 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5193000000000001e-05 2024-09-13T15:01:30.628 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.628 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.628 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.628 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646786+0000", 2024-09-13T15:01:30.628 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2516999999999997e-05 2024-09-13T15:01:30.629 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.629 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.629 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.629 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.629 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.629 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142851 9.3 9:c9bb45f3:::benchmark_data_smithi096_256909_object23443:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:01:30.629 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.267504+0000", 2024-09-13T15:01:30.629 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 100.30127939, 2024-09-13T15:01:30.629 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.379523225, 2024-09-13T15:01:30.629 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.629 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.629 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.629 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.629 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.630 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142851 2024-09-13T15:01:30.630 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.630 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.630 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.630 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.630 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267504+0000", 2024-09-13T15:01:30.630 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.630 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.630 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.630 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.630 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267504+0000", 2024-09-13T15:01:30.630 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.630 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.630 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.631 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.631 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267502+0000", 2024-09-13T15:01:30.631 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:01:30.631 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.631 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.631 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.631 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267509+0000", 2024-09-13T15:01:30.631 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7800000000000003e-06 2024-09-13T15:01:30.631 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.631 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.631 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.631 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267522+0000", 2024-09-13T15:01:30.631 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2819e-05 2024-09-13T15:01:30.632 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.632 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.632 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.632 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267526+0000", 2024-09-13T15:01:30.632 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.887e-06 2024-09-13T15:01:30.632 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.632 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.632 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.632 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267574+0000", 2024-09-13T15:01:30.632 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7586000000000001e-05 2024-09-13T15:01:30.632 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.632 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.632 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.633 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267583+0000", 2024-09-13T15:01:30.633 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.9269999999999994e-06 2024-09-13T15:01:30.633 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.633 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.633 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.633 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.616374+0000", 2024-09-13T15:01:30.633 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3487907029999999 2024-09-13T15:01:30.633 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.633 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.633 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.633 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.616499+0000", 2024-09-13T15:01:30.633 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012532299999999999 2024-09-13T15:01:30.633 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.634 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.634 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.634 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.616633+0000", 2024-09-13T15:01:30.634 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013409899999999999 2024-09-13T15:01:30.634 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.634 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.634 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.634 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.626407+0000", 2024-09-13T15:01:30.634 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0097741200000000007 2024-09-13T15:01:30.634 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.634 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.634 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.634 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646812+0000", 2024-09-13T15:01:30.635 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.020404723999999999 2024-09-13T15:01:30.635 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.635 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.635 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.635 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646871+0000", 2024-09-13T15:01:30.635 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8733999999999997e-05 2024-09-13T15:01:30.635 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.635 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.635 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.635 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646913+0000", 2024-09-13T15:01:30.635 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1918000000000001e-05 2024-09-13T15:01:30.635 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.635 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.636 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.636 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647027+0000", 2024-09-13T15:01:30.636 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000114276 2024-09-13T15:01:30.636 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.636 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.636 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.636 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.636 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.636 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142853 9.3 9:ccfce356:::benchmark_data_smithi096_256909_object23445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:01:30.636 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.268877+0000", 2024-09-13T15:01:30.636 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 100.29990645399999, 2024-09-13T15:01:30.636 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.378334902, 2024-09-13T15:01:30.636 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.637 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.637 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.637 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.637 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.637 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142853 2024-09-13T15:01:30.637 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.637 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.637 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.637 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.637 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268877+0000", 2024-09-13T15:01:30.637 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.637 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.637 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.638 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.638 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268877+0000", 2024-09-13T15:01:30.638 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.638 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.638 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.638 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.638 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268876+0000", 2024-09-13T15:01:30.638 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:30.638 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.638 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.638 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.638 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268879+0000", 2024-09-13T15:01:30.638 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8609999999999998e-06 2024-09-13T15:01:30.638 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.639 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.639 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.639 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268884+0000", 2024-09-13T15:01:30.639 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5879999999999999e-06 2024-09-13T15:01:30.639 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.639 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.639 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.639 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268886+0000", 2024-09-13T15:01:30.639 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4729999999999999e-06 2024-09-13T15:01:30.639 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.639 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.639 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.639 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268918+0000", 2024-09-13T15:01:30.640 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2483000000000001e-05 2024-09-13T15:01:30.640 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.640 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.640 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.640 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268923+0000", 2024-09-13T15:01:30.640 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1450000000000001e-06 2024-09-13T15:01:30.640 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.640 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.640 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.640 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.618816+0000", 2024-09-13T15:01:30.640 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3498932770000001 2024-09-13T15:01:30.640 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.641 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.641 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.641 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.618914+0000", 2024-09-13T15:01:30.641 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.8220000000000005e-05 2024-09-13T15:01:30.641 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.641 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.641 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.641 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.618996+0000", 2024-09-13T15:01:30.641 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1409000000000006e-05 2024-09-13T15:01:30.641 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.641 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.642 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.642 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.632113+0000", 2024-09-13T15:01:30.642 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013117403 2024-09-13T15:01:30.642 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.642 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.642 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.642 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646842+0000", 2024-09-13T15:01:30.642 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.014728774 2024-09-13T15:01:30.642 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.642 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.642 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.642 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647095+0000", 2024-09-13T15:01:30.642 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00025298900000000001 2024-09-13T15:01:30.643 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.643 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.643 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.643 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647134+0000", 2024-09-13T15:01:30.643 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8890999999999997e-05 2024-09-13T15:01:30.643 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.643 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.643 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.643 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647212+0000", 2024-09-13T15:01:30.643 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7939e-05 2024-09-13T15:01:30.643 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.643 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.643 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.643 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.643 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.644 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142872 9.3 9:c83c4353:::benchmark_data_smithi096_256909_object23464:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:01:30.644 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.294230+0000", 2024-09-13T15:01:30.644 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 100.274553109, 2024-09-13T15:01:30.644 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.353197628, 2024-09-13T15:01:30.644 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.644 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.644 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.644 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.644 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.644 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142872 2024-09-13T15:01:30.644 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.644 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.644 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.644 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.644 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294230+0000", 2024-09-13T15:01:30.645 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.645 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.645 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.645 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.645 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294230+0000", 2024-09-13T15:01:30.645 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.645 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.645 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.645 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.645 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294228+0000", 2024-09-13T15:01:30.645 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:30.645 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.645 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294236+0000", 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1640000000000004e-06 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294250+0000", 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4419999999999999e-05 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294256+0000", 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6590000000000004e-06 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.646 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.647 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294304+0000", 2024-09-13T15:01:30.647 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7942999999999998e-05 2024-09-13T15:01:30.647 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.647 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.647 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.647 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294312+0000", 2024-09-13T15:01:30.647 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5690000000000005e-06 2024-09-13T15:01:30.647 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.647 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.647 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.647 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.621367+0000", 2024-09-13T15:01:30.647 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.327055071 2024-09-13T15:01:30.647 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.647 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.647 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.648 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.621487+0000", 2024-09-13T15:01:30.648 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119842 2024-09-13T15:01:30.648 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.648 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.648 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.648 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.621613+0000", 2024-09-13T15:01:30.648 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000125585 2024-09-13T15:01:30.648 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.648 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.648 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.648 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.632118+0000", 2024-09-13T15:01:30.648 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010505504000000001 2024-09-13T15:01:30.648 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.649 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.649 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.649 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647246+0000", 2024-09-13T15:01:30.649 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.015128252999999999 2024-09-13T15:01:30.649 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.649 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.649 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.649 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647305+0000", 2024-09-13T15:01:30.649 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8207e-05 2024-09-13T15:01:30.649 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.649 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.650 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.650 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647344+0000", 2024-09-13T15:01:30.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.650 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:30.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:30.651 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9419999999999999e-05 2024-09-13T15:01:30.651 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.652 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.652 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.652 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647428+0000", 2024-09-13T15:01:30.652 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3546000000000003e-05 2024-09-13T15:01:30.652 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.652 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.652 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.652 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.652 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.652 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142877 9.3 9:cb7a6d5e:::benchmark_data_smithi096_256909_object23469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:01:30.652 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.300920+0000", 2024-09-13T15:01:30.652 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 100.267862876, 2024-09-13T15:01:30.653 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3466536950000001, 2024-09-13T15:01:30.653 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.653 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.653 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.653 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.653 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.653 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142877 2024-09-13T15:01:30.653 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.653 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.653 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.653 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.653 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.300920+0000", 2024-09-13T15:01:30.653 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.653 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.654 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.654 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.654 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.300920+0000", 2024-09-13T15:01:30.654 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.654 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.654 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.654 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.654 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.300919+0000", 2024-09-13T15:01:30.654 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:30.654 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.654 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.654 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.654 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.300925+0000", 2024-09-13T15:01:30.654 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4690000000000002e-06 2024-09-13T15:01:30.654 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.655 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.655 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.655 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.300947+0000", 2024-09-13T15:01:30.655 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1826000000000001e-05 2024-09-13T15:01:30.655 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.655 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.655 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.655 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.300952+0000", 2024-09-13T15:01:30.655 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4199999999999998e-06 2024-09-13T15:01:30.655 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.655 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.655 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.655 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.301014+0000", 2024-09-13T15:01:30.655 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1659e-05 2024-09-13T15:01:30.656 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.656 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.656 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.656 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.301023+0000", 2024-09-13T15:01:30.656 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4470000000000006e-06 2024-09-13T15:01:30.656 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.656 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.656 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.656 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.623982+0000", 2024-09-13T15:01:30.656 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3229591359999999 2024-09-13T15:01:30.656 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.656 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.656 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.656 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.624062+0000", 2024-09-13T15:01:30.657 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0673000000000005e-05 2024-09-13T15:01:30.657 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.657 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.657 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.657 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.624150+0000", 2024-09-13T15:01:30.657 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7698e-05 2024-09-13T15:01:30.657 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.657 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.657 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.657 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.636952+0000", 2024-09-13T15:01:30.657 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012801619 2024-09-13T15:01:30.657 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.657 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.657 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.658 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647278+0000", 2024-09-13T15:01:30.658 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010326633 2024-09-13T15:01:30.658 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.658 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.658 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.658 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647461+0000", 2024-09-13T15:01:30.658 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018296100000000001 2024-09-13T15:01:30.658 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.658 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.658 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.658 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647497+0000", 2024-09-13T15:01:30.658 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6089e-05 2024-09-13T15:01:30.658 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.658 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.658 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.659 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647574+0000", 2024-09-13T15:01:30.659 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6555000000000001e-05 2024-09-13T15:01:30.659 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.659 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.659 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.659 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.659 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.659 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142896 9.3 9:cf57ea2d:::benchmark_data_smithi096_256909_object23488:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:01:30.659 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.329447+0000", 2024-09-13T15:01:30.659 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 100.239335846, 2024-09-13T15:01:30.659 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3183072769999999, 2024-09-13T15:01:30.659 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.659 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.659 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.660 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.660 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.660 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142896 2024-09-13T15:01:30.660 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.660 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.660 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.660 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.660 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329447+0000", 2024-09-13T15:01:30.660 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.660 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.660 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.660 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.660 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329447+0000", 2024-09-13T15:01:30.661 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.661 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.661 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.661 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.661 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329446+0000", 2024-09-13T15:01:30.661 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:30.661 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.661 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.661 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.661 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329453+0000", 2024-09-13T15:01:30.661 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6909999999999997e-06 2024-09-13T15:01:30.661 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.661 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.662 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.662 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329466+0000", 2024-09-13T15:01:30.662 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2835e-05 2024-09-13T15:01:30.662 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.662 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.662 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.662 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329471+0000", 2024-09-13T15:01:30.662 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0089999999999998e-06 2024-09-13T15:01:30.662 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.662 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.662 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.662 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329522+0000", 2024-09-13T15:01:30.662 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0895e-05 2024-09-13T15:01:30.663 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.663 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.663 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.663 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329529+0000", 2024-09-13T15:01:30.663 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.435e-06 2024-09-13T15:01:30.663 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.663 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.663 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.663 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.626453+0000", 2024-09-13T15:01:30.663 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.296923563 2024-09-13T15:01:30.663 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.663 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.663 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.664 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.626576+0000", 2024-09-13T15:01:30.664 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012282499999999999 2024-09-13T15:01:30.664 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.664 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.664 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.664 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.626781+0000", 2024-09-13T15:01:30.664 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00020488400000000001 2024-09-13T15:01:30.664 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.664 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.664 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.664 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.636955+0000", 2024-09-13T15:01:30.664 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010174325 2024-09-13T15:01:30.664 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.664 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.665 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.665 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647609+0000", 2024-09-13T15:01:30.665 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010653635 2024-09-13T15:01:30.665 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.665 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.665 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.665 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647637+0000", 2024-09-13T15:01:30.665 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8668999999999999e-05 2024-09-13T15:01:30.665 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.665 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.665 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.665 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647674+0000", 2024-09-13T15:01:30.665 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7128000000000001e-05 2024-09-13T15:01:30.666 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.666 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.666 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.666 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647755+0000", 2024-09-13T15:01:30.666 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0012000000000006e-05 2024-09-13T15:01:30.666 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.666 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.666 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.666 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.666 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.666 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142921 9.3 9:c414e2ce:::benchmark_data_smithi096_256909_object23513:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:01:30.666 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.368915+0000", 2024-09-13T15:01:30.666 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 100.19986838600001, 2024-09-13T15:01:30.666 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2790645789999999, 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142921 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.368915+0000", 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.667 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.368915+0000", 2024-09-13T15:01:30.668 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.668 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.668 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.668 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.668 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.368914+0000", 2024-09-13T15:01:30.668 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:01:30.668 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.668 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.668 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.668 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.368921+0000", 2024-09-13T15:01:30.668 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.3300000000000001e-06 2024-09-13T15:01:30.668 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.669 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.669 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.669 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.368941+0000", 2024-09-13T15:01:30.669 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0217e-05 2024-09-13T15:01:30.669 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.669 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.669 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.669 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.368947+0000", 2024-09-13T15:01:30.669 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7860000000000002e-06 2024-09-13T15:01:30.669 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.669 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.669 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.669 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.369000+0000", 2024-09-13T15:01:30.669 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2757999999999999e-05 2024-09-13T15:01:30.670 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.670 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.670 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.670 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.369008+0000", 2024-09-13T15:01:30.670 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0879999999999995e-06 2024-09-13T15:01:30.670 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.670 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.670 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.670 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.629551+0000", 2024-09-13T15:01:30.670 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2605434019999999 2024-09-13T15:01:30.670 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.670 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.670 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.671 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.629673+0000", 2024-09-13T15:01:30.671 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012139 2024-09-13T15:01:30.671 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.671 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.671 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.671 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.629802+0000", 2024-09-13T15:01:30.671 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000129729 2024-09-13T15:01:30.671 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.671 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.671 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.671 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.642703+0000", 2024-09-13T15:01:30.671 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012900688 2024-09-13T15:01:30.671 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.671 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.671 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.672 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647789+0000", 2024-09-13T15:01:30.672 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0050858270000000002 2024-09-13T15:01:30.672 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.672 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.672 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.672 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647858+0000", 2024-09-13T15:01:30.672 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9139000000000003e-05 2024-09-13T15:01:30.672 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.672 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.672 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.672 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647895+0000", 2024-09-13T15:01:30.672 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6597999999999998e-05 2024-09-13T15:01:30.672 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.672 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.672 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.673 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647979+0000", 2024-09-13T15:01:30.673 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.454e-05 2024-09-13T15:01:30.673 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.673 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.673 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.673 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.673 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.673 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142944 9.3 9:ca2419a8:::benchmark_data_smithi096_256909_object23536:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:01:30.673 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.407675+0000", 2024-09-13T15:01:30.673 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 100.161107648, 2024-09-13T15:01:30.673 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.240453279, 2024-09-13T15:01:30.673 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.673 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.673 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.674 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.674 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.674 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142944 2024-09-13T15:01:30.674 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.674 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.674 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.674 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.674 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407675+0000", 2024-09-13T15:01:30.674 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.674 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.674 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.674 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.674 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407675+0000", 2024-09-13T15:01:30.674 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.674 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.675 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.675 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.675 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407675+0000", 2024-09-13T15:01:30.675 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:30.675 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.675 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.675 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.675 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407678+0000", 2024-09-13T15:01:30.675 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1350000000000001e-06 2024-09-13T15:01:30.675 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.675 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.675 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.675 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407683+0000", 2024-09-13T15:01:30.675 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4939999999999997e-06 2024-09-13T15:01:30.675 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.676 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.676 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.676 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407687+0000", 2024-09-13T15:01:30.676 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.014e-06 2024-09-13T15:01:30.676 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.676 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.676 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.676 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407738+0000", 2024-09-13T15:01:30.676 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1585999999999997e-05 2024-09-13T15:01:30.676 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.676 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.676 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.676 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407745+0000", 2024-09-13T15:01:30.676 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1169999999999998e-06 2024-09-13T15:01:30.677 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.677 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.677 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.677 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.632152+0000", 2024-09-13T15:01:30.677 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2244062090000001 2024-09-13T15:01:30.677 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.677 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.677 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.677 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.632227+0000", 2024-09-13T15:01:30.677 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5765000000000004e-05 2024-09-13T15:01:30.677 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.677 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.677 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.677 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.632313+0000", 2024-09-13T15:01:30.678 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5833000000000003e-05 2024-09-13T15:01:30.678 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.678 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.678 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.678 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.642711+0000", 2024-09-13T15:01:30.678 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010397805 2024-09-13T15:01:30.678 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.678 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.678 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.678 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647828+0000", 2024-09-13T15:01:30.678 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0051175079999999998 2024-09-13T15:01:30.678 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.678 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.678 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.679 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648011+0000", 2024-09-13T15:01:30.679 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018260600000000001 2024-09-13T15:01:30.679 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.679 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.679 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.679 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648047+0000", 2024-09-13T15:01:30.679 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5902999999999998e-05 2024-09-13T15:01:30.679 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.679 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.680 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.680 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648129+0000", 2024-09-13T15:01:30.680 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1773000000000005e-05 2024-09-13T15:01:30.680 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.680 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.680 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.680 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.680 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.680 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142964 9.3 9:c3b98c18:::benchmark_data_smithi096_256909_object23556:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:01:30.680 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.448630+0000", 2024-09-13T15:01:30.680 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 100.12015341199999, 2024-09-13T15:01:30.680 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.199687605, 2024-09-13T15:01:30.681 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.681 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.681 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.681 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.681 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.681 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142964 2024-09-13T15:01:30.681 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.681 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.681 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.681 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448630+0000", 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448630+0000", 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448629+0000", 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.683 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448637+0000", 2024-09-13T15:01:30.683 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4489999999999999e-06 2024-09-13T15:01:30.683 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.683 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.683 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.683 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448647+0000", 2024-09-13T15:01:30.683 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7040000000000006e-06 2024-09-13T15:01:30.683 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.683 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.683 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.683 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448653+0000", 2024-09-13T15:01:30.683 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6629999999999998e-06 2024-09-13T15:01:30.683 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.683 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.684 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.684 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448699+0000", 2024-09-13T15:01:30.684 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6468999999999998e-05 2024-09-13T15:01:30.684 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.684 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.684 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.684 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448707+0000", 2024-09-13T15:01:30.684 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8939999999999993e-06 2024-09-13T15:01:30.684 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.684 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.684 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.684 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.634596+0000", 2024-09-13T15:01:30.684 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.185889134 2024-09-13T15:01:30.685 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.685 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.685 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.685 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.634677+0000", 2024-09-13T15:01:30.685 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1176999999999999e-05 2024-09-13T15:01:30.685 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.685 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.685 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.685 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.634764+0000", 2024-09-13T15:01:30.685 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7028999999999996e-05 2024-09-13T15:01:30.685 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.685 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.686 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.686 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.645444+0000", 2024-09-13T15:01:30.686 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010679711 2024-09-13T15:01:30.686 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.686 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.686 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.686 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648162+0000", 2024-09-13T15:01:30.686 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0027181900000000001 2024-09-13T15:01:30.686 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.686 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.686 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.686 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648189+0000", 2024-09-13T15:01:30.686 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7257999999999999e-05 2024-09-13T15:01:30.687 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.687 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.687 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.687 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648228+0000", 2024-09-13T15:01:30.687 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8260000000000003e-05 2024-09-13T15:01:30.687 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.687 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.687 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.687 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648317+0000", 2024-09-13T15:01:30.687 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.9621000000000001e-05 2024-09-13T15:01:30.687 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.687 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.687 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.688 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.688 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.688 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142987 9.3 9:c82bbcac:::benchmark_data_smithi096_256909_object23579:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:01:30.688 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.519042+0000", 2024-09-13T15:01:30.688 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 100.04974131900001, 2024-09-13T15:01:30.688 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1294554480000001, 2024-09-13T15:01:30.688 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.688 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.688 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.688 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.688 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.688 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142987 2024-09-13T15:01:30.688 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.689 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.689 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.689 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.689 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519042+0000", 2024-09-13T15:01:30.689 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.689 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.689 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.689 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.689 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519042+0000", 2024-09-13T15:01:30.689 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.689 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.689 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.689 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.690 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519040+0000", 2024-09-13T15:01:30.690 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:30.690 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.690 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.690 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.690 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519046+0000", 2024-09-13T15:01:30.690 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9780000000000002e-06 2024-09-13T15:01:30.690 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.690 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.690 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.690 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519059+0000", 2024-09-13T15:01:30.690 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2181000000000001e-05 2024-09-13T15:01:30.690 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.691 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.691 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.691 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519064+0000", 2024-09-13T15:01:30.691 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5799999999999999e-06 2024-09-13T15:01:30.691 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.691 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.691 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.691 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519127+0000", 2024-09-13T15:01:30.691 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2706999999999994e-05 2024-09-13T15:01:30.691 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.691 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.691 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.691 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519137+0000", 2024-09-13T15:01:30.692 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7170000000000003e-06 2024-09-13T15:01:30.692 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.692 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.692 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.692 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.636985+0000", 2024-09-13T15:01:30.692 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.11784839 2024-09-13T15:01:30.692 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.692 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.692 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.692 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.637113+0000", 2024-09-13T15:01:30.692 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012767599999999999 2024-09-13T15:01:30.692 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.692 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.693 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.693 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.637249+0000", 2024-09-13T15:01:30.693 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000136668 2024-09-13T15:01:30.693 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.693 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.693 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.693 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.645451+0000", 2024-09-13T15:01:30.693 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0082014380000000001 2024-09-13T15:01:30.693 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.693 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.693 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.693 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648351+0000", 2024-09-13T15:01:30.693 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002900026 2024-09-13T15:01:30.693 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.693 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.694 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.694 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648380+0000", 2024-09-13T15:01:30.694 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8717e-05 2024-09-13T15:01:30.694 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.694 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.694 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.694 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648414+0000", 2024-09-13T15:01:30.694 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4921e-05 2024-09-13T15:01:30.694 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.694 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.694 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.694 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648497+0000", 2024-09-13T15:01:30.694 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2725999999999994e-05 2024-09-13T15:01:30.694 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.694 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.695 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.695 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.695 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.695 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:143035 9.3 9:c1b6d37f:::benchmark_data_smithi096_256909_object23627:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:01:30.695 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.658945+0000", 2024-09-13T15:01:30.695 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 99.909838390999994, 2024-09-13T15:01:30.695 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.98972997100000004, 2024-09-13T15:01:30.695 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.695 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.695 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.695 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.695 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.695 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 143035 2024-09-13T15:01:30.696 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.696 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.696 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.696 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.696 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.658945+0000", 2024-09-13T15:01:30.696 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.696 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.696 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.696 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.696 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.658945+0000", 2024-09-13T15:01:30.696 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.696 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.696 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.697 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.697 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.658943+0000", 2024-09-13T15:01:30.697 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:01:30.697 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.697 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.697 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.697 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.658951+0000", 2024-09-13T15:01:30.697 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9889999999999992e-06 2024-09-13T15:01:30.697 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.697 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.697 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.697 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.658964+0000", 2024-09-13T15:01:30.697 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2819999999999999e-05 2024-09-13T15:01:30.698 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.698 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.698 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.698 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.658969+0000", 2024-09-13T15:01:30.698 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2510000000000002e-06 2024-09-13T15:01:30.698 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.698 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.698 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.698 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.659017+0000", 2024-09-13T15:01:30.698 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7599999999999998e-05 2024-09-13T15:01:30.698 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.698 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.698 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.699 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.659023+0000", 2024-09-13T15:01:30.699 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9079999999999998e-06 2024-09-13T15:01:30.699 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.699 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.699 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.699 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.640211+0000", 2024-09-13T15:01:30.699 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.98118727900000002 2024-09-13T15:01:30.699 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.699 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.699 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.699 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.640301+0000", 2024-09-13T15:01:30.699 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.0450000000000003e-05 2024-09-13T15:01:30.699 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.700 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.700 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.700 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.640380+0000", 2024-09-13T15:01:30.700 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9170000000000003e-05 2024-09-13T15:01:30.700 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.700 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.700 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.700 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.645658+0000", 2024-09-13T15:01:30.700 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0052772649999999997 2024-09-13T15:01:30.700 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.700 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.700 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.700 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648530+0000", 2024-09-13T15:01:30.701 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0028719829999999998 2024-09-13T15:01:30.701 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.701 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.701 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.701 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648558+0000", 2024-09-13T15:01:30.701 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8326e-05 2024-09-13T15:01:30.701 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.701 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.701 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.701 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648594+0000", 2024-09-13T15:01:30.701 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6508999999999999e-05 2024-09-13T15:01:30.701 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.701 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.702 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.702 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648675+0000", 2024-09-13T15:01:30.702 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0189000000000001e-05 2024-09-13T15:01:30.702 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.702 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.702 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.702 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.702 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.702 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:143100 9.3 9:c04321e8:::benchmark_data_smithi096_256909_object23692:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:01:30.702 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:51.036355+0000", 2024-09-13T15:01:30.702 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 99.532428373000002, 2024-09-13T15:01:30.702 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.61255426899999998, 2024-09-13T15:01:30.702 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.703 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.703 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.703 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.703 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.703 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 143100 2024-09-13T15:01:30.703 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.703 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.703 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.703 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.703 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036355+0000", 2024-09-13T15:01:30.703 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.703 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.703 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.703 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.704 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036355+0000", 2024-09-13T15:01:30.704 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.704 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.704 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.704 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.704 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036352+0000", 2024-09-13T15:01:30.704 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999976 2024-09-13T15:01:30.704 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.704 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.704 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.704 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036363+0000", 2024-09-13T15:01:30.704 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0889e-05 2024-09-13T15:01:30.704 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.704 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.705 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.705 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036385+0000", 2024-09-13T15:01:30.705 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1515e-05 2024-09-13T15:01:30.705 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.705 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.705 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.705 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036395+0000", 2024-09-13T15:01:30.705 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0142e-05 2024-09-13T15:01:30.705 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.705 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.705 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.705 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036441+0000", 2024-09-13T15:01:30.705 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5670000000000002e-05 2024-09-13T15:01:30.705 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.706 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.706 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:01:30.706 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036449+0000", 2024-09-13T15:01:30.706 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4719999999999995e-06 2024-09-13T15:01:30.706 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.706 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.706 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.706 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.642757+0000", 2024-09-13T15:01:30.706 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.60630773500000001 2024-09-13T15:01:30.706 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.706 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.706 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.706 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.642878+0000", 2024-09-13T15:01:30.706 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012154099999999999 2024-09-13T15:01:30.706 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.707 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.707 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.707 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.643019+0000", 2024-09-13T15:01:30.707 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014106700000000001 2024-09-13T15:01:30.707 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.707 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.707 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.707 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.645995+0000", 2024-09-13T15:01:30.707 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029752060000000002 2024-09-13T15:01:30.707 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.707 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.707 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.707 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648705+0000", 2024-09-13T15:01:30.707 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0027105470000000002 2024-09-13T15:01:30.708 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.708 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.708 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:30.708 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.708 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648796+0000", 2024-09-13T15:01:30.709 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1189999999999999e-05 2024-09-13T15:01:30.709 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.709 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.709 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.709 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648832+0000", 2024-09-13T15:01:30.709 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5352000000000003e-05 2024-09-13T15:01:30.709 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.709 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.709 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.709 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648909+0000", 2024-09-13T15:01:30.709 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7176999999999996e-05 2024-09-13T15:01:30.709 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.709 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.710 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.710 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.710 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.710 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:180718 9.d 9:b5ba0b3d:::benchmark_data_smithi096_256909_object61310:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T15:01:30.710 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:35.692709+0000", 2024-09-13T15:01:30.710 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 54.876074013999997, 2024-09-13T15:01:30.710 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.14518642100000001, 2024-09-13T15:01:30.710 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.710 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.711 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.711 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.711 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.711 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 180718 2024-09-13T15:01:30.711 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.711 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.711 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.711 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.711 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.692709+0000", 2024-09-13T15:01:30.711 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.711 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.711 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.711 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.711 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.692709+0000", 2024-09-13T15:01:30.712 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.712 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.712 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.712 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.712 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.692708+0000", 2024-09-13T15:01:30.712 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:30.712 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.712 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.712 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.712 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.692712+0000", 2024-09-13T15:01:30.712 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5290000000000002e-06 2024-09-13T15:01:30.712 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.712 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.712 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.713 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.692722+0000", 2024-09-13T15:01:30.713 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.5119999999999997e-06 2024-09-13T15:01:30.713 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.713 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.713 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.713 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.692726+0000", 2024-09-13T15:01:30.713 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3069999999999996e-06 2024-09-13T15:01:30.713 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.713 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.713 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.713 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.693079+0000", 2024-09-13T15:01:30.713 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00035255799999999999 2024-09-13T15:01:30.713 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.713 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.714 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.714 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.693175+0000", 2024-09-13T15:01:30.714 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6409000000000005e-05 2024-09-13T15:01:30.714 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.714 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.714 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.714 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.695269+0000", 2024-09-13T15:01:30.714 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0020938660000000002 2024-09-13T15:01:30.714 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.714 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.714 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.714 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.705161+0000", 2024-09-13T15:01:30.714 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0098916820000000006 2024-09-13T15:01:30.714 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.714 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.715 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.715 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.705982+0000", 2024-09-13T15:01:30.715 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00082113100000000001 2024-09-13T15:01:30.715 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.715 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.715 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.715 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.837809+0000", 2024-09-13T15:01:30.715 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.131827102 2024-09-13T15:01:30.715 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.715 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.715 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.715 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.837856+0000", 2024-09-13T15:01:30.715 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7113000000000001e-05 2024-09-13T15:01:30.715 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.716 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.716 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.716 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.837895+0000", 2024-09-13T15:01:30.716 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9601000000000003e-05 2024-09-13T15:01:30.716 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.716 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.716 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.716 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.716 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.716 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:180724 9.d 9:bc715091:::benchmark_data_smithi096_256909_object61316:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T15:01:30.716 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:00:35.695181+0000", 2024-09-13T15:01:30.716 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 54.873602403, 2024-09-13T15:01:30.716 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.14281270900000001, 2024-09-13T15:01:30.717 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:01:30.717 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:30.717 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:01:30.717 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:01:30.717 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:01:30.717 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 180724 2024-09-13T15:01:30.717 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.717 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:01:30.717 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.717 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:01:30.717 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.695181+0000", 2024-09-13T15:01:30.717 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.717 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.718 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.718 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:01:30.718 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.695181+0000", 2024-09-13T15:01:30.718 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:01:30.718 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.718 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.718 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:01:30.718 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.695179+0000", 2024-09-13T15:01:30.718 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:30.718 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.718 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.719 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:01:30.719 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.695185+0000", 2024-09-13T15:01:30.719 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.049e-06 2024-09-13T15:01:30.719 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.719 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.719 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:01:30.719 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.695195+0000", 2024-09-13T15:01:30.719 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.4590000000000004e-06 2024-09-13T15:01:30.719 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.719 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.719 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:01:30.719 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.695200+0000", 2024-09-13T15:01:30.719 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6029999999999998e-06 2024-09-13T15:01:30.719 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.720 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.720 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:01:30.720 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.699845+0000", 2024-09-13T15:01:30.720 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00464511 2024-09-13T15:01:30.720 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.720 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.720 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:01:30.720 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.699968+0000", 2024-09-13T15:01:30.720 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012326700000000001 2024-09-13T15:01:30.720 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.720 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.720 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:30.721 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.700082+0000", 2024-09-13T15:01:30.721 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000113676 2024-09-13T15:01:30.721 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.721 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.721 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.721 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.705975+0000", 2024-09-13T15:01:30.721 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0058929819999999997 2024-09-13T15:01:30.721 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.721 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.721 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:01:30.721 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.705986+0000", 2024-09-13T15:01:30.721 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.116e-05 2024-09-13T15:01:30.721 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.721 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.722 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:30.722 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.837924+0000", 2024-09-13T15:01:30.722 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.131938526 2024-09-13T15:01:30.722 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.722 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.722 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:01:30.722 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.837965+0000", 2024-09-13T15:01:30.722 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0691000000000001e-05 2024-09-13T15:01:30.722 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:01:30.722 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:01:30.722 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:01:30.722 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:00:35.837993+0000", 2024-09-13T15:01:30.722 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8413999999999999e-05 2024-09-13T15:01:30.722 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.723 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.723 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.723 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:01:30.723 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:01:30.723 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:01:30.751 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:30.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.799 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:30.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:30.900 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:30.928 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:30.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:31.030 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:31.060 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:31.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:31.162 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:31.195 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:31.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:31.297 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:31.334 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:31.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:31.436 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:31.470 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:31.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:31.572 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:31.603 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:31.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:31.705 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:31.736 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:31.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:31.753 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:01:31.838 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:31.876 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:31.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:31.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:31 smithi204 ceph-mon[87071]: pgmap v1129: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.1 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 936 op/s; 24264/53316 objects degraded (45.510%); 3.8 MiB/s, 97 objects/s recovering 2024-09-13T15:01:31.978 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:32.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.011 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:32.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:32.112 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:32.139 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:32.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:32.233 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:31 smithi096 ceph-mon[98901]: pgmap v1129: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.1 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 936 op/s; 24264/53316 objects degraded (45.510%); 3.8 MiB/s, 97 objects/s recovering 2024-09-13T15:01:32.234 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:31 smithi096 ceph-mon[102928]: pgmap v1129: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.1 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 936 op/s; 24264/53316 objects degraded (45.510%); 3.8 MiB/s, 97 objects/s recovering 2024-09-13T15:01:32.242 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:32.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.264 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:32.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:32.365 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:32.386 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:32.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:32.487 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:32.519 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:32.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:32.620 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:32.653 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:32.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:32.754 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:32.784 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:32.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:32.886 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:32.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.922 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:32.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:32.941 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T15:01:33.023 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:33.061 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:33.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:33.163 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:33.167 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:32 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3283154940' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:01:33.168 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:32 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:01:33.168 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:32 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3283154940' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:01:33.168 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:32 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:01:33.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.200 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:33.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:33.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:32 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3283154940' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:01:33.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:32 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:01:33.301 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:33.333 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:33.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:33.435 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:33.479 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:33.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:33.581 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:33.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.620 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:33.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:33.721 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:33.743 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:33.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:33.844 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:33.878 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:33.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:33.934 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T15:01:33.957 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T15:01:33.979 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:34.035 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:34.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:34.136 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:34.155 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:34.188 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:33 smithi204 ceph-mon[87071]: pgmap v1130: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.1 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 24264/53316 objects degraded (45.510%); 1.2 MiB/s, 55 objects/s recovering 2024-09-13T15:01:34.188 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:33 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:01:34.188 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:33 smithi204 ceph-mon[87071]: osdmap e342: 8 total, 8 up, 7 in 2024-09-13T15:01:34.188 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:33 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3631081188' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:01:34.189 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T15:01:34.196 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:34.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:34.231 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:33 smithi096 ceph-mon[98901]: pgmap v1130: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.1 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 24264/53316 objects degraded (45.510%); 1.2 MiB/s, 55 objects/s recovering 2024-09-13T15:01:34.231 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:33 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:01:34.231 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:33 smithi096 ceph-mon[98901]: osdmap e342: 8 total, 8 up, 7 in 2024-09-13T15:01:34.231 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:33 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3631081188' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:01:34.231 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:33 smithi096 ceph-mon[102928]: pgmap v1130: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.1 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 24264/53316 objects degraded (45.510%); 1.2 MiB/s, 55 objects/s recovering 2024-09-13T15:01:34.231 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:33 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:01:34.232 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:33 smithi096 ceph-mon[102928]: osdmap e342: 8 total, 8 up, 7 in 2024-09-13T15:01:34.232 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:33 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3631081188' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:01:34.298 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:34.336 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:34.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:34.438 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:34.472 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:34.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:34.574 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:34.605 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:34.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:34.707 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:34.751 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:34.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:34.853 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:34.887 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:34.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:34.989 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:35.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.035 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:35.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:35.136 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:35.167 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:35.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:35.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:34 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:35.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:34 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3631081188' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:01:35.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:34 smithi204 ceph-mon[87071]: osdmap e343: 8 total, 8 up, 7 in 2024-09-13T15:01:35.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:34 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 24264/53316 objects degraded (45.510%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:35.269 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:35.290 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:34 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:35.290 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:34 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3631081188' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:01:35.291 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:34 smithi096 ceph-mon[98901]: osdmap e343: 8 total, 8 up, 7 in 2024-09-13T15:01:35.291 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:34 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 24264/53316 objects degraded (45.510%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:35.291 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:34 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:35.291 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:34 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3631081188' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:01:35.291 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:34 smithi096 ceph-mon[102928]: osdmap e343: 8 total, 8 up, 7 in 2024-09-13T15:01:35.291 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:34 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 24264/53316 objects degraded (45.510%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:35.303 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:35.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:35.405 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:35.440 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:35.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:35.542 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:35.583 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:35.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:35.684 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:35.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.724 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:35.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:35.825 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:35.855 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:35.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:35.957 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:35.970 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:35.996 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:35.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:36.098 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:36.130 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:36.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:35 smithi204 ceph-mon[87071]: pgmap v1133: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 916 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 993 op/s; 23300/43821 objects degraded (53.171%); 1.3 MiB/s, 122 objects/s recovering 2024-09-13T15:01:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:36.232 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:36.267 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:36.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:35 smithi096 ceph-mon[98901]: pgmap v1133: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 916 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 993 op/s; 23300/43821 objects degraded (53.171%); 1.3 MiB/s, 122 objects/s recovering 2024-09-13T15:01:36.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:35 smithi096 ceph-mon[102928]: pgmap v1133: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 916 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 993 op/s; 23300/43821 objects degraded (53.171%); 1.3 MiB/s, 122 objects/s recovering 2024-09-13T15:01:36.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:36.369 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:36.397 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:36.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:36.500 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:36.530 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:36.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:36.632 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:36.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.668 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:36.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:36.769 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:36.808 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:36.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:36.910 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:36.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.942 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:36.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:36.999 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:36 smithi096 ceph-mon[98901]: pgmap v1134: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 850 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 23044/40662 objects degraded (56.672%); 0 B/s, 55 objects/s recovering 2024-09-13T15:01:37.000 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:36 smithi096 ceph-mon[102928]: pgmap v1134: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 850 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 23044/40662 objects degraded (56.672%); 0 B/s, 55 objects/s recovering 2024-09-13T15:01:37.043 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:37.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.072 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:37.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:37.089 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:01:37.173 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:36 smithi204 ceph-mon[87071]: pgmap v1134: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 850 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 23044/40662 objects degraded (56.672%); 0 B/s, 55 objects/s recovering 2024-09-13T15:01:37.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.234 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:37.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:37.335 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:37.366 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:37.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:37.468 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:37.507 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:37.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:37.608 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:37.647 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:37.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:37.749 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:37.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.785 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:37.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:37.886 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:37.918 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:37.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:38.020 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:38.069 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:38.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:38.171 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:38.199 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:38.205 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:38.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:38.254 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:01:38.254 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:01:38.255 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:01:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:01:37] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.51.0" 2024-09-13T15:01:38.286 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:01:38.286 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:01:38.307 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:38.314 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:01:38.314 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:01:38.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.343 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:38.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:38.343 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:01:38.343 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:01:38.373 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:01:38.373 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:01:38.405 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:01:38.405 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:01:38.438 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:01:38.438 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:01:38.444 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:38.469 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:01:38.469 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:01:38.473 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:38.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:38.575 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:38.611 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:38.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:38.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:38.748 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:38.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:38.850 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:38.889 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:38.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:38.991 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:39.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.022 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:39.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:39.104 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:01:39.105 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:01:39.122 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:39.163 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:39.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:39.265 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:39.301 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:39.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:39.403 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:39.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.429 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:39.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:39.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:39 smithi204 ceph-mon[87071]: pgmap v1135: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 694 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 837 op/s; 22113/33186 objects degraded (66.634%); 0 B/s, 137 objects/s recovering 2024-09-13T15:01:39.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:39 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2145422943' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:39.530 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:39.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.565 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:39.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:39.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:39 smithi096 ceph-mon[98901]: pgmap v1135: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 694 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 837 op/s; 22113/33186 objects degraded (66.634%); 0 B/s, 137 objects/s recovering 2024-09-13T15:01:39.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:39 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2145422943' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:39.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:39 smithi096 ceph-mon[102928]: pgmap v1135: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 694 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 837 op/s; 22113/33186 objects degraded (66.634%); 0 B/s, 137 objects/s recovering 2024-09-13T15:01:39.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:39 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2145422943' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:39.666 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:39.698 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:39.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:39.799 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:39.830 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:39.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:39.932 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:39.955 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:39.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:40.057 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:40.080 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:40.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:40.183 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:40.204 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:40.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:40.214 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 15:01:39 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a[74335]: logger=cleanup t=2024-09-13T15:01:39.893816776Z level=info msg="Completed cleanup jobs" duration=47.094737ms 2024-09-13T15:01:40.306 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:40.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:40.431 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:40.464 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:40.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:40 smithi204 ceph-mon[87071]: osdmap e344: 8 total, 8 up, 7 in 2024-09-13T15:01:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:40 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 22113/33186 objects degraded (66.634%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:40 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:01:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:01:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:01:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:01:40.566 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:40.577 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:40 smithi096 ceph-mon[98901]: osdmap e344: 8 total, 8 up, 7 in 2024-09-13T15:01:40.577 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:40 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 22113/33186 objects degraded (66.634%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:40.577 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:40 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:01:40.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:40 smithi096 ceph-mon[102928]: osdmap e344: 8 total, 8 up, 7 in 2024-09-13T15:01:40.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:40 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 22113/33186 objects degraded (66.634%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:40.577 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:40 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:01:40.600 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:40.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:40.702 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:40.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.738 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:40.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:40.839 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:40.873 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:40.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:40.974 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:41.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.020 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:41.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:41.121 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:41.155 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:41.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:41.257 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:41.291 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:41.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:41.393 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:41.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.431 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:41.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:41.481 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:41 smithi204 ceph-mon[87071]: pgmap v1137: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 575 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 21578/27474 objects degraded (78.540%); 0 B/s, 90 objects/s recovering 2024-09-13T15:01:41.481 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:41 smithi204 ceph-mon[87071]: osdmap e345: 8 total, 8 up, 7 in 2024-09-13T15:01:41.532 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:41.563 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:41.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:41.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:41 smithi096 ceph-mon[98901]: pgmap v1137: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 575 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 21578/27474 objects degraded (78.540%); 0 B/s, 90 objects/s recovering 2024-09-13T15:01:41.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:41 smithi096 ceph-mon[98901]: osdmap e345: 8 total, 8 up, 7 in 2024-09-13T15:01:41.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:41 smithi096 ceph-mon[102928]: pgmap v1137: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 575 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 21578/27474 objects degraded (78.540%); 0 B/s, 90 objects/s recovering 2024-09-13T15:01:41.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:41 smithi096 ceph-mon[102928]: osdmap e345: 8 total, 8 up, 7 in 2024-09-13T15:01:41.665 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:41.700 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:41.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:41.802 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:41.836 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:41.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:41.938 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:41.999 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:42.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:42.101 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:42.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.130 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:42.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:42.230 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:42.263 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:42.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:42.365 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:42.400 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:42.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:42.502 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:42.535 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:42.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:42.636 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:42.669 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:42.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:42.771 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:42.809 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:42.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:42.911 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:42.940 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:42.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:43.042 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:43.075 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:43.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:43.177 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:43.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.206 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:43.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:43.307 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:43.346 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:43.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:43.448 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:43.492 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:43.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:43.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:43 smithi096 ceph-mon[98901]: pgmap v1139: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 575 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 21578/27474 objects degraded (78.540%); 0 B/s, 81 objects/s recovering 2024-09-13T15:01:43.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:43 smithi096 ceph-mon[102928]: pgmap v1139: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 575 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 21578/27474 objects degraded (78.540%); 0 B/s, 81 objects/s recovering 2024-09-13T15:01:43.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:43.626 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:43.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:43.637 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:43 smithi204 ceph-mon[87071]: pgmap v1139: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 575 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 21578/27474 objects degraded (78.540%); 0 B/s, 81 objects/s recovering 2024-09-13T15:01:43.728 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:43.765 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:43.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:43.867 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:43.899 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:43.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:44.001 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:44.054 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:44.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:44.131 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:44.156 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:44.175 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T15:01:44.183 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:44.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:44.285 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:44.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.317 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:44.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:44.419 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:44.458 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:44.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:44.560 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:44.597 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:44.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:44.699 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:44.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.735 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:44.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:44.836 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:44.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.868 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:44.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:44.969 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:45.007 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:45.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:45.109 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:45.147 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:45.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:45.248 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:45.299 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:45.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:45.358 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:45 smithi096 ceph-mon[98901]: pgmap v1140: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 113 active+clean; 400 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 20527/19074 objects degraded (107.618%); 0 B/s, 153 objects/s recovering 2024-09-13T15:01:45.358 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:45 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 20527/19074 objects degraded (107.618%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:45.359 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:45 smithi096 ceph-mon[102928]: pgmap v1140: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 113 active+clean; 400 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 20527/19074 objects degraded (107.618%); 0 B/s, 153 objects/s recovering 2024-09-13T15:01:45.359 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:45 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 20527/19074 objects degraded (107.618%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:45.401 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:45.443 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:45.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:45.544 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:45.578 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:45.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:45.680 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:45.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:45 smithi204 ceph-mon[87071]: pgmap v1140: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 113 active+clean; 400 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 20527/19074 objects degraded (107.618%); 0 B/s, 153 objects/s recovering 2024-09-13T15:01:45.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:45 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 20527/19074 objects degraded (107.618%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:45.724 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:45.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:45.826 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:01:45.826 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:45.872 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:45.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:45.974 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:46.016 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:46.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:46.118 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:46.155 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:46.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:46.256 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:46.288 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:46.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:46.361 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T15:01:46.390 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:46.416 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:46.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:46.520 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:46.552 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:46.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:46.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:46 smithi096 ceph-mon[98901]: 1.7 scrub starts 2024-09-13T15:01:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:46 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:01:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:46 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:01:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:46 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3650233077' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:01:46.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:46 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:01:46.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:46 smithi096 ceph-mon[102928]: 1.7 scrub starts 2024-09-13T15:01:46.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:46 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:01:46.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:46 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:01:46.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:46 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3650233077' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:01:46.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:46 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:01:46.654 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:46.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.686 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:46.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:46.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:46 smithi204 ceph-mon[87071]: 1.7 scrub starts 2024-09-13T15:01:46.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:46 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:01:46.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:46 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:01:46.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:46 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3650233077' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:01:46.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:46 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:01:46.787 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:46.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.824 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:46.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:46.928 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:46.958 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:46.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:47.059 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:47.094 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:47.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:47.196 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:47.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.237 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:47.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:47.338 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:47.369 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:47.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:47.471 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:47.496 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:47.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:47.598 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:47.634 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:47.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:47.696 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:47 smithi096 ceph-mon[98901]: pgmap v1141: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 113 active+clean; 345 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 20320/16422 objects degraded (123.736%); 696 KiB/s, 82 objects/s recovering 2024-09-13T15:01:47.696 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:47 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:01:47.696 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:47 smithi096 ceph-mon[98901]: osdmap e346: 8 total, 8 up, 7 in 2024-09-13T15:01:47.697 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:47 smithi096 ceph-mon[102928]: pgmap v1141: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 113 active+clean; 345 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 20320/16422 objects degraded (123.736%); 696 KiB/s, 82 objects/s recovering 2024-09-13T15:01:47.697 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:47 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:01:47.697 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:47 smithi096 ceph-mon[102928]: osdmap e346: 8 total, 8 up, 7 in 2024-09-13T15:01:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:47 smithi204 ceph-mon[87071]: pgmap v1141: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 113 active+clean; 345 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 20320/16422 objects degraded (123.736%); 696 KiB/s, 82 objects/s recovering 2024-09-13T15:01:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:47 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:01:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:47 smithi204 ceph-mon[87071]: osdmap e346: 8 total, 8 up, 7 in 2024-09-13T15:01:47.736 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:47.773 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:47.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:47.875 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:47.932 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:47.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:48.034 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:48.074 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:48.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:48.176 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:48.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.214 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:48.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:48.239 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:01:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:01:47] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.51.0" 2024-09-13T15:01:48.314 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:48.341 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:48.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:48.443 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:48.476 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:48.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:48.578 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:48.602 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:48.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:48.704 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:48.737 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:48.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:48.839 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:48.873 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:48.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:48.975 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:49.002 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:49.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:49.104 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:49.107 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T15:01:49.133 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:49.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:49.235 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:49.271 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:49.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:49.356 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:49.373 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:49.401 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:49.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:49.503 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:49.540 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:49.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:49.610 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:49 smithi096 ceph-mon[98901]: pgmap v1143: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 113 active+clean; 207 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 19990/9804 objects degraded (203.896%); 696 KiB/s, 94 objects/s recovering 2024-09-13T15:01:49.610 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:49 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 19990/9804 objects degraded (203.896%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:49.610 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:49 smithi096 ceph-mon[102928]: pgmap v1143: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 113 active+clean; 207 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 19990/9804 objects degraded (203.896%); 696 KiB/s, 94 objects/s recovering 2024-09-13T15:01:49.610 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:49 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 19990/9804 objects degraded (203.896%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:49.643 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:49.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.678 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:49.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:49.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:49 smithi204 ceph-mon[87071]: pgmap v1143: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 113 active+clean; 207 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 19990/9804 objects degraded (203.896%); 696 KiB/s, 94 objects/s recovering 2024-09-13T15:01:49.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:49 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 19990/9804 objects degraded (203.896%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:49.768 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:49.779 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:49.834 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:49.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:49.936 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:49.964 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:49.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:50.065 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:50.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.114 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:50.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:50.184 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Removed 119405 objects 2024-09-13T15:01:50.184 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Clean up completed and total clean up time :148.558 2024-09-13T15:01:50.215 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:50.237 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:50.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:50.339 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:50.362 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:50.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:50.365 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T15:01:50.464 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:50.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:50 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3923628871' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:01:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:01:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:01:50.487 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:50.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:50.589 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:50.611 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:50.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:50.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:50.734 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:50.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:50.805 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:50 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3923628871' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:50.805 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:50 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3923628871' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:50.837 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:50.859 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:50.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:50.961 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:50.983 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:50.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:51.085 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:51.107 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:51.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:51.209 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:51.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.231 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:51.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:51.332 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:51.353 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:51.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:51.455 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:51.475 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:51 smithi096 ceph-mon[98901]: pgmap v1144: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 113 active+clean; 97 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 19347/4527 objects degraded (427.369%); 2.3 MiB/s, 114 objects/s recovering 2024-09-13T15:01:51.475 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:51 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:01:51.475 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:51 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3923628871' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:51.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:51 smithi096 ceph-mon[98901]: osdmap e347: 8 total, 8 up, 7 in 2024-09-13T15:01:51.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:01:51.476 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:51.476 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:51 smithi096 ceph-mon[102928]: pgmap v1144: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 113 active+clean; 97 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 19347/4527 objects degraded (427.369%); 2.3 MiB/s, 114 objects/s recovering 2024-09-13T15:01:51.476 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:51 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:01:51.476 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:51 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3923628871' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:51.476 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:51 smithi096 ceph-mon[102928]: osdmap e347: 8 total, 8 up, 7 in 2024-09-13T15:01:51.476 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:01:51.476 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:51.478 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:51.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:51.580 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:51.601 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:51.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:51.703 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:51 smithi204 ceph-mon[87071]: pgmap v1144: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 113 active+clean; 97 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 19347/4527 objects degraded (427.369%); 2.3 MiB/s, 114 objects/s recovering 2024-09-13T15:01:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:51 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:01:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:51 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3923628871' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:51 smithi204 ceph-mon[87071]: osdmap e347: 8 total, 8 up, 7 in 2024-09-13T15:01:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:01:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:51.725 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:51.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:51.827 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:51.850 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:51.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:51.952 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:51.974 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:51.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:52.076 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:52.098 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:52.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:52.200 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:52.222 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:52.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:52.324 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:52.345 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:52.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:52.448 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:52.472 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:52.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:52.574 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:52.596 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:52.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:52.698 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:52.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:52 smithi204 ceph-mon[87071]: 1.6 scrub starts 2024-09-13T15:01:52.720 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:52.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:52.822 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:52.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:52 smithi096 ceph-mon[98901]: 1.6 scrub starts 2024-09-13T15:01:52.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:52 smithi096 ceph-mon[102928]: 1.6 scrub starts 2024-09-13T15:01:52.844 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:52.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:52.946 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:52.968 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:52.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:53.070 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:53.092 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:53.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:53.236 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:53.242 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:01:53.258 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:53.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:53.361 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:53.368 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:53 smithi096 ceph-mon[98901]: 1.16 deep-scrub starts 2024-09-13T15:01:53.368 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:53 smithi096 ceph-mon[98901]: pgmap v1146: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 113 active+clean; 97 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 19347/4527 objects degraded (427.369%); 2.8 MiB/s, 70 objects/s recovering 2024-09-13T15:01:53.369 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:53 smithi096 ceph-mon[102928]: 1.16 deep-scrub starts 2024-09-13T15:01:53.369 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:53 smithi096 ceph-mon[102928]: pgmap v1146: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 113 active+clean; 97 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 19347/4527 objects degraded (427.369%); 2.8 MiB/s, 70 objects/s recovering 2024-09-13T15:01:53.385 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:53.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:53.489 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:53.512 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:53.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:53.568 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:01:53.568 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:01:53.614 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:53.636 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:53.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:53 smithi204 ceph-mon[87071]: 1.16 deep-scrub starts 2024-09-13T15:01:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:53 smithi204 ceph-mon[87071]: pgmap v1146: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 113 active+clean; 97 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 19347/4527 objects degraded (427.369%); 2.8 MiB/s, 70 objects/s recovering 2024-09-13T15:01:53.738 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:53.759 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:53.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:53.861 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:53.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.884 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:53.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:53.985 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:54.007 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:54.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:54.109 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:54.131 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:54.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:54.233 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:54.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.256 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:54.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:54.357 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:54.381 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:54.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:54.482 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:54.504 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:54.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:54.589 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2024-09-13T15:01:54.589 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2024-09-13T15:01:54.605 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:54.627 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:54.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:54.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:54 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 19347/4527 objects degraded (427.369%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:54.729 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:54.751 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:54.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:54.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:54 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 19347/4527 objects degraded (427.369%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:54.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:54 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 19347/4527 objects degraded (427.369%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:54.852 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:54.874 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:54.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:54.975 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:54.997 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:54.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:55.078 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:55.099 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:55.120 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:55.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:55.222 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:55.244 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:55.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:55.346 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:55.368 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:55.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:55.470 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:55.493 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:55.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:55.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:55.616 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:55.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:55.718 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:55 smithi204 ceph-mon[87071]: pgmap v1147: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 14 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 18546/552 objects degraded (3359.783%); 2.1 MiB/s, 134 objects/s recovering 2024-09-13T15:01:55.739 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:55.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:55.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:55 smithi096 ceph-mon[98901]: pgmap v1147: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 14 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 18546/552 objects degraded (3359.783%); 2.1 MiB/s, 134 objects/s recovering 2024-09-13T15:01:55.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:55 smithi096 ceph-mon[102928]: pgmap v1147: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 14 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 18546/552 objects degraded (3359.783%); 2.1 MiB/s, 134 objects/s recovering 2024-09-13T15:01:55.841 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:55.866 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:55.867 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T15:01:55.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:55.969 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:55.991 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:55.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:56.093 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:56.114 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:56.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:56.216 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:56.237 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:56.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:56.339 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:56.360 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:56.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:56.462 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:56.484 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:56.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:56.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:56 smithi096 ceph-mon[98901]: 1.17 scrub starts 2024-09-13T15:01:56.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:56 smithi096 ceph-mon[102928]: 1.17 scrub starts 2024-09-13T15:01:56.585 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:56.606 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:56.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:56.708 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:56.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:56 smithi204 ceph-mon[87071]: 1.17 scrub starts 2024-09-13T15:01:56.730 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:56.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:56.831 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:56.853 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:56.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:56.955 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:56.977 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:56.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:57.079 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:57.103 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:57.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:57.170 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:57.172 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:57.205 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:57.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.228 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:57.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:57.329 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:57.350 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:57.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:57.452 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:57.475 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:57.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:57.525 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:57 smithi096 ceph-mon[98901]: pgmap v1148: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 18504/84 objects degraded (22028.571%); 2.0 MiB/s, 114 objects/s recovering 2024-09-13T15:01:57.525 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:57 smithi096 ceph-mon[98901]: 1.7 scrub starts 2024-09-13T15:01:57.526 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:57 smithi096 ceph-mon[102928]: pgmap v1148: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 18504/84 objects degraded (22028.571%); 2.0 MiB/s, 114 objects/s recovering 2024-09-13T15:01:57.526 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:57 smithi096 ceph-mon[102928]: 1.7 scrub starts 2024-09-13T15:01:57.577 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:57.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.599 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:57.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:57.700 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:57.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:57 smithi204 ceph-mon[87071]: pgmap v1148: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 18504/84 objects degraded (22028.571%); 2.0 MiB/s, 114 objects/s recovering 2024-09-13T15:01:57.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:57 smithi204 ceph-mon[87071]: 1.7 scrub starts 2024-09-13T15:01:57.721 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:57.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:57.823 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:57.845 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:57.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:57.947 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:57.970 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:57.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:57.972 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:01:57.972 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:01:57.993 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:01:57.993 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:01:58.010 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:01:58.011 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:01:58.028 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:01:58.028 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:01:58.046 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:01:58.046 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:01:58.064 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:01:58.064 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:01:58.073 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:58.084 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:01:58.084 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:01:58.094 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:58.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:58.100 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:01:58.100 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:01:58.192 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:01:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:01:57] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.51.0" 2024-09-13T15:01:58.196 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:58.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.221 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:58.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:58.323 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:58.345 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:58.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:58.447 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:58.471 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:58.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:58.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:58 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4074175822' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:58.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:58 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4074175822' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:58.573 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:58.596 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:58.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:58.698 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:58.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:58 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4074175822' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:58.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.726 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:58.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:58.827 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:58.848 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:58.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:58.950 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:58.972 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:58.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:59.074 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:59.097 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:59.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:59.198 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:59.220 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:59.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:59.322 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:59.344 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:59.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:59.446 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:59.449 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:59.468 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:59.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:59.570 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:59.591 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:59.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:59.693 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:59.714 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:01:59.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:59 smithi204 ceph-mon[87071]: pgmap v1149: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 18046/84 objects degraded (21483.333%); 2.0 MiB/s, 148 objects/s recovering 2024-09-13T15:01:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:01:59 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 18046/84 objects degraded (21483.333%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:59.719 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:59 smithi096 ceph-mon[98901]: pgmap v1149: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 18046/84 objects degraded (21483.333%); 2.0 MiB/s, 148 objects/s recovering 2024-09-13T15:01:59.719 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:01:59 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 18046/84 objects degraded (21483.333%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:59.719 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:59 smithi096 ceph-mon[102928]: pgmap v1149: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 18046/84 objects degraded (21483.333%); 2.0 MiB/s, 148 objects/s recovering 2024-09-13T15:01:59.720 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:01:59 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 18046/84 objects degraded (21483.333%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:59.816 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:59.840 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:59.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:59.941 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:59.964 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:01:59.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:00.065 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:00.087 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:00.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:00.189 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:00.210 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:00.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:00.311 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:00.333 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:00.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:00.404 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T15:02:00.435 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:00.457 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:00.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:00 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3858810769' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:02:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:00 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:02:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:02:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:02:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:02:00.553 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:00 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3858810769' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:02:00.553 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:00 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:02:00.553 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:00 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3858810769' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:02:00.553 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:00 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:02:00.559 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:00.581 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:00.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:00.683 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:00.705 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:00.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:00.807 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:00.830 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:00.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:00.931 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:00.953 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:00.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:01.055 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:01.077 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:01.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:01.179 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:01.200 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:01.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:01.302 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:01.324 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:01.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:01.426 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:01.448 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:01.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:01.550 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:01.571 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:01.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:01.673 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:01.695 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:01 smithi096 ceph-mon[98901]: 1.6 deep-scrub starts 2024-09-13T15:02:01.695 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:01 smithi096 ceph-mon[98901]: pgmap v1150: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 17611/84 objects degraded (20965.476%); 282 KiB/s, 153 objects/s recovering 2024-09-13T15:02:01.695 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:01 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:02:01.695 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:01 smithi096 ceph-mon[98901]: osdmap e348: 8 total, 8 up, 7 in 2024-09-13T15:02:01.695 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:01 smithi096 ceph-mon[102928]: 1.6 deep-scrub starts 2024-09-13T15:02:01.695 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:01 smithi096 ceph-mon[102928]: pgmap v1150: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 17611/84 objects degraded (20965.476%); 282 KiB/s, 153 objects/s recovering 2024-09-13T15:02:01.695 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:01 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:02:01.695 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:01 smithi096 ceph-mon[102928]: osdmap e348: 8 total, 8 up, 7 in 2024-09-13T15:02:01.696 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:01.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:01 smithi204 ceph-mon[87071]: 1.6 deep-scrub starts 2024-09-13T15:02:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:01 smithi204 ceph-mon[87071]: pgmap v1150: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 17611/84 objects degraded (20965.476%); 282 KiB/s, 153 objects/s recovering 2024-09-13T15:02:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:01 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:02:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:01 smithi204 ceph-mon[87071]: osdmap e348: 8 total, 8 up, 7 in 2024-09-13T15:02:01.798 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:01.819 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:01.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:01.921 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:01.942 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:01.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:02.044 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:02.067 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:02.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:02.169 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:02.190 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:02.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:02.292 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:02.314 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:02.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:02.416 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:02.437 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:02.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:02.539 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:02.561 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:02.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:02.663 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:02.686 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:02.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:02.787 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:02.809 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:02.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:02.910 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:02.932 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:02.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:03.034 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:03.055 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:03.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:03.157 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:03.178 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:03.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:03.280 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:03.303 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:03.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:03.404 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:03.427 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:03.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:03.529 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:03.556 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:03.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:03.658 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:03.679 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:03.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:03.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:03 smithi204 ceph-mon[87071]: 1.16 scrub starts 2024-09-13T15:02:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:03 smithi204 ceph-mon[87071]: pgmap v1152: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 17611/84 objects degraded (20965.476%); 282 KiB/s, 153 objects/s recovering 2024-09-13T15:02:03.781 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:03.804 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:03.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:03.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:03 smithi096 ceph-mon[98901]: 1.16 scrub starts 2024-09-13T15:02:03.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:03 smithi096 ceph-mon[98901]: pgmap v1152: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 17611/84 objects degraded (20965.476%); 282 KiB/s, 153 objects/s recovering 2024-09-13T15:02:03.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:03 smithi096 ceph-mon[102928]: 1.16 scrub starts 2024-09-13T15:02:03.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:03 smithi096 ceph-mon[102928]: pgmap v1152: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 17611/84 objects degraded (20965.476%); 282 KiB/s, 153 objects/s recovering 2024-09-13T15:02:03.906 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:03.928 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:03.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:04.030 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:04.052 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:04.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:04.154 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:04.175 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:04.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:04.277 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:04.298 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:04.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:04.400 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:04.422 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:04.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:04.524 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:04.546 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:04.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:04.648 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:04.670 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:04.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:04.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:04 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 17611/84 objects degraded (20965.476%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:04.772 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:04.794 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:04.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:04.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:04 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 17611/84 objects degraded (20965.476%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:04.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:04 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 17611/84 objects degraded (20965.476%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:04.895 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:04.917 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:04.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:05.019 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:05.041 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:05.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:05.143 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:05.164 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:05.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:05.266 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:05.287 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:05.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:05.389 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:05.411 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:05.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:05.512 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:05.533 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:05.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:05.635 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:05.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.659 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:05.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:05 smithi204 ceph-mon[87071]: pgmap v1153: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 15 op/s; 16629/84 objects degraded (19796.429%); 282 KiB/s, 191 objects/s recovering 2024-09-13T15:02:05.759 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:05.780 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:05.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:05.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:05 smithi096 ceph-mon[98901]: pgmap v1153: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 15 op/s; 16629/84 objects degraded (19796.429%); 282 KiB/s, 191 objects/s recovering 2024-09-13T15:02:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:05 smithi096 ceph-mon[102928]: pgmap v1153: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 15 op/s; 16629/84 objects degraded (19796.429%); 282 KiB/s, 191 objects/s recovering 2024-09-13T15:02:05.882 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:05.904 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:05.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:06.006 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:06.027 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:06.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:06.129 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:06.152 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:06.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:06.254 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:06.276 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:06.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:06.378 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:06.400 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:06.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:06.502 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:06.523 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:06.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:06.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:06.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:06.625 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:06.647 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:06.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:06.749 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:06.771 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:06.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:06.873 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:06.895 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:06.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:06.997 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:07.018 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:07.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:07.120 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:07.141 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:07.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:07.243 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:07.265 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:07.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:07.366 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:07.388 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:07.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:07.490 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:07.512 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:07.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:07.614 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:07.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.637 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:07.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:07 smithi204 ceph-mon[87071]: pgmap v1154: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 16629/84 objects degraded (19796.429%); 0 B/s, 187 objects/s recovering 2024-09-13T15:02:07.737 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:07.760 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:07.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:07.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:07 smithi096 ceph-mon[98901]: pgmap v1154: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 16629/84 objects degraded (19796.429%); 0 B/s, 187 objects/s recovering 2024-09-13T15:02:07.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:07 smithi096 ceph-mon[102928]: pgmap v1154: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 16629/84 objects degraded (19796.429%); 0 B/s, 187 objects/s recovering 2024-09-13T15:02:07.862 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:07.884 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:07.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:07.985 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:08.007 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:08.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:08.108 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:08.130 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:08.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:08.231 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:08.254 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:08.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:08.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:02:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:02:07] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.51.0" 2024-09-13T15:02:08.355 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:08.378 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:08.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:08.480 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:08.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.503 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:08.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:08.604 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:08.626 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:08.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:08.728 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:08.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.750 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:08.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:08.851 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:08.873 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:08.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:08.975 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:08.996 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:08.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:09.098 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:09.120 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:09.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:09.221 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:09.244 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:09.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:09.346 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:09.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.368 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:09.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:09.469 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:09.491 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:09.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:09.593 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:09.615 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:09.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:09.717 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:09.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:09 smithi204 ceph-mon[87071]: 1.17 deep-scrub starts 2024-09-13T15:02:09.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:09 smithi204 ceph-mon[87071]: pgmap v1155: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 16085/84 objects degraded (19148.810%); 0 B/s, 196 objects/s recovering 2024-09-13T15:02:09.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:09 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 16085/84 objects degraded (19148.810%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:09.738 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:09.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:09.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:09 smithi096 ceph-mon[102928]: 1.17 deep-scrub starts 2024-09-13T15:02:09.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:09 smithi096 ceph-mon[102928]: pgmap v1155: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 16085/84 objects degraded (19148.810%); 0 B/s, 196 objects/s recovering 2024-09-13T15:02:09.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:09 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 16085/84 objects degraded (19148.810%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:09.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:09 smithi096 ceph-mon[98901]: 1.17 deep-scrub starts 2024-09-13T15:02:09.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:09 smithi096 ceph-mon[98901]: pgmap v1155: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 16085/84 objects degraded (19148.810%); 0 B/s, 196 objects/s recovering 2024-09-13T15:02:09.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:09 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 16085/84 objects degraded (19148.810%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:09.840 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:09.861 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:09.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:09.963 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:09.985 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:09.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:10.088 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:10.109 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:10.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:10.211 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:10.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.233 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:10.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:10.334 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:10.356 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:10.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:10.457 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:02:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:02:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:02:10.479 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:10.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:10.581 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:10.603 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:10.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:10.704 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:10.726 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:10.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:10.828 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:10.849 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:10.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:10.951 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:10.974 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:10.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:11.076 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:11.098 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:11.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:11.200 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:11.222 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:11.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:11.324 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:11.347 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:11.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:11.448 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:11.470 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:11.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:11.571 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:11.594 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:11.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:11.696 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:11 smithi204 ceph-mon[87071]: pgmap v1156: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 15563/84 objects degraded (18527.381%); 0 B/s, 204 objects/s recovering 2024-09-13T15:02:11.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.720 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:11.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:11.778 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:11 smithi096 ceph-mon[98901]: pgmap v1156: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 15563/84 objects degraded (18527.381%); 0 B/s, 204 objects/s recovering 2024-09-13T15:02:11.778 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:11 smithi096 ceph-mon[102928]: pgmap v1156: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 15563/84 objects degraded (18527.381%); 0 B/s, 204 objects/s recovering 2024-09-13T15:02:11.821 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:11.842 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:11.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:11.944 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:11.967 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:11.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:12.069 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:12.094 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:12.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:12.195 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:12.216 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:12.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:12.318 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:12.339 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:12.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:12.441 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:12.462 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:12.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:12.564 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:12.586 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:12.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:12.688 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:12.709 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:12.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:12.811 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:12.833 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:12.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:12.935 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:12.958 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:12.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:13.059 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:13.082 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:13.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:13.184 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:13.206 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:13.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:13.308 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:13.329 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:13.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:13.431 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:13.453 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:13.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:13.555 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:13.577 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:13.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:13.679 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:13.701 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:13.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:13.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:13 smithi204 ceph-mon[87071]: pgmap v1157: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 15563/84 objects degraded (18527.381%); 0 B/s, 174 objects/s recovering 2024-09-13T15:02:13.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:13 smithi204 ceph-mon[87071]: 1.7 scrub starts 2024-09-13T15:02:13.803 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:13.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:13 smithi096 ceph-mon[98901]: pgmap v1157: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 15563/84 objects degraded (18527.381%); 0 B/s, 174 objects/s recovering 2024-09-13T15:02:13.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:13 smithi096 ceph-mon[98901]: 1.7 scrub starts 2024-09-13T15:02:13.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:13 smithi096 ceph-mon[102928]: pgmap v1157: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 15563/84 objects degraded (18527.381%); 0 B/s, 174 objects/s recovering 2024-09-13T15:02:13.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:13 smithi096 ceph-mon[102928]: 1.7 scrub starts 2024-09-13T15:02:13.825 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:13.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:13.926 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:13.950 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:13.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:14.052 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:14.073 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:14.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:14.175 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:14.197 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:14.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:14.299 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:14.321 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:14.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:14.423 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:14.445 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:14.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:14.547 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:14.569 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:14.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:14.671 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:14.693 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:14.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:14.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:14 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 15563/84 objects degraded (18527.381%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:14.795 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:14.817 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:14.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:14.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:14 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 15563/84 objects degraded (18527.381%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:14.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:14 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 15563/84 objects degraded (18527.381%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:14.919 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:14.941 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:14.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:15.043 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:15.064 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:15.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:15.166 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:15.188 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:15.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:15.290 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:15.312 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:15.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:15.414 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:15.436 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:15.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:15.537 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:15.561 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:15.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:15.663 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:15.684 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:15.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:15.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:15 smithi204 ceph-mon[87071]: pgmap v1158: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 14380/84 objects degraded (17119.048%); 0 B/s, 269 objects/s recovering 2024-09-13T15:02:15.786 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:15.807 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:15.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:15.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:15 smithi096 ceph-mon[98901]: pgmap v1158: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 14380/84 objects degraded (17119.048%); 0 B/s, 269 objects/s recovering 2024-09-13T15:02:15.822 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:15 smithi096 ceph-mon[102928]: pgmap v1158: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 14380/84 objects degraded (17119.048%); 0 B/s, 269 objects/s recovering 2024-09-13T15:02:15.909 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:15.930 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:15.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:16.033 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:16.054 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:16.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:16.156 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:16.180 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:16.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:16.282 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:16.305 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:16.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:16.406 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:16.430 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:16.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:16.531 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:16.554 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:16.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:16.655 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:16.678 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:16.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:16.780 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:16.802 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:16.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:16.904 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:16.927 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:16.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:17.028 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:17.050 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:17.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:17.152 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:17.174 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:17.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:17.276 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:17.298 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:17.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:17.399 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:17.421 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:17.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:17.522 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:17.546 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:17.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:17.648 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:17.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.672 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:17.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:17.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:17 smithi204 ceph-mon[87071]: 1.6 scrub starts 2024-09-13T15:02:17.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:17 smithi204 ceph-mon[87071]: pgmap v1159: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 14380/84 objects degraded (17119.048%); 0 B/s, 187 objects/s recovering 2024-09-13T15:02:17.773 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:17.790 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:17 smithi096 ceph-mon[98901]: 1.6 scrub starts 2024-09-13T15:02:17.790 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:17 smithi096 ceph-mon[98901]: pgmap v1159: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 14380/84 objects degraded (17119.048%); 0 B/s, 187 objects/s recovering 2024-09-13T15:02:17.791 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:17 smithi096 ceph-mon[102928]: 1.6 scrub starts 2024-09-13T15:02:17.791 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:17 smithi096 ceph-mon[102928]: pgmap v1159: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 14380/84 objects degraded (17119.048%); 0 B/s, 187 objects/s recovering 2024-09-13T15:02:17.797 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:17.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:17.819 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T15:02:17.899 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:17.930 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:17.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:18.032 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:18.060 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:18.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:18.072 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:02:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:02:17] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.51.0" 2024-09-13T15:02:18.162 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:18.184 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:18.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:18.286 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:18.307 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:18.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:18.409 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:18.430 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:18.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:18.440 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T15:02:18.532 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:18.553 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:18.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:18.655 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:18.677 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:18.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:18.686 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:18 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3609620292' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:18.686 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:18 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3609620292' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:18.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:18 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3609620292' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:18.779 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:18.801 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:18.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:18.903 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:18.929 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:18.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:19.030 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:19.055 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:19.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:19.156 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:19.181 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:19.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:19.283 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:19.306 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:19.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:19.407 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:19.433 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:19.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:19.534 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:19.556 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:19.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:19.659 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:19.684 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:19.705 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:19 smithi096 ceph-mon[98901]: 1.16 scrub starts 2024-09-13T15:02:19.705 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:19 smithi096 ceph-mon[98901]: pgmap v1160: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 13689/84 objects degraded (16296.429%); 0 B/s, 244 objects/s recovering 2024-09-13T15:02:19.705 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:19 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:19.705 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:19 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3609620292' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:02:19.705 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:19 smithi096 ceph-mon[98901]: osdmap e349: 8 total, 8 up, 7 in 2024-09-13T15:02:19.705 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:19 smithi096 ceph-mon[98901]: Health check update: Degraded data redundancy: 13689/84 objects degraded (16296.429%), 3 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:19.705 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:19 smithi096 ceph-mon[102928]: 1.16 scrub starts 2024-09-13T15:02:19.705 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:19 smithi096 ceph-mon[102928]: pgmap v1160: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 13689/84 objects degraded (16296.429%); 0 B/s, 244 objects/s recovering 2024-09-13T15:02:19.706 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:19 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:19.706 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:19 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3609620292' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:02:19.706 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:19 smithi096 ceph-mon[102928]: osdmap e349: 8 total, 8 up, 7 in 2024-09-13T15:02:19.706 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:19 smithi096 ceph-mon[102928]: Health check update: Degraded data redundancy: 13689/84 objects degraded (16296.429%), 3 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:19.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:19 smithi204 ceph-mon[87071]: 1.16 scrub starts 2024-09-13T15:02:19.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:19 smithi204 ceph-mon[87071]: pgmap v1160: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 13689/84 objects degraded (16296.429%); 0 B/s, 244 objects/s recovering 2024-09-13T15:02:19.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:19 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:19.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:19 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3609620292' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:02:19.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:19 smithi204 ceph-mon[87071]: osdmap e349: 8 total, 8 up, 7 in 2024-09-13T15:02:19.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:19 smithi204 ceph-mon[87071]: Health check update: Degraded data redundancy: 13689/84 objects degraded (16296.429%), 3 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:19.786 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:19.808 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:19.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:19.910 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:19.935 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:19.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:20.037 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:20.059 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:20.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:20.161 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:20.185 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:20.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:20.287 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:20.312 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:20.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:20.414 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:20.435 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:20.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:02:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:02:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:02:20.538 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:20.560 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:20.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:20.662 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:20.680 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:02:20.686 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:20.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:20.788 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:20.816 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:20.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:20.919 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:20.941 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:20.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:21.043 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:21.064 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:21.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:21.166 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:21.191 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:21.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:21.293 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:21.318 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:21.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:21.359 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T15:02:21.359 INFO:teuthology.orchestra.run.smithi096.stdout: "size": 20, 2024-09-13T15:02:21.359 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 600, 2024-09-13T15:02:21.359 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T15:02:21.359 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.359 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:211669 9.8 e338/336 9:10ebbe69:::benchmark_data_smithi096_256909_object92261:head v 338'13393, mlcod=317'7126)", 2024-09-13T15:02:21.359 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:15.820646+0000", 2024-09-13T15:02:21.359 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 65.535528889000005, 2024-09-13T15:02:21.359 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.077027830000000005, 2024-09-13T15:02:21.359 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.359 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.360 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.360 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.360 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.360 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.360 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.820646+0000", 2024-09-13T15:02:21.360 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.360 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.360 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.360 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.360 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.820646+0000", 2024-09-13T15:02:21.360 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.360 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.361 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.361 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.361 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.820647+0000", 2024-09-13T15:02:21.361 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3069999999999999e-06 2024-09-13T15:02:21.361 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.361 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.361 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.361 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.820659+0000", 2024-09-13T15:02:21.361 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1561e-05 2024-09-13T15:02:21.361 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.361 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.361 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.361 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.820677+0000", 2024-09-13T15:02:21.362 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8172000000000002e-05 2024-09-13T15:02:21.362 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.362 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.362 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.362 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.820687+0000", 2024-09-13T15:02:21.362 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0084e-05 2024-09-13T15:02:21.362 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.362 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.362 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.362 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.820767+0000", 2024-09-13T15:02:21.362 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0309999999999995e-05 2024-09-13T15:02:21.362 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.362 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.363 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.363 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.820794+0000", 2024-09-13T15:02:21.363 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6723000000000001e-05 2024-09-13T15:02:21.363 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.363 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.363 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.363 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.897565+0000", 2024-09-13T15:02:21.363 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.076771159000000005 2024-09-13T15:02:21.363 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.363 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.363 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.363 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.897674+0000", 2024-09-13T15:02:21.364 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000108514 2024-09-13T15:02:21.364 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.364 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.364 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.364 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.364 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.364 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:211673 9.8 e338/336 9:10884aee:::benchmark_data_smithi096_256909_object92265:head v 338'13394, mlcod=317'7126)", 2024-09-13T15:02:21.364 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:15.827474+0000", 2024-09-13T15:02:21.364 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 65.528700810999993, 2024-09-13T15:02:21.364 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.070758474000000002, 2024-09-13T15:02:21.364 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.364 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.364 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.364 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.365 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.365 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.365 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.827474+0000", 2024-09-13T15:02:21.365 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.365 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.365 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.365 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.365 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.827474+0000", 2024-09-13T15:02:21.365 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.365 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.365 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.365 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.365 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.827475+0000", 2024-09-13T15:02:21.366 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.294e-06 2024-09-13T15:02:21.366 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.366 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.366 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.366 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.827483+0000", 2024-09-13T15:02:21.366 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9650000000000008e-06 2024-09-13T15:02:21.366 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.366 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.366 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.366 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.827497+0000", 2024-09-13T15:02:21.366 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4037e-05 2024-09-13T15:02:21.366 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.366 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.367 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.367 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.827506+0000", 2024-09-13T15:02:21.367 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6719999999999993e-06 2024-09-13T15:02:21.367 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.367 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.367 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.367 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.832908+0000", 2024-09-13T15:02:21.367 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0054020470000000001 2024-09-13T15:02:21.367 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.367 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.367 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.367 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.832934+0000", 2024-09-13T15:02:21.367 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6217999999999999e-05 2024-09-13T15:02:21.367 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.367 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.368 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.368 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.897679+0000", 2024-09-13T15:02:21.368 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.064745225000000003 2024-09-13T15:02:21.368 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.368 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.368 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.368 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.898232+0000", 2024-09-13T15:02:21.368 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00055301600000000003 2024-09-13T15:02:21.368 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.368 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.368 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.368 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.368 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.369 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:211682 9.8 e338/336 9:1f8a0277:::benchmark_data_smithi096_256909_object92274:head v 338'13395, mlcod=317'7126)", 2024-09-13T15:02:21.369 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:15.838268+0000", 2024-09-13T15:02:21.369 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 65.517906637999999, 2024-09-13T15:02:21.369 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.060003268999999998, 2024-09-13T15:02:21.369 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.369 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.369 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.369 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.369 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.369 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.369 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.838268+0000", 2024-09-13T15:02:21.369 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.369 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.369 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.370 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.370 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.838268+0000", 2024-09-13T15:02:21.370 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.370 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.370 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.370 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.370 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.838270+0000", 2024-09-13T15:02:21.370 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.55e-06 2024-09-13T15:02:21.370 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.370 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.370 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.370 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.838281+0000", 2024-09-13T15:02:21.370 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0947999999999999e-05 2024-09-13T15:02:21.371 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.371 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.371 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.371 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.838300+0000", 2024-09-13T15:02:21.371 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9553999999999999e-05 2024-09-13T15:02:21.371 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.371 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.371 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.371 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.838310+0000", 2024-09-13T15:02:21.371 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3360000000000001e-06 2024-09-13T15:02:21.371 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.371 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.371 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.371 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.846522+0000", 2024-09-13T15:02:21.372 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.008212291 2024-09-13T15:02:21.372 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.372 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.372 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.372 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.846556+0000", 2024-09-13T15:02:21.372 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4591000000000001e-05 2024-09-13T15:02:21.372 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.372 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.372 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.372 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.898240+0000", 2024-09-13T15:02:21.372 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.051683373999999997 2024-09-13T15:02:21.372 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.372 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.372 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.373 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.898271+0000", 2024-09-13T15:02:21.373 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.1625000000000003e-05 2024-09-13T15:02:21.373 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.373 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.373 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.373 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.373 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.373 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:211813 9.8 e338/336 9:17486883:::benchmark_data_smithi096_256909_object92405:head v 338'13405, mlcod=317'7126)", 2024-09-13T15:02:21.373 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:15.981174+0000", 2024-09-13T15:02:21.373 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 65.375001197000003, 2024-09-13T15:02:21.374 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.072260748999999999, 2024-09-13T15:02:21.375 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.375 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.375 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.375 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.375 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.375 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.375 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.981174+0000", 2024-09-13T15:02:21.375 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.375 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.375 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.375 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.375 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.981174+0000", 2024-09-13T15:02:21.375 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.375 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.376 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.376 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.376 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.981175+0000", 2024-09-13T15:02:21.376 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.294e-06 2024-09-13T15:02:21.376 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.376 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.376 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.376 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.981183+0000", 2024-09-13T15:02:21.376 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0090000000000007e-06 2024-09-13T15:02:21.376 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.376 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.376 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.376 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.981200+0000", 2024-09-13T15:02:21.377 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7342000000000001e-05 2024-09-13T15:02:21.377 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.377 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.377 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.377 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.981209+0000", 2024-09-13T15:02:21.377 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8380000000000008e-06 2024-09-13T15:02:21.377 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.377 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.377 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.377 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.991473+0000", 2024-09-13T15:02:21.377 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.010264143999999999 2024-09-13T15:02:21.377 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.377 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.378 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.378 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.991538+0000", 2024-09-13T15:02:21.378 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.4581999999999999e-05 2024-09-13T15:02:21.378 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.378 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.378 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.378 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.053390+0000", 2024-09-13T15:02:21.378 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.061852529000000003 2024-09-13T15:02:21.378 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.378 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.378 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.378 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.053434+0000", 2024-09-13T15:02:21.379 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4011000000000003e-05 2024-09-13T15:02:21.379 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.379 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.379 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.379 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.379 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.379 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:211815 9.8 e338/336 9:153dbef5:::benchmark_data_smithi096_256909_object92407:head v 338'13406, mlcod=317'7126)", 2024-09-13T15:02:21.379 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:15.985960+0000", 2024-09-13T15:02:21.379 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 65.370215029999997, 2024-09-13T15:02:21.379 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.067507540000000005, 2024-09-13T15:02:21.379 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.379 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.379 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.380 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.380 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.380 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.380 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.985960+0000", 2024-09-13T15:02:21.380 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.380 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.380 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.380 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.380 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.985960+0000", 2024-09-13T15:02:21.380 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.380 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.380 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.380 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.380 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.985961+0000", 2024-09-13T15:02:21.380 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1999999999999999e-06 2024-09-13T15:02:21.381 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.381 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.381 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.381 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.985970+0000", 2024-09-13T15:02:21.381 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5779999999999999e-06 2024-09-13T15:02:21.381 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.381 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.381 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.381 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.985988+0000", 2024-09-13T15:02:21.381 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8340000000000001e-05 2024-09-13T15:02:21.381 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.381 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.381 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.381 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.985997+0000", 2024-09-13T15:02:21.382 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8890000000000008e-06 2024-09-13T15:02:21.382 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.382 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.382 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.382 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.001289+0000", 2024-09-13T15:02:21.382 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.015291880000000001 2024-09-13T15:02:21.382 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.382 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.382 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.382 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.001315+0000", 2024-09-13T15:02:21.382 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6389000000000001e-05 2024-09-13T15:02:21.382 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.382 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.382 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.382 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.053439+0000", 2024-09-13T15:02:21.383 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.052123823 2024-09-13T15:02:21.383 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.383 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.383 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.383 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.053467+0000", 2024-09-13T15:02:21.383 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.8441000000000001e-05 2024-09-13T15:02:21.383 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.383 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.383 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.383 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.383 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.383 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:211828 9.8 e338/336 9:128db0e5:::benchmark_data_smithi096_256909_object92420:head v 338'13407, mlcod=317'7126)", 2024-09-13T15:02:21.383 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:15.997379+0000", 2024-09-13T15:02:21.384 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 65.358795602000001, 2024-09-13T15:02:21.384 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.064530964999999996, 2024-09-13T15:02:21.384 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.384 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.384 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.384 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.384 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.384 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.384 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.997379+0000", 2024-09-13T15:02:21.384 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.384 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.384 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.385 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.385 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.997379+0000", 2024-09-13T15:02:21.385 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.385 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.385 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.385 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.385 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.997381+0000", 2024-09-13T15:02:21.385 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.35e-06 2024-09-13T15:02:21.385 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.385 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.385 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.385 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.997389+0000", 2024-09-13T15:02:21.385 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.2279999999999998e-06 2024-09-13T15:02:21.385 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.386 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.386 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.386 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.997407+0000", 2024-09-13T15:02:21.386 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8502e-05 2024-09-13T15:02:21.386 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.386 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.386 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.386 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:15.997416+0000", 2024-09-13T15:02:21.386 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5189999999999992e-06 2024-09-13T15:02:21.386 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.386 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.386 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.386 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.013407+0000", 2024-09-13T15:02:21.386 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.015991037999999999 2024-09-13T15:02:21.387 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.387 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.387 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.387 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.013424+0000", 2024-09-13T15:02:21.387 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7442e-05 2024-09-13T15:02:21.387 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.387 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.387 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.387 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.061855+0000", 2024-09-13T15:02:21.388 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.048430220000000003 2024-09-13T15:02:21.388 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.388 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.388 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.388 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.061910+0000", 2024-09-13T15:02:21.388 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5665999999999999e-05 2024-09-13T15:02:21.388 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.388 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.389 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.389 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.389 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.389 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:211829 9.8 e338/336 9:12cf9932:::benchmark_data_smithi096_256909_object92421:head v 338'13408, mlcod=317'7126)", 2024-09-13T15:02:21.389 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:16.002266+0000", 2024-09-13T15:02:21.389 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 65.353908570000002, 2024-09-13T15:02:21.389 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.059674247999999999, 2024-09-13T15:02:21.389 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.389 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.389 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.389 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.389 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.389 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.389 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.002266+0000", 2024-09-13T15:02:21.390 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.390 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.390 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.390 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.390 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.002266+0000", 2024-09-13T15:02:21.390 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.390 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.390 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.390 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.390 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.002267+0000", 2024-09-13T15:02:21.390 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1179999999999999e-06 2024-09-13T15:02:21.390 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.390 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.391 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.391 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.002276+0000", 2024-09-13T15:02:21.391 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.2290000000000003e-06 2024-09-13T15:02:21.391 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.391 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.391 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.391 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.002291+0000", 2024-09-13T15:02:21.391 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5777000000000002e-05 2024-09-13T15:02:21.391 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.391 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.391 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.391 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.002300+0000", 2024-09-13T15:02:21.392 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5450000000000003e-06 2024-09-13T15:02:21.392 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.392 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.392 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.392 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.024175+0000", 2024-09-13T15:02:21.392 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.021875084999999999 2024-09-13T15:02:21.392 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.392 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.392 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.392 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.024203+0000", 2024-09-13T15:02:21.392 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.8053999999999999e-05 2024-09-13T15:02:21.392 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.392 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.392 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.393 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.061915+0000", 2024-09-13T15:02:21.393 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.037711608000000001 2024-09-13T15:02:21.393 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.393 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.393 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.393 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.061940+0000", 2024-09-13T15:02:21.393 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.5831999999999999e-05 2024-09-13T15:02:21.393 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.393 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.393 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.393 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.393 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.393 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:212105 9.8 e338/336 9:1639e7fe:::benchmark_data_smithi096_256909_object92697:head v 338'13425, mlcod=317'7126)", 2024-09-13T15:02:21.394 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:16.297966+0000", 2024-09-13T15:02:21.394 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 65.058208984999993, 2024-09-13T15:02:21.394 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.057485538000000003, 2024-09-13T15:02:21.394 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.394 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.394 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.394 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.394 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.394 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.394 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.297966+0000", 2024-09-13T15:02:21.394 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.394 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.394 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.394 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.395 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.297966+0000", 2024-09-13T15:02:21.395 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.395 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.395 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.395 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.395 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.297968+0000", 2024-09-13T15:02:21.395 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.778e-06 2024-09-13T15:02:21.395 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.395 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.396 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.396 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.297976+0000", 2024-09-13T15:02:21.396 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.4160000000000004e-06 2024-09-13T15:02:21.396 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.396 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.396 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.396 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.298028+0000", 2024-09-13T15:02:21.396 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2062999999999999e-05 2024-09-13T15:02:21.396 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.396 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.396 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.396 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.298037+0000", 2024-09-13T15:02:21.396 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8910000000000001e-06 2024-09-13T15:02:21.396 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.397 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.397 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.397 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.306249+0000", 2024-09-13T15:02:21.397 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0082119979999999999 2024-09-13T15:02:21.397 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.397 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.397 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.397 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.306282+0000", 2024-09-13T15:02:21.397 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.3207000000000002e-05 2024-09-13T15:02:21.397 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.397 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.397 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.397 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.355395+0000", 2024-09-13T15:02:21.398 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.049113282000000001 2024-09-13T15:02:21.398 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.398 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.398 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.398 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.355451+0000", 2024-09-13T15:02:21.398 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5903000000000003e-05 2024-09-13T15:02:21.398 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.398 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.398 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.398 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.398 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.398 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:212111 9.8 e338/336 9:10c19221:::benchmark_data_smithi096_256909_object92703:head v 338'13427, mlcod=317'7126)", 2024-09-13T15:02:21.398 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:16.306726+0000", 2024-09-13T15:02:21.399 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 65.049449152999998, 2024-09-13T15:02:21.399 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.058957217999999999, 2024-09-13T15:02:21.399 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.399 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.399 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.399 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.399 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.399 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.399 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.306726+0000", 2024-09-13T15:02:21.399 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.399 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.399 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.400 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.400 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.306726+0000", 2024-09-13T15:02:21.400 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.400 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.400 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.400 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.400 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.306727+0000", 2024-09-13T15:02:21.400 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3149999999999999e-06 2024-09-13T15:02:21.401 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.401 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.401 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.401 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.306734+0000", 2024-09-13T15:02:21.401 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2479999999999999e-06 2024-09-13T15:02:21.401 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.401 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.401 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.401 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.306749+0000", 2024-09-13T15:02:21.401 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.487e-05 2024-09-13T15:02:21.401 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.401 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.401 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.402 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.306757+0000", 2024-09-13T15:02:21.402 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.4499999999999998e-06 2024-09-13T15:02:21.402 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.402 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.402 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.402 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.331573+0000", 2024-09-13T15:02:21.402 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.024815953000000002 2024-09-13T15:02:21.402 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.402 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.402 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.402 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.331628+0000", 2024-09-13T15:02:21.402 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5719e-05 2024-09-13T15:02:21.402 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.402 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.403 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.403 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.365636+0000", 2024-09-13T15:02:21.403 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.034007605000000003 2024-09-13T15:02:21.403 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.403 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.403 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.403 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.365683+0000", 2024-09-13T15:02:21.403 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7058000000000002e-05 2024-09-13T15:02:21.403 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.403 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.403 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.403 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.403 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.404 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:212127 9.8 e339/336 9:1a82ac06:::benchmark_data_smithi096_256909_object92719:head v 339'13429, mlcod=317'7126)", 2024-09-13T15:02:21.404 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:16.344075+0000", 2024-09-13T15:02:21.404 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 65.012099741, 2024-09-13T15:02:21.404 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.070444038, 2024-09-13T15:02:21.404 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.404 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.404 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.404 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.404 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.404 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.404 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.344075+0000", 2024-09-13T15:02:21.404 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.404 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.404 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.405 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.405 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.344075+0000", 2024-09-13T15:02:21.405 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.405 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.405 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.405 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.405 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.344076+0000", 2024-09-13T15:02:21.405 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1340000000000001e-06 2024-09-13T15:02:21.405 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.405 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.405 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.405 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.344085+0000", 2024-09-13T15:02:21.405 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.1409999999999994e-06 2024-09-13T15:02:21.406 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.406 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.406 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.406 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.344105+0000", 2024-09-13T15:02:21.406 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9414000000000001e-05 2024-09-13T15:02:21.406 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.406 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.406 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.406 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.344113+0000", 2024-09-13T15:02:21.406 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.4810000000000006e-06 2024-09-13T15:02:21.406 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.406 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.406 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.406 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.355700+0000", 2024-09-13T15:02:21.406 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011586920000000001 2024-09-13T15:02:21.407 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.407 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.407 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.407 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.355728+0000", 2024-09-13T15:02:21.407 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7364000000000001e-05 2024-09-13T15:02:21.407 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.407 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.407 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.407 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.414467+0000", 2024-09-13T15:02:21.407 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.058739606999999999 2024-09-13T15:02:21.407 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.407 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.408 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.408 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.414519+0000", 2024-09-13T15:02:21.408 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.1977000000000001e-05 2024-09-13T15:02:21.408 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.408 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.408 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.408 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.408 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.408 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:212460 9.8 e339/336 9:1e553ac2:::benchmark_data_smithi096_256909_object93052:head v 339'13452, mlcod=317'7126)", 2024-09-13T15:02:21.408 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:16.725503+0000", 2024-09-13T15:02:21.408 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 64.630672164000003, 2024-09-13T15:02:21.408 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.074223681, 2024-09-13T15:02:21.408 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.409 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.409 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.409 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.409 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.409 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.409 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.725503+0000", 2024-09-13T15:02:21.409 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.409 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.409 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.409 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.409 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.725503+0000", 2024-09-13T15:02:21.409 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.409 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.409 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.410 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.410 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.725504+0000", 2024-09-13T15:02:21.410 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2780000000000001e-06 2024-09-13T15:02:21.410 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.410 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.410 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.410 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.725513+0000", 2024-09-13T15:02:21.410 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.4879999999999996e-06 2024-09-13T15:02:21.410 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.410 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.410 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.410 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.725534+0000", 2024-09-13T15:02:21.410 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0403000000000001e-05 2024-09-13T15:02:21.410 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.411 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.411 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.411 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.725543+0000", 2024-09-13T15:02:21.411 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2629999999999992e-06 2024-09-13T15:02:21.411 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.411 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.411 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.411 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.725632+0000", 2024-09-13T15:02:21.411 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9262e-05 2024-09-13T15:02:21.411 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.411 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.411 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.411 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.725675+0000", 2024-09-13T15:02:21.411 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.2202e-05 2024-09-13T15:02:21.412 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.412 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.412 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.412 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.799617+0000", 2024-09-13T15:02:21.412 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.073942020999999997 2024-09-13T15:02:21.412 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.412 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.412 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.412 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.799726+0000", 2024-09-13T15:02:21.412 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000109764 2024-09-13T15:02:21.412 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.412 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.412 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.413 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.413 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.413 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:212465 9.8 e339/336 9:16bd1a2c:::benchmark_data_smithi096_256909_object93057:head v 339'13453, mlcod=317'7126)", 2024-09-13T15:02:21.413 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:16.730740+0000", 2024-09-13T15:02:21.413 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 64.625434424000005, 2024-09-13T15:02:21.413 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.069046022999999998, 2024-09-13T15:02:21.413 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.413 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.413 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.413 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.413 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.413 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.413 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.730740+0000", 2024-09-13T15:02:21.413 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.414 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.414 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.414 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.414 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.730740+0000", 2024-09-13T15:02:21.414 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.414 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.414 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.414 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.414 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.730742+0000", 2024-09-13T15:02:21.414 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6190000000000001e-06 2024-09-13T15:02:21.415 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.415 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.415 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.415 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.730749+0000", 2024-09-13T15:02:21.415 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3220000000000004e-06 2024-09-13T15:02:21.415 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.415 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.415 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.415 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.730765+0000", 2024-09-13T15:02:21.415 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6073e-05 2024-09-13T15:02:21.415 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.415 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.415 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.416 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.730774+0000", 2024-09-13T15:02:21.416 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8650000000000007e-06 2024-09-13T15:02:21.416 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.416 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.416 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.416 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.738687+0000", 2024-09-13T15:02:21.416 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0079128029999999995 2024-09-13T15:02:21.416 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.416 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.416 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.416 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.738715+0000", 2024-09-13T15:02:21.416 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.8398000000000002e-05 2024-09-13T15:02:21.416 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.417 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.417 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.417 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.799729+0000", 2024-09-13T15:02:21.417 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.061013533000000002 2024-09-13T15:02:21.417 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.417 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.417 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.417 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.799786+0000", 2024-09-13T15:02:21.417 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7410000000000001e-05 2024-09-13T15:02:21.417 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.417 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.417 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.418 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.418 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.418 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:212471 9.8 e339/336 9:155aabec:::benchmark_data_smithi096_256909_object93063:head v 339'13454, mlcod=317'7126)", 2024-09-13T15:02:21.418 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:16.735404+0000", 2024-09-13T15:02:21.418 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 64.620770832000005, 2024-09-13T15:02:21.418 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.064440665999999994, 2024-09-13T15:02:21.418 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.418 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.418 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.418 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.418 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.418 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.418 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.735404+0000", 2024-09-13T15:02:21.418 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.419 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.419 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.419 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.419 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.735404+0000", 2024-09-13T15:02:21.419 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.419 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.419 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.419 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.419 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.735406+0000", 2024-09-13T15:02:21.419 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8059999999999999e-06 2024-09-13T15:02:21.419 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.419 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.419 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.419 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.735415+0000", 2024-09-13T15:02:21.420 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:21.421 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2709999999999998e-06 2024-09-13T15:02:21.421 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.421 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.421 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.421 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.735435+0000", 2024-09-13T15:02:21.421 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0352000000000001e-05 2024-09-13T15:02:21.421 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.422 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.422 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.422 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.735445+0000", 2024-09-13T15:02:21.422 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.9650000000000003e-06 2024-09-13T15:02:21.422 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.422 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.422 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.422 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.790348+0000", 2024-09-13T15:02:21.422 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.054903023000000002 2024-09-13T15:02:21.422 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.422 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.422 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.422 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.790375+0000", 2024-09-13T15:02:21.423 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6526e-05 2024-09-13T15:02:21.423 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.423 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.423 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.423 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.799788+0000", 2024-09-13T15:02:21.423 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0094135150000000008 2024-09-13T15:02:21.423 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.423 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.423 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.423 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:16.799845+0000", 2024-09-13T15:02:21.423 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6208000000000001e-05 2024-09-13T15:02:21.423 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.423 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.423 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.424 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.424 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.424 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:222083 9.8 e341/336 9:1dba5c20:::benchmark_data_smithi096_256909_object102675:head v 341'14021, mlcod=317'7126)", 2024-09-13T15:02:21.424 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:29.371767+0000", 2024-09-13T15:02:21.424 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 51.98440824, 2024-09-13T15:02:21.424 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.064751352999999998, 2024-09-13T15:02:21.424 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.424 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.424 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.424 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.424 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.424 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.424 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.371767+0000", 2024-09-13T15:02:21.425 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.425 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.425 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.425 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.425 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.371767+0000", 2024-09-13T15:02:21.425 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.425 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.425 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.425 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.425 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.371768+0000", 2024-09-13T15:02:21.425 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1319999999999999e-06 2024-09-13T15:02:21.425 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.425 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.426 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.426 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.371777+0000", 2024-09-13T15:02:21.426 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3309999999999993e-06 2024-09-13T15:02:21.426 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.426 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.426 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.426 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.371793+0000", 2024-09-13T15:02:21.426 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5474000000000001e-05 2024-09-13T15:02:21.426 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.426 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.426 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.426 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.371801+0000", 2024-09-13T15:02:21.426 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.9209999999999998e-06 2024-09-13T15:02:21.426 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.427 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.427 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.427 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.371853+0000", 2024-09-13T15:02:21.427 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.1987000000000002e-05 2024-09-13T15:02:21.427 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.427 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.427 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.427 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.371877+0000", 2024-09-13T15:02:21.427 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3841000000000001e-05 2024-09-13T15:02:21.427 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.427 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.427 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.427 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.436466+0000", 2024-09-13T15:02:21.428 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.064588396000000006 2024-09-13T15:02:21.428 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.428 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.428 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.428 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.436518+0000", 2024-09-13T15:02:21.428 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2271000000000001e-05 2024-09-13T15:02:21.428 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.428 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.428 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.428 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.428 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.428 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:222087 9.8 e341/336 9:147b9247:::benchmark_data_smithi096_256909_object102679:head v 341'14022, mlcod=317'7126)", 2024-09-13T15:02:21.428 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:29.376070+0000", 2024-09-13T15:02:21.429 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 51.980104673, 2024-09-13T15:02:21.429 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.060493911999999997, 2024-09-13T15:02:21.429 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.429 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.429 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.429 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.429 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.429 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.429 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.376070+0000", 2024-09-13T15:02:21.430 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.430 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.430 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.430 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.430 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.376070+0000", 2024-09-13T15:02:21.430 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.430 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.430 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.430 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.430 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.376072+0000", 2024-09-13T15:02:21.430 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6139999999999999e-06 2024-09-13T15:02:21.430 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.430 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.430 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.431 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.376080+0000", 2024-09-13T15:02:21.431 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.3110000000000006e-06 2024-09-13T15:02:21.431 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.431 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.431 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.431 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.376100+0000", 2024-09-13T15:02:21.431 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9517999999999998e-05 2024-09-13T15:02:21.431 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.431 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.431 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.431 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.376109+0000", 2024-09-13T15:02:21.431 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3190000000000001e-06 2024-09-13T15:02:21.431 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.431 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.432 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.432 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.380547+0000", 2024-09-13T15:02:21.432 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.004438025 2024-09-13T15:02:21.432 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.432 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.432 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.432 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.380580+0000", 2024-09-13T15:02:21.432 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.3009000000000002e-05 2024-09-13T15:02:21.432 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.432 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.432 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.432 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.436522+0000", 2024-09-13T15:02:21.432 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.055942200999999997 2024-09-13T15:02:21.433 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.433 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.433 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.433 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.436564+0000", 2024-09-13T15:02:21.433 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1915000000000001e-05 2024-09-13T15:02:21.433 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.433 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.433 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.433 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.433 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.433 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:222498 9.8 e341/336 9:1b2158f2:::benchmark_data_smithi096_256909_object103090:head v 341'14052, mlcod=317'7126)", 2024-09-13T15:02:21.433 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:29.816082+0000", 2024-09-13T15:02:21.433 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 51.540093116000001, 2024-09-13T15:02:21.434 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.064291547000000004, 2024-09-13T15:02:21.434 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.434 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.434 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.434 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.434 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.434 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.434 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.816082+0000", 2024-09-13T15:02:21.434 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.434 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.434 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.434 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.435 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.816082+0000", 2024-09-13T15:02:21.435 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.435 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.435 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.435 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.435 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.816083+0000", 2024-09-13T15:02:21.435 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3659999999999999e-06 2024-09-13T15:02:21.435 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.435 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.435 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.435 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.816092+0000", 2024-09-13T15:02:21.435 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6810000000000004e-06 2024-09-13T15:02:21.435 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.435 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.436 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.436 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.816112+0000", 2024-09-13T15:02:21.436 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.0381e-05 2024-09-13T15:02:21.436 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.436 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.436 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.436 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.816121+0000", 2024-09-13T15:02:21.436 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.0599999999999997e-06 2024-09-13T15:02:21.436 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.436 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.436 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.436 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.832745+0000", 2024-09-13T15:02:21.436 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.016624135000000002 2024-09-13T15:02:21.437 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.437 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.437 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.437 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.832780+0000", 2024-09-13T15:02:21.437 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4499999999999998e-05 2024-09-13T15:02:21.437 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.437 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.437 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.437 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.880299+0000", 2024-09-13T15:02:21.437 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.047519485 2024-09-13T15:02:21.437 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.437 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.438 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.438 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:29.880373+0000", 2024-09-13T15:02:21.438 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3938999999999998e-05 2024-09-13T15:02:21.438 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.438 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.438 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.438 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.438 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.438 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:222783 9.8 e341/336 9:1cd68920:::benchmark_data_smithi096_256909_object103375:head v 341'14078, mlcod=317'7126)", 2024-09-13T15:02:21.438 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:30.129691+0000", 2024-09-13T15:02:21.438 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 51.226483344999998, 2024-09-13T15:02:21.438 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.063561644, 2024-09-13T15:02:21.438 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.438 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.439 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.439 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.439 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.439 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.439 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.129691+0000", 2024-09-13T15:02:21.439 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.439 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.439 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.439 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.439 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.129691+0000", 2024-09-13T15:02:21.439 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.439 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.439 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.440 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.440 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.129693+0000", 2024-09-13T15:02:21.440 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2049999999999999e-06 2024-09-13T15:02:21.440 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.440 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.440 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.440 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.129700+0000", 2024-09-13T15:02:21.440 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.2150000000000004e-06 2024-09-13T15:02:21.440 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.440 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.440 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.440 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.129716+0000", 2024-09-13T15:02:21.440 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6388e-05 2024-09-13T15:02:21.440 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.441 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.441 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.441 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.129723+0000", 2024-09-13T15:02:21.441 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5069999999999997e-06 2024-09-13T15:02:21.441 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.441 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.441 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.441 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.160274+0000", 2024-09-13T15:02:21.441 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.030551373 2024-09-13T15:02:21.441 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.441 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.441 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.441 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.160300+0000", 2024-09-13T15:02:21.441 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6279999999999999e-05 2024-09-13T15:02:21.442 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.442 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.442 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.442 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.193206+0000", 2024-09-13T15:02:21.442 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.032905200000000003 2024-09-13T15:02:21.442 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.442 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.442 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.442 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.193253+0000", 2024-09-13T15:02:21.442 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.7475999999999997e-05 2024-09-13T15:02:21.442 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.442 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.443 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.443 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.443 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.443 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:222831 9.8 e341/336 9:121c2894:::benchmark_data_smithi096_256909_object103423:head v 341'14083, mlcod=317'7126)", 2024-09-13T15:02:21.443 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:30.182988+0000", 2024-09-13T15:02:21.443 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 51.173187165000002, 2024-09-13T15:02:21.443 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.062019074, 2024-09-13T15:02:21.443 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.443 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.443 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.443 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.443 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.443 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.443 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.182988+0000", 2024-09-13T15:02:21.444 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.444 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.444 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.444 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.444 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.182988+0000", 2024-09-13T15:02:21.444 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.444 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.444 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.444 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.444 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.182989+0000", 2024-09-13T15:02:21.444 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3099999999999996e-07 2024-09-13T15:02:21.444 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.444 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.444 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.444 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.182996+0000", 2024-09-13T15:02:21.445 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.4869999999999996e-06 2024-09-13T15:02:21.445 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.445 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.445 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.445 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.183011+0000", 2024-09-13T15:02:21.445 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5379999999999998e-05 2024-09-13T15:02:21.445 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.445 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.445 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.445 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.183020+0000", 2024-09-13T15:02:21.445 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.439e-06 2024-09-13T15:02:21.445 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.445 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.445 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.202581+0000", 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.019560845 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.202634+0000", 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3499999999999999e-05 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.244951+0000", 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.042317129000000002 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.446 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:30.245007+0000", 2024-09-13T15:02:21.447 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.5362999999999998e-05 2024-09-13T15:02:21.447 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.447 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.447 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.447 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.447 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.447 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:228764 9.8 e343/336 9:13fb469e:::benchmark_data_smithi096_256909_object109356:head v 343'14423, mlcod=317'7126)", 2024-09-13T15:02:21.447 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:37.185175+0000", 2024-09-13T15:02:21.447 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 44.170999805000001, 2024-09-13T15:02:21.447 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.074881649999999994, 2024-09-13T15:02:21.447 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.447 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.447 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.448 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.448 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.448 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.448 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:37.185175+0000", 2024-09-13T15:02:21.448 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.448 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.448 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.448 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.448 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:37.185175+0000", 2024-09-13T15:02:21.448 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.448 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.448 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.448 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.449 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:37.185175+0000", 2024-09-13T15:02:21.449 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.15e-07 2024-09-13T15:02:21.449 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.449 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.449 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.449 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:37.185179+0000", 2024-09-13T15:02:21.449 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.2040000000000002e-06 2024-09-13T15:02:21.449 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.449 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.449 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.449 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:37.185184+0000", 2024-09-13T15:02:21.449 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7629999999999997e-06 2024-09-13T15:02:21.449 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.450 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.450 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.450 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:37.185188+0000", 2024-09-13T15:02:21.450 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.092e-06 2024-09-13T15:02:21.450 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.450 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.450 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.450 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:37.209205+0000", 2024-09-13T15:02:21.450 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.024016289 2024-09-13T15:02:21.450 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.450 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.450 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.451 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:37.209232+0000", 2024-09-13T15:02:21.451 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7056999999999999e-05 2024-09-13T15:02:21.451 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.451 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.451 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.451 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:37.259999+0000", 2024-09-13T15:02:21.451 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.050767251999999999 2024-09-13T15:02:21.451 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.451 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.451 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.451 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:37.260057+0000", 2024-09-13T15:02:21.451 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7677999999999999e-05 2024-09-13T15:02:21.451 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.451 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.452 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.452 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.452 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.452 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:229594 9.8 e343/336 9:10ed88ce:::benchmark_data_smithi096_256909_object110186:head v 343'14476, mlcod=317'7126)", 2024-09-13T15:02:21.452 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:38.098391+0000", 2024-09-13T15:02:21.452 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 43.257784030000003, 2024-09-13T15:02:21.452 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.062191822000000001, 2024-09-13T15:02:21.452 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:02:21.452 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:02:21.452 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:21.452 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:02:21.452 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.452 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:02:21.453 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:38.098391+0000", 2024-09-13T15:02:21.453 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.453 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.453 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.453 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:02:21.453 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:38.098391+0000", 2024-09-13T15:02:21.453 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:02:21.453 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.453 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.453 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:02:21.453 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:38.098392+0000", 2024-09-13T15:02:21.453 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0240000000000001e-06 2024-09-13T15:02:21.453 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.453 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.453 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:02:21.454 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:38.098399+0000", 2024-09-13T15:02:21.454 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9120000000000001e-06 2024-09-13T15:02:21.454 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.454 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.454 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:02:21.454 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:38.098413+0000", 2024-09-13T15:02:21.454 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4472e-05 2024-09-13T15:02:21.454 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.454 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.454 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:02:21.454 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:38.098420+0000", 2024-09-13T15:02:21.454 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1180000000000003e-06 2024-09-13T15:02:21.454 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.454 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.454 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:02:21.455 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:38.108289+0000", 2024-09-13T15:02:21.455 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0098690970000000003 2024-09-13T15:02:21.455 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.455 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.455 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:02:21.455 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:38.108314+0000", 2024-09-13T15:02:21.455 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4647999999999998e-05 2024-09-13T15:02:21.455 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.455 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.455 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:02:21.455 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:38.160466+0000", 2024-09-13T15:02:21.455 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.052151492000000001 2024-09-13T15:02:21.455 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:02:21.455 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:02:21.456 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:02:21.456 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:38.160583+0000", 2024-09-13T15:02:21.456 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011705900000000001 2024-09-13T15:02:21.456 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.456 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.456 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.456 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:02:21.456 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:02:21.456 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T15:02:21.479 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:21.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:21.581 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:21.592 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:02:21.605 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:21.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:21.707 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:21.729 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:21.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:21.831 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:21.853 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:21.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:21.868 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'unique_pool_4' removed 2024-09-13T15:02:21.881 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:21 smithi096 ceph-mon[98901]: pgmap v1162: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 13027/84 objects degraded (15508.333%); 0 B/s, 253 objects/s recovering 2024-09-13T15:02:21.881 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:21.881 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:21.881 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:21 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4207592663' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:02:21.881 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:21 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:02:21.882 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:21 smithi096 ceph-mon[102928]: pgmap v1162: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 13027/84 objects degraded (15508.333%); 0 B/s, 253 objects/s recovering 2024-09-13T15:02:21.882 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:21.882 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:21.882 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:21 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4207592663' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:02:21.882 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:21 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:02:21.954 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:21.978 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:21.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:22.079 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:22.101 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:22.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:22.203 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:21 smithi204 ceph-mon[87071]: pgmap v1162: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 113 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 13027/84 objects degraded (15508.333%); 0 B/s, 253 objects/s recovering 2024-09-13T15:02:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:21 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4207592663' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:02:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:21 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:02:22.227 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:22.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:22.329 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:22.373 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:22.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:22.475 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:22.497 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:22.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:22.598 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:22.620 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:22.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:22.717 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T15:02:22.718 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T15:02:22.719 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2024-09-13T15:02:22.719 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_5 16 2024-09-13T15:02:22.721 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:22.743 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:22.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:22.845 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:22.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.870 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:22.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:22.971 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:22.992 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:22.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:23.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:22 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:02:23.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:22 smithi096 ceph-mon[98901]: osdmap e350: 8 total, 8 up, 7 in 2024-09-13T15:02:23.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:22 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-13T15:02:23.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:22 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:02:23.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:22 smithi096 ceph-mon[102928]: osdmap e350: 8 total, 8 up, 7 in 2024-09-13T15:02:23.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:22 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-13T15:02:23.094 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:23.114 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:23.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:23.216 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:23.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:22 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:02:23.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:22 smithi204 ceph-mon[87071]: osdmap e350: 8 total, 8 up, 7 in 2024-09-13T15:02:23.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:22 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-13T15:02:23.237 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:23.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:23.339 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:23.365 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:23.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:23.467 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:23.493 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:23.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:23.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:23.619 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:23.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:23.720 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:23.746 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:23.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:23.848 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:24.005 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:24.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:24.107 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:24.132 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:24.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:24.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:23 smithi204 ceph-mon[87071]: pgmap v1164: 103 pgs: 103 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:24.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:23 smithi204 ceph-mon[87071]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13027/84 objects degraded (15508.333%), 3 pgs degraded, 5 pgs undersized) 2024-09-13T15:02:24.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:23 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-09-13T15:02:24.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:23 smithi204 ceph-mon[87071]: osdmap e351: 8 total, 8 up, 7 in 2024-09-13T15:02:24.234 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:24.259 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:24.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:24.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:23 smithi096 ceph-mon[98901]: pgmap v1164: 103 pgs: 103 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:24.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:23 smithi096 ceph-mon[98901]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13027/84 objects degraded (15508.333%), 3 pgs degraded, 5 pgs undersized) 2024-09-13T15:02:24.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:23 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-09-13T15:02:24.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:23 smithi096 ceph-mon[98901]: osdmap e351: 8 total, 8 up, 7 in 2024-09-13T15:02:24.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:23 smithi096 ceph-mon[102928]: pgmap v1164: 103 pgs: 103 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:24.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:23 smithi096 ceph-mon[102928]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13027/84 objects degraded (15508.333%), 3 pgs degraded, 5 pgs undersized) 2024-09-13T15:02:24.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:23 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-09-13T15:02:24.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:23 smithi096 ceph-mon[102928]: osdmap e351: 8 total, 8 up, 7 in 2024-09-13T15:02:24.361 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:24.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.388 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:24.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:24.488 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:24.514 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:24.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:24.616 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:24.642 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:24.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:24.744 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:24.769 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:24.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:24.871 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:24.896 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:24.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:24.998 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:25.024 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:25.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:25.126 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:25.150 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:25.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:25.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:24 smithi204 ceph-mon[87071]: osdmap e352: 8 total, 8 up, 7 in 2024-09-13T15:02:25.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:24 smithi204 ceph-mon[87071]: 1.16 scrub starts 2024-09-13T15:02:25.252 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:25.278 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:25.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:25.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:24 smithi096 ceph-mon[98901]: osdmap e352: 8 total, 8 up, 7 in 2024-09-13T15:02:25.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:24 smithi096 ceph-mon[98901]: 1.16 scrub starts 2024-09-13T15:02:25.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:24 smithi096 ceph-mon[102928]: osdmap e352: 8 total, 8 up, 7 in 2024-09-13T15:02:25.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:24 smithi096 ceph-mon[102928]: 1.16 scrub starts 2024-09-13T15:02:25.379 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:25.405 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:25.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:25.507 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:25.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.531 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:25.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:25.632 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:25.658 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:25.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:25.760 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:25.785 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:25.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:25.887 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:25.913 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:25.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:26.015 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:26.041 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:26.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:26.142 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:26.167 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:26.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:26.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:25 smithi204 ceph-mon[87071]: pgmap v1167: 103 pgs: 1 activating, 102 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:26.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:25 smithi204 ceph-mon[87071]: 1.19 scrub starts 2024-09-13T15:02:26.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:25 smithi204 ceph-mon[87071]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2024-09-13T15:02:26.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:25 smithi204 ceph-mon[87071]: 1.1d scrub starts 2024-09-13T15:02:26.268 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:26.293 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:26.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:26.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:25 smithi096 ceph-mon[98901]: pgmap v1167: 103 pgs: 1 activating, 102 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:26.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:25 smithi096 ceph-mon[98901]: 1.19 scrub starts 2024-09-13T15:02:26.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:25 smithi096 ceph-mon[98901]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2024-09-13T15:02:26.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:25 smithi096 ceph-mon[98901]: 1.1d scrub starts 2024-09-13T15:02:26.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:25 smithi096 ceph-mon[102928]: pgmap v1167: 103 pgs: 1 activating, 102 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:26.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:25 smithi096 ceph-mon[102928]: 1.19 scrub starts 2024-09-13T15:02:26.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:25 smithi096 ceph-mon[102928]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2024-09-13T15:02:26.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:25 smithi096 ceph-mon[102928]: 1.1d scrub starts 2024-09-13T15:02:26.395 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:26.420 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:26.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:26.509 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:26.519 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T15:02:26.522 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:26.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.584 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:26.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:26.685 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:26.711 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:26.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:26.813 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:26.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.835 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:26.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:26.936 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:26.957 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:26.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:27.059 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:27.080 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:27.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:27.182 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:27.203 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:27.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:27.305 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:27.326 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:27.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:27.428 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:27.449 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:27.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:27.551 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:27.573 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:27.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:27.675 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:27.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.697 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:27.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:27.798 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:27.819 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:27.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:27.921 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:27.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.943 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:27.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:27.947 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:27 smithi096 ceph-mon[98901]: pgmap v1168: 103 pgs: 1 activating, 102 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:27.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:27 smithi096 ceph-mon[102928]: pgmap v1168: 103 pgs: 1 activating, 102 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:28.044 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:28.070 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:28.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:28.172 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:28.193 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:28.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:28.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:27 smithi204 ceph-mon[87071]: pgmap v1168: 103 pgs: 1 activating, 102 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:28.295 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:28.321 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:28.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:28.325 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:02:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:02:27] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-13T15:02:28.425 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:28.450 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:28.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:28.552 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:28.572 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:28.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:28.666 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:28.669 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:28.671 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:28.675 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:28.701 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:28.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:28.803 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:28.825 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:28.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:28.880 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:02:28.880 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:02:28.892 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:28.903 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:02:28.903 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:02:28.925 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:02:28.925 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:02:28.925 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:28 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4035467418' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:28.925 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:28 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4035467418' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:28.927 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:28.941 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:02:28.941 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:02:28.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.954 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:28.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:28.958 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:02:28.961 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:02:28.974 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:02:28.974 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:02:28.990 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:02:28.990 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:02:29.005 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:02:29.005 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:02:29.055 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:29.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.081 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:29.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:29.182 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:29.207 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:29.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:28 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4035467418' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:29.308 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:29.331 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:29.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:29.433 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:29.459 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:29.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:29.561 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:29.583 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:29.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:29.685 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:29.706 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:29.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:29.807 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:29.828 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:29.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:29.930 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:29.956 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:29.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:30.034 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:29 smithi096 ceph-mon[98901]: pgmap v1169: 103 pgs: 1 activating, 102 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 5.4 KiB/s rd, 12 KiB/s wr, 3 op/s; 45 KiB/s, 0 objects/s recovering 2024-09-13T15:02:30.034 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:29 smithi096 ceph-mon[102928]: pgmap v1169: 103 pgs: 1 activating, 102 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 5.4 KiB/s rd, 12 KiB/s wr, 3 op/s; 45 KiB/s, 0 objects/s recovering 2024-09-13T15:02:30.058 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:30.079 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:30.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:30.181 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:30.207 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:30.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:30.213 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:29 smithi204 ceph-mon[87071]: pgmap v1169: 103 pgs: 1 activating, 102 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 5.4 KiB/s rd, 12 KiB/s wr, 3 op/s; 45 KiB/s, 0 objects/s recovering 2024-09-13T15:02:30.309 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:30.330 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:30.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:30.431 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:30.458 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:30.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:02:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:02:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:02:30.560 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:30.585 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:30.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:30.687 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:30.708 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:30.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:30.810 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:30.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.837 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:30.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:30.938 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:30.960 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:30.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:31.063 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:31.088 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:31.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:31.107 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:30 smithi096 ceph-mon[98901]: 1.19 scrub starts 2024-09-13T15:02:31.107 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:30 smithi096 ceph-mon[98901]: 1.16 scrub starts 2024-09-13T15:02:31.107 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:30 smithi096 ceph-mon[102928]: 1.19 scrub starts 2024-09-13T15:02:31.107 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:30 smithi096 ceph-mon[102928]: 1.16 scrub starts 2024-09-13T15:02:31.190 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:31.213 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:31.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:31.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:30 smithi204 ceph-mon[87071]: 1.19 scrub starts 2024-09-13T15:02:31.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:30 smithi204 ceph-mon[87071]: 1.16 scrub starts 2024-09-13T15:02:31.314 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:31.340 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:31.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:31.442 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:31.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.469 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:31.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:31.572 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:31.596 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:31.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:31.698 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:31.720 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:31.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:31.821 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:31.842 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:31.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:31.944 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:31.970 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:31.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:32.072 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:32.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:31 smithi096 ceph-mon[102928]: pgmap v1170: 103 pgs: 2 active+clean+scrubbing, 101 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4.2 KiB/s rd, 9.2 KiB/s wr, 2 op/s; 35 KiB/s, 0 objects/s recovering 2024-09-13T15:02:32.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:31 smithi096 ceph-mon[102928]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2024-09-13T15:02:32.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:31 smithi096 ceph-mon[102928]: 1.1d scrub starts 2024-09-13T15:02:32.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:31 smithi096 ceph-mon[98901]: pgmap v1170: 103 pgs: 2 active+clean+scrubbing, 101 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4.2 KiB/s rd, 9.2 KiB/s wr, 2 op/s; 35 KiB/s, 0 objects/s recovering 2024-09-13T15:02:32.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:31 smithi096 ceph-mon[98901]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2024-09-13T15:02:32.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:31 smithi096 ceph-mon[98901]: 1.1d scrub starts 2024-09-13T15:02:32.093 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:32.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:32.195 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:32.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:31 smithi204 ceph-mon[87071]: pgmap v1170: 103 pgs: 2 active+clean+scrubbing, 101 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4.2 KiB/s rd, 9.2 KiB/s wr, 2 op/s; 35 KiB/s, 0 objects/s recovering 2024-09-13T15:02:32.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:31 smithi204 ceph-mon[87071]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2024-09-13T15:02:32.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:31 smithi204 ceph-mon[87071]: 1.1d scrub starts 2024-09-13T15:02:32.242 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:32.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:32.344 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:32.365 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:32.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:32.467 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:32.490 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:32.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:32.592 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:32.613 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:32.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:32.715 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:32.737 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:32.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:32.839 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:32.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.861 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:32.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:32.962 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:32.984 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:32.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:33.086 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:33.108 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:33.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:33.210 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:33.236 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:33.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:33.338 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:33.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.361 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:33.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:33.462 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:33.488 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:33.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:33.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:33 smithi096 ceph-mon[98901]: pgmap v1171: 103 pgs: 2 active+clean+scrubbing, 101 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 3.7 KiB/s rd, 8.0 KiB/s wr, 2 op/s; 30 KiB/s, 0 objects/s recovering 2024-09-13T15:02:33.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:33.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:33 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:33.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:33 smithi096 ceph-mon[102928]: pgmap v1171: 103 pgs: 2 active+clean+scrubbing, 101 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 3.7 KiB/s rd, 8.0 KiB/s wr, 2 op/s; 30 KiB/s, 0 objects/s recovering 2024-09-13T15:02:33.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:33.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:33 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:33.590 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:33.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.617 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:33.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:33.718 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:33.742 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:33.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:33.844 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:33.867 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:33.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:33.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:33 smithi204 ceph-mon[87071]: pgmap v1171: 103 pgs: 2 active+clean+scrubbing, 101 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 3.7 KiB/s rd, 8.0 KiB/s wr, 2 op/s; 30 KiB/s, 0 objects/s recovering 2024-09-13T15:02:33.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:33.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:33 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:33.969 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:33.993 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:33.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:34.095 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:34.120 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:34.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:34.222 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:34.248 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:34.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:34.349 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:34.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.372 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:34.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:34.473 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:34.499 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:34.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:34.601 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:34.626 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:34.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:34.728 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:34.751 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:34.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:34.852 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:34.875 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:34.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:34.976 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:35.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.000 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:35.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:35.037 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:02:35.101 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:35.124 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:35.225 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:35.249 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:35.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:35.350 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:35.373 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:35.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:35.475 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:35.499 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:35.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:35.596 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T15:02:35.601 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:35.604 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T15:02:35.624 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:35.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:35.726 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:35.748 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:35.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:35.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:35 smithi096 ceph-mon[98901]: pgmap v1172: 103 pgs: 2 active+clean+scrubbing, 101 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 3.3 KiB/s rd, 7.2 KiB/s wr, 1 op/s; 27 KiB/s, 0 objects/s recovering 2024-09-13T15:02:35.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:02:35.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:02:35.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:35.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:35 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3231843646' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:02:35.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:35 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1073628755' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:35.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:35 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:35.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:35 smithi096 ceph-mon[102928]: pgmap v1172: 103 pgs: 2 active+clean+scrubbing, 101 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 3.3 KiB/s rd, 7.2 KiB/s wr, 1 op/s; 27 KiB/s, 0 objects/s recovering 2024-09-13T15:02:35.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:02:35.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:02:35.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:35.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:35 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3231843646' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:02:35.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:35 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1073628755' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:35.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:35 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:35.850 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:35.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.876 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:35.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:35 smithi204 ceph-mon[87071]: pgmap v1172: 103 pgs: 2 active+clean+scrubbing, 101 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 3.3 KiB/s rd, 7.2 KiB/s wr, 1 op/s; 27 KiB/s, 0 objects/s recovering 2024-09-13T15:02:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:02:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:02:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:35 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3231843646' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:02:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:35 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1073628755' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:35 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:35.977 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:36.001 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:36.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:36.103 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:36.126 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:36.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:36.228 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:36.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.250 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:36.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:36.351 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:36.376 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:36.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:36.478 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:36.502 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:36.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:36.604 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:36.628 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:36.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:36.729 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:36.753 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:36.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:36.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[98901]: 1.19 scrub starts 2024-09-13T15:02:36.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:36.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3231843646' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:02:36.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:02:36.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[98901]: osdmap e353: 8 total, 8 up, 7 in 2024-09-13T15:02:36.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:36.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:36.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[98901]: 1.16 scrub starts 2024-09-13T15:02:36.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:36.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[102928]: 1.19 scrub starts 2024-09-13T15:02:36.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:36.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3231843646' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:02:36.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:02:36.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[102928]: osdmap e353: 8 total, 8 up, 7 in 2024-09-13T15:02:36.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:36.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:36.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[102928]: 1.16 scrub starts 2024-09-13T15:02:36.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:36.855 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:36.878 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:36.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:36.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:36 smithi204 ceph-mon[87071]: 1.19 scrub starts 2024-09-13T15:02:36.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:36 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:36.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:36 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3231843646' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:02:36.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:36 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:02:36.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:36 smithi204 ceph-mon[87071]: osdmap e353: 8 total, 8 up, 7 in 2024-09-13T15:02:36.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:36.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:36.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:36 smithi204 ceph-mon[87071]: 1.16 scrub starts 2024-09-13T15:02:36.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:36.980 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:37.002 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:37.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:37.104 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:37.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.127 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:37.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:37.228 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:37.233 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T15:02:37.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.251 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:37.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:37.352 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:37.374 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:37.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:37.476 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:37.499 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:37.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:37.601 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:37.622 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:37.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:37.724 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:37.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.746 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:37.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:37.847 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:37.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.874 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:37.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:37.975 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:37.997 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:37.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:38.099 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:38.121 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:38.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:38.206 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:37 smithi096 ceph-mon[98901]: pgmap v1174: 103 pgs: 3 active+clean+scrubbing, 100 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 3.4 KiB/s rd, 7.4 KiB/s wr, 1 op/s; 28 KiB/s, 0 objects/s recovering 2024-09-13T15:02:38.206 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:37 smithi096 ceph-mon[98901]: 1.1d scrub starts 2024-09-13T15:02:38.207 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:37 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 0]} : dispatch 2024-09-13T15:02:38.207 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:37 smithi096 ceph-mon[102928]: pgmap v1174: 103 pgs: 3 active+clean+scrubbing, 100 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 3.4 KiB/s rd, 7.4 KiB/s wr, 1 op/s; 28 KiB/s, 0 objects/s recovering 2024-09-13T15:02:38.207 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:37 smithi096 ceph-mon[102928]: 1.1d scrub starts 2024-09-13T15:02:38.207 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:37 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 0]} : dispatch 2024-09-13T15:02:38.207 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:02:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:02:37] "GET /metrics HTTP/1.1" 200 34181 "" "Prometheus/2.51.0" 2024-09-13T15:02:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:37 smithi204 ceph-mon[87071]: pgmap v1174: 103 pgs: 3 active+clean+scrubbing, 100 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 3.4 KiB/s rd, 7.4 KiB/s wr, 1 op/s; 28 KiB/s, 0 objects/s recovering 2024-09-13T15:02:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:37 smithi204 ceph-mon[87071]: 1.1d scrub starts 2024-09-13T15:02:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:37 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 0]} : dispatch 2024-09-13T15:02:38.223 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:38.245 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:38.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:38.347 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:38.369 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:38.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:38.471 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:38.492 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:38.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:38.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:38.617 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:38.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:38.622 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:38.718 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:38.743 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:38.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:38.845 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:38.849 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:02:38.850 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:02:38.867 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'unique_pool_5' created 2024-09-13T15:02:38.867 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:38.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:38.969 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:38.994 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:38.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:39.096 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:39.102 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:38 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 0]}]': finished 2024-09-13T15:02:39.102 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:38 smithi096 ceph-mon[98901]: osdmap e354: 8 total, 8 up, 7 in 2024-09-13T15:02:39.102 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:38 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/811894238' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T15:02:39.102 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:38 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/811894238' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-13T15:02:39.102 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:38 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 0]}]': finished 2024-09-13T15:02:39.103 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:38 smithi096 ceph-mon[102928]: osdmap e354: 8 total, 8 up, 7 in 2024-09-13T15:02:39.103 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:38 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/811894238' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T15:02:39.103 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:38 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/811894238' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-13T15:02:39.120 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:39.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:38 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 0]}]': finished 2024-09-13T15:02:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:38 smithi204 ceph-mon[87071]: osdmap e354: 8 total, 8 up, 7 in 2024-09-13T15:02:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:38 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/811894238' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T15:02:39.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:38 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/811894238' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-13T15:02:39.223 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:39.244 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:39.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:39.346 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:39.372 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:39.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:39.474 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:39.495 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:39.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:39.597 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:39.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.620 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:39.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:39.721 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:39.742 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:39.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:39.844 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:39.870 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:39.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:39.973 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:39.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.995 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:39.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:40.003 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:02:40.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:39 smithi096 ceph-mon[98901]: pgmap v1176: 103 pgs: 3 active+clean+scrubbing, 100 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:40.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:39 smithi096 ceph-mon[98901]: osdmap e355: 8 total, 8 up, 7 in 2024-09-13T15:02:40.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:39 smithi096 ceph-mon[98901]: osdmap e356: 8 total, 8 up, 7 in 2024-09-13T15:02:40.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:39 smithi096 ceph-mon[102928]: pgmap v1176: 103 pgs: 3 active+clean+scrubbing, 100 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:40.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:39 smithi096 ceph-mon[102928]: osdmap e355: 8 total, 8 up, 7 in 2024-09-13T15:02:40.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:39 smithi096 ceph-mon[102928]: osdmap e356: 8 total, 8 up, 7 in 2024-09-13T15:02:40.096 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:40.117 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:40.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:40.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:39 smithi204 ceph-mon[87071]: pgmap v1176: 103 pgs: 3 active+clean+scrubbing, 100 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:40.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:39 smithi204 ceph-mon[87071]: osdmap e355: 8 total, 8 up, 7 in 2024-09-13T15:02:40.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:39 smithi204 ceph-mon[87071]: osdmap e356: 8 total, 8 up, 7 in 2024-09-13T15:02:40.219 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:40.239 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:40.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:40.341 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:40.366 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:40.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:40.467 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:02:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:02:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:02:40.491 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:40.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:40.555 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2024-09-13T15:02:40.593 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:40.619 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:40.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:40.721 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:40.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.743 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:40.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:40.844 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:40.869 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:40.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:40.971 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:40.992 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:40.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:41.037 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:41.043 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T15:02:41.095 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:41.120 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:41.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:41.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:40 smithi204 ceph-mon[87071]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:02:41.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:40 smithi204 ceph-mon[87071]: osdmap e357: 8 total, 8 up, 7 in 2024-09-13T15:02:41.223 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:41.250 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:41.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:41.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:40 smithi096 ceph-mon[98901]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:02:41.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:40 smithi096 ceph-mon[98901]: osdmap e357: 8 total, 8 up, 7 in 2024-09-13T15:02:41.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:40 smithi096 ceph-mon[102928]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:02:41.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:40 smithi096 ceph-mon[102928]: osdmap e357: 8 total, 8 up, 7 in 2024-09-13T15:02:41.351 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:41.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.374 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:41.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:41.475 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:41.499 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:41.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:41.598 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:41.601 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:41.625 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:41.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:41.727 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:41.751 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:41.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:41.853 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:41.877 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:41.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:41.979 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:42.003 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:42.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:42.105 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:42.128 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:42.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:42.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:41 smithi204 ceph-mon[87071]: pgmap v1179: 119 pgs: 4 creating+peering, 12 unknown, 103 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:42.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:41 smithi204 ceph-mon[87071]: 1.16 scrub starts 2024-09-13T15:02:42.230 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:42.255 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:42.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:42.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:41 smithi096 ceph-mon[102928]: pgmap v1179: 119 pgs: 4 creating+peering, 12 unknown, 103 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:42.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:41 smithi096 ceph-mon[102928]: 1.16 scrub starts 2024-09-13T15:02:42.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:41 smithi096 ceph-mon[98901]: pgmap v1179: 119 pgs: 4 creating+peering, 12 unknown, 103 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:42.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:41 smithi096 ceph-mon[98901]: 1.16 scrub starts 2024-09-13T15:02:42.357 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:42.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.384 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:42.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:42.485 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:42.509 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:42.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:42.611 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:42.633 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:42.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:42.735 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:42.761 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:42.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:42.863 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:42.889 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:42.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:42.991 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:43.015 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:43.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:43.117 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:43.140 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:43.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:43.206 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:42 smithi096 ceph-mon[98901]: 1.19 scrub starts 2024-09-13T15:02:43.206 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:42 smithi096 ceph-mon[98901]: 1.1d scrub starts 2024-09-13T15:02:43.206 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:42 smithi096 ceph-mon[98901]: pgmap v1181: 119 pgs: 4 creating+peering, 115 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:43.206 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:42 smithi096 ceph-mon[102928]: 1.19 scrub starts 2024-09-13T15:02:43.206 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:42 smithi096 ceph-mon[102928]: 1.1d scrub starts 2024-09-13T15:02:43.206 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:42 smithi096 ceph-mon[102928]: pgmap v1181: 119 pgs: 4 creating+peering, 115 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:43.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:42 smithi204 ceph-mon[87071]: 1.19 scrub starts 2024-09-13T15:02:43.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:42 smithi204 ceph-mon[87071]: 1.1d scrub starts 2024-09-13T15:02:43.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:42 smithi204 ceph-mon[87071]: pgmap v1181: 119 pgs: 4 creating+peering, 115 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:43.242 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:43.265 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:43.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:43.367 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:43.390 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:43.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:43.412 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:43.492 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:43.515 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:43.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:43.617 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:43.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.640 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:43.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:43.741 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:43.762 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:43.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:43.863 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:43.889 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:43.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:43.991 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:44.012 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:44.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:44.114 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:44.135 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:44.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:44.237 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:44.258 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:44.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:44.360 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:44.382 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:44.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:44.484 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:44.510 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:44.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:44.524 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:02:44.524 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:02:44.529 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:02:44.529 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:02:44.545 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:02:44.545 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:02:44.561 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:02:44.561 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:02:44.577 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:02:44.577 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:02:44.593 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:02:44.593 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:02:44.609 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:02:44.609 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:02:44.611 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:44.625 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:02:44.625 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:02:44.636 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:44.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:44.738 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:44.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.761 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:44.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:44.862 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:44.883 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:44.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:44.985 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:45.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.008 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:45.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:45.109 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:45.130 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:45.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:45.232 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:45.258 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:45.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:45.360 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:45.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.383 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:45.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:45.484 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:45.505 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:45 smithi096 ceph-mon[98901]: pgmap v1182: 119 pgs: 1 active+clean+scrubbing, 3 creating+peering, 115 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:45.505 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:45 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2896378446' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:45.505 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:45 smithi096 ceph-mon[102928]: pgmap v1182: 119 pgs: 1 active+clean+scrubbing, 3 creating+peering, 115 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:45.506 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:45 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2896378446' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:45.509 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:45.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:45.611 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:45.636 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:45.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:45.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:45 smithi204 ceph-mon[87071]: pgmap v1182: 119 pgs: 1 active+clean+scrubbing, 3 creating+peering, 115 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:45.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:45 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2896378446' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:45.737 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:45.762 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:45.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:45.863 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:45.887 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:45.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:45.989 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:46.012 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:46.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:46.113 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:46.136 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:46.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:46.237 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:46.260 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:46.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:46.362 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:46.384 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:46.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:46.486 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:46.508 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:46.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:46.610 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:46.633 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:46.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:46.735 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:46.756 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:46.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:46.858 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:46.881 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:46.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:46.983 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:47.006 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:47.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:47.108 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:47.130 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:47.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:47.228 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T15:02:47.232 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:47.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.258 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:47.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:47.359 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:47.382 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:47.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:47.484 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:47.503 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:47 smithi096 ceph-mon[98901]: 1.16 scrub starts 2024-09-13T15:02:47.503 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:47 smithi096 ceph-mon[98901]: pgmap v1183: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:47.503 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:47 smithi096 ceph-mon[98901]: 1.19 scrub starts 2024-09-13T15:02:47.503 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:47 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3735067530' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:02:47.503 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:47 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:02:47.503 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:47 smithi096 ceph-mon[102928]: 1.16 scrub starts 2024-09-13T15:02:47.504 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:47 smithi096 ceph-mon[102928]: pgmap v1183: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:47.504 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:47 smithi096 ceph-mon[102928]: 1.19 scrub starts 2024-09-13T15:02:47.504 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:47 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3735067530' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:02:47.504 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:47 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:02:47.506 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:47.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:47.608 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:47.629 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:47.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:47 smithi204 ceph-mon[87071]: 1.16 scrub starts 2024-09-13T15:02:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:47 smithi204 ceph-mon[87071]: pgmap v1183: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:47 smithi204 ceph-mon[87071]: 1.19 scrub starts 2024-09-13T15:02:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:47 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3735067530' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:02:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:47 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:02:47.731 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:47.756 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:47.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:47.858 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:47.882 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:47.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:47.984 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:48.005 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:48.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:48.031 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T15:02:48.107 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:48.134 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:48.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:48.236 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:02:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:02:47] "GET /metrics HTTP/1.1" 200 34181 "" "Prometheus/2.51.0" 2024-09-13T15:02:48.236 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:48.240 INFO:teuthology.orchestra.run.smithi096.stderr:enabled application 'rados' on pool 'unique_pool_5' 2024-09-13T15:02:48.262 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:48.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:48.278 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:48.364 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:48.391 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:48.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:48.398 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:02:48.492 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:48.516 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:48.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:48.517 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:48 smithi096 ceph-mon[98901]: 1.1d scrub starts 2024-09-13T15:02:48.517 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:48 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:02:48.518 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:48 smithi096 ceph-mon[98901]: osdmap e358: 8 total, 8 up, 7 in 2024-09-13T15:02:48.518 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:48 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3929850304' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:02:48.518 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:48 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:02:48.518 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:48 smithi096 ceph-mon[102928]: 1.1d scrub starts 2024-09-13T15:02:48.518 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:48 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:02:48.518 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:48 smithi096 ceph-mon[102928]: osdmap e358: 8 total, 8 up, 7 in 2024-09-13T15:02:48.518 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:48 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3929850304' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:02:48.518 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:48 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:02:48.618 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:48.644 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:48.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:48.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:48 smithi204 ceph-mon[87071]: 1.1d scrub starts 2024-09-13T15:02:48.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:48 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:02:48.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:48 smithi204 ceph-mon[87071]: osdmap e358: 8 total, 8 up, 7 in 2024-09-13T15:02:48.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:48 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3929850304' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:02:48.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:48 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:02:48.746 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:48.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.768 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:48.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:48.869 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:48.896 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:48.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:48.999 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:49.025 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:49.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:49.127 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:49.151 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:49.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:49.240 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T15:02:49.253 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:49.278 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:49.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:49.379 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:49.404 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:49.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:49.506 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:49.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.529 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:49.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:49.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:49 smithi096 ceph-mon[98901]: pgmap v1185: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:49.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:49 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:02:49.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:49 smithi096 ceph-mon[98901]: osdmap e359: 8 total, 8 up, 7 in 2024-09-13T15:02:49.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:49 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1320528614' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:02:49.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:49 smithi096 ceph-mon[102928]: pgmap v1185: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:49.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:49 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:02:49.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:49 smithi096 ceph-mon[102928]: osdmap e359: 8 total, 8 up, 7 in 2024-09-13T15:02:49.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:49 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1320528614' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:02:49.630 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:49.652 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:49.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:49.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:49 smithi204 ceph-mon[87071]: pgmap v1185: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:49.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:49 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:02:49.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:49 smithi204 ceph-mon[87071]: osdmap e359: 8 total, 8 up, 7 in 2024-09-13T15:02:49.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:49 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1320528614' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:02:49.754 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:49.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.776 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:49.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:49.877 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:49.903 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:49.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:50.005 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:50.026 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:50.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:50.034 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:50.069 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:50.105 DEBUG:teuthology.orchestra.run.smithi096:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2024-09-13T15:02:50.108 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T15:02:50.108 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T15:02:50.129 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:50.153 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:hints = 1 2024-09-13T15:02:50.154 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T15:02:50.154 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Object prefix: benchmark_data_smithi096_288650 2024-09-13T15:02:50.154 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:02:50.154 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 0 0 0 0 0 0 - 0 2024-09-13T15:02:50.156 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:50.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:50.259 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:50.294 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:50.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:50.396 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:50.431 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:50.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:02:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:02:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:02:50.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:50 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:50.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:50 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1320528614' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:02:50.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:50 smithi204 ceph-mon[87071]: osdmap e360: 8 total, 8 up, 7 in 2024-09-13T15:02:50.533 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:50.566 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:50.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:50.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:50 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:50.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:50 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1320528614' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:02:50.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:50 smithi096 ceph-mon[98901]: osdmap e360: 8 total, 8 up, 7 in 2024-09-13T15:02:50.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:50 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:50.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:50 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1320528614' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:02:50.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:50 smithi096 ceph-mon[102928]: osdmap e360: 8 total, 8 up, 7 in 2024-09-13T15:02:50.669 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:50.702 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:50.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:50.805 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:50.841 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:50.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:50.943 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:50.983 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:50.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:51.085 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:51.118 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:51.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:51.153 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 1 15 4081 4066 254.111 254.125 0.00565279 0.00392428 2024-09-13T15:02:51.220 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:51.256 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:51.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:51.358 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:51.397 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:51.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:51.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[98901]: 1.7 scrub starts 2024-09-13T15:02:51.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[98901]: 1.7 scrub ok 2024-09-13T15:02:51.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[98901]: pgmap v1188: 119 pgs: 119 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:51.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[98901]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:02:51.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[98901]: 1.9 scrub starts 2024-09-13T15:02:51.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[98901]: 1.9 scrub ok 2024-09-13T15:02:51.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:51.475 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:51.475 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[102928]: 1.7 scrub starts 2024-09-13T15:02:51.475 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[102928]: 1.7 scrub ok 2024-09-13T15:02:51.475 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[102928]: pgmap v1188: 119 pgs: 119 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:51.475 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[102928]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:02:51.475 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[102928]: 1.9 scrub starts 2024-09-13T15:02:51.475 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[102928]: 1.9 scrub ok 2024-09-13T15:02:51.475 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:51.475 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:51.500 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:51.535 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:51.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:51.637 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:51.673 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:51.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:51 smithi204 ceph-mon[87071]: 1.7 scrub starts 2024-09-13T15:02:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:51 smithi204 ceph-mon[87071]: 1.7 scrub ok 2024-09-13T15:02:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:51 smithi204 ceph-mon[87071]: pgmap v1188: 119 pgs: 119 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-09-13T15:02:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:51 smithi204 ceph-mon[87071]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:02:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:51 smithi204 ceph-mon[87071]: 1.9 scrub starts 2024-09-13T15:02:51.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:51 smithi204 ceph-mon[87071]: 1.9 scrub ok 2024-09-13T15:02:51.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:51.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:02:51.775 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:51.808 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:51.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:51.910 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:51.945 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:51.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:52.047 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:52.080 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:52.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:52.153 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 2 16 7846 7830 244.67 235.25 0.0040066 0.00407895 2024-09-13T15:02:52.183 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:52.219 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:52.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:52.321 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:52.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.358 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:52.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:52.420 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:52 smithi096 ceph-mon[102928]: 1.16 scrub starts 2024-09-13T15:02:52.420 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:52 smithi096 ceph-mon[102928]: 1.16 scrub ok 2024-09-13T15:02:52.421 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:52 smithi096 ceph-mon[98901]: 1.16 scrub starts 2024-09-13T15:02:52.421 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:52 smithi096 ceph-mon[98901]: 1.16 scrub ok 2024-09-13T15:02:52.459 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:52.492 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:52.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:52.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:52.636 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:52.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:52.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:52 smithi204 ceph-mon[87071]: 1.16 scrub starts 2024-09-13T15:02:52.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:52 smithi204 ceph-mon[87071]: 1.16 scrub ok 2024-09-13T15:02:52.738 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:52.744 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T15:02:52.744 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T15:02:52.745 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T15:02:52.745 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T15:02:52.745 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T15:02:52.771 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:52.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:52.873 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:52.910 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:52.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:53.012 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:53.043 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:53.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:53.144 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:53.153 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 3 16 11182 11166 232.608 208.5 0.00378229 0.00429079 2024-09-13T15:02:53.180 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:53.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:53.282 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:53.312 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:53.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:53.414 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:53.451 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:53.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:53.458 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:02:53.458 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:02:53.487 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:02:53.487 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:02:53.515 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:02:53.515 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:02:53.516 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[98901]: 1.6 scrub starts 2024-09-13T15:02:53.516 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[98901]: 1.6 scrub ok 2024-09-13T15:02:53.516 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[98901]: 1.1d scrub starts 2024-09-13T15:02:53.516 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[98901]: 1.1d scrub ok 2024-09-13T15:02:53.517 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[98901]: pgmap v1189: 119 pgs: 119 active+clean; 261 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 43 MiB/s wr, 684 op/s 2024-09-13T15:02:53.517 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[98901]: 1.5 scrub starts 2024-09-13T15:02:53.517 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[98901]: 1.5 scrub ok 2024-09-13T15:02:53.517 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[98901]: 1.19 scrub starts 2024-09-13T15:02:53.517 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[98901]: 1.19 scrub ok 2024-09-13T15:02:53.517 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[102928]: 1.6 scrub starts 2024-09-13T15:02:53.517 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[102928]: 1.6 scrub ok 2024-09-13T15:02:53.517 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[102928]: 1.1d scrub starts 2024-09-13T15:02:53.517 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[102928]: 1.1d scrub ok 2024-09-13T15:02:53.517 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[102928]: pgmap v1189: 119 pgs: 119 active+clean; 261 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 43 MiB/s wr, 684 op/s 2024-09-13T15:02:53.517 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[102928]: 1.5 scrub starts 2024-09-13T15:02:53.518 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[102928]: 1.5 scrub ok 2024-09-13T15:02:53.518 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[102928]: 1.19 scrub starts 2024-09-13T15:02:53.518 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:53 smithi096 ceph-mon[102928]: 1.19 scrub ok 2024-09-13T15:02:53.550 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:02:53.551 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:02:53.553 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:53.582 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:02:53.582 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:02:53.585 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:53.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:53.616 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:02:53.616 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:02:53.648 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:02:53.649 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:02:53.678 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:02:53.678 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:02:53.687 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:53 smithi204 ceph-mon[87071]: 1.6 scrub starts 2024-09-13T15:02:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:53 smithi204 ceph-mon[87071]: 1.6 scrub ok 2024-09-13T15:02:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:53 smithi204 ceph-mon[87071]: 1.1d scrub starts 2024-09-13T15:02:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:53 smithi204 ceph-mon[87071]: 1.1d scrub ok 2024-09-13T15:02:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:53 smithi204 ceph-mon[87071]: pgmap v1189: 119 pgs: 119 active+clean; 261 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 43 MiB/s wr, 684 op/s 2024-09-13T15:02:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:53 smithi204 ceph-mon[87071]: 1.5 scrub starts 2024-09-13T15:02:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:53 smithi204 ceph-mon[87071]: 1.5 scrub ok 2024-09-13T15:02:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:53 smithi204 ceph-mon[87071]: 1.19 scrub starts 2024-09-13T15:02:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:53 smithi204 ceph-mon[87071]: 1.19 scrub ok 2024-09-13T15:02:53.723 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:53.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:53.825 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:53.858 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:53.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:53.960 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:53.991 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:53.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:54.065 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T15:02:54.093 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:54.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.134 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:54.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:54.153 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 4 16 14383 14367 224.467 200.062 0.00499351 0.00444927 2024-09-13T15:02:54.235 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:54.272 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:54.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:54.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:54 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/598398661' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:54.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:54 smithi096 ceph-mon[98901]: 1.d scrub starts 2024-09-13T15:02:54.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:54 smithi096 ceph-mon[98901]: 1.d scrub ok 2024-09-13T15:02:54.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:54 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/598398661' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:54.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:54 smithi096 ceph-mon[102928]: 1.d scrub starts 2024-09-13T15:02:54.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:54 smithi096 ceph-mon[102928]: 1.d scrub ok 2024-09-13T15:02:54.374 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:54.428 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:54.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:54.530 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:54.634 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:54.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:54.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:54 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/598398661' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:54.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:54 smithi204 ceph-mon[87071]: 1.d scrub starts 2024-09-13T15:02:54.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:54 smithi204 ceph-mon[87071]: 1.d scrub ok 2024-09-13T15:02:54.736 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:54.772 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:54.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:54.874 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:54.908 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:54.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:55.010 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:55.049 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:55.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:55.151 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:55.154 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 5 16 17362 17346 216.808 186.188 0.00628669 0.00460731 2024-09-13T15:02:55.181 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:55.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:55.283 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:55.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.320 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:55.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:55.421 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:55.451 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:55.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:55.553 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:55.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:55 smithi096 ceph-mon[98901]: pgmap v1190: 119 pgs: 119 active+clean; 315 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 45 MiB/s wr, 714 op/s 2024-09-13T15:02:55.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:55 smithi096 ceph-mon[98901]: 1.4 scrub starts 2024-09-13T15:02:55.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:55 smithi096 ceph-mon[98901]: 1.4 scrub ok 2024-09-13T15:02:55.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:55 smithi096 ceph-mon[102928]: pgmap v1190: 119 pgs: 119 active+clean; 315 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 45 MiB/s wr, 714 op/s 2024-09-13T15:02:55.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:55 smithi096 ceph-mon[102928]: 1.4 scrub starts 2024-09-13T15:02:55.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:55 smithi096 ceph-mon[102928]: 1.4 scrub ok 2024-09-13T15:02:55.589 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:55.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:55.690 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:55 smithi204 ceph-mon[87071]: pgmap v1190: 119 pgs: 119 active+clean; 315 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 45 MiB/s wr, 714 op/s 2024-09-13T15:02:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:55 smithi204 ceph-mon[87071]: 1.4 scrub starts 2024-09-13T15:02:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:55 smithi204 ceph-mon[87071]: 1.4 scrub ok 2024-09-13T15:02:55.721 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:55.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:55.823 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:55.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.856 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:55.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:55.957 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:55.990 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:55.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:56.092 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:56.126 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:56.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:56.158 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 6 16 20243 20227 210.515 180.062 0.00413812 0.00474432 2024-09-13T15:02:56.228 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:56.262 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:56.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:56.364 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:56.400 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:56.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:56.502 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:56.538 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:56.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:56.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:56 smithi096 ceph-mon[98901]: 1.e scrub starts 2024-09-13T15:02:56.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:56 smithi096 ceph-mon[98901]: 1.e scrub ok 2024-09-13T15:02:56.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:56 smithi096 ceph-mon[98901]: 1.1e scrub starts 2024-09-13T15:02:56.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:56 smithi096 ceph-mon[98901]: 1.1e scrub ok 2024-09-13T15:02:56.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:56 smithi096 ceph-mon[98901]: 1.15 scrub starts 2024-09-13T15:02:56.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:56 smithi096 ceph-mon[98901]: 1.15 scrub ok 2024-09-13T15:02:56.572 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:56 smithi096 ceph-mon[102928]: 1.e scrub starts 2024-09-13T15:02:56.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:56 smithi096 ceph-mon[102928]: 1.e scrub ok 2024-09-13T15:02:56.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:56 smithi096 ceph-mon[102928]: 1.1e scrub starts 2024-09-13T15:02:56.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:56 smithi096 ceph-mon[102928]: 1.1e scrub ok 2024-09-13T15:02:56.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:56 smithi096 ceph-mon[102928]: 1.15 scrub starts 2024-09-13T15:02:56.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:56 smithi096 ceph-mon[102928]: 1.15 scrub ok 2024-09-13T15:02:56.638 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:56.641 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:56.682 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:56.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:56.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:56 smithi204 ceph-mon[87071]: 1.e scrub starts 2024-09-13T15:02:56.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:56 smithi204 ceph-mon[87071]: 1.e scrub ok 2024-09-13T15:02:56.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:56 smithi204 ceph-mon[87071]: 1.1e scrub starts 2024-09-13T15:02:56.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:56 smithi204 ceph-mon[87071]: 1.1e scrub ok 2024-09-13T15:02:56.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:56 smithi204 ceph-mon[87071]: 1.15 scrub starts 2024-09-13T15:02:56.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:56 smithi204 ceph-mon[87071]: 1.15 scrub ok 2024-09-13T15:02:56.784 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:56.816 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:56.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:56.918 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:56.957 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:56.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:57.059 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:57.092 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:57.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:57.161 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 7 15 23012 22997 205.175 173.125 0.00526034 0.00486892 2024-09-13T15:02:57.195 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:57.229 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:57.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:57.331 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:57.367 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:57.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:57.469 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:57.499 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:57.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:57.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:57 smithi096 ceph-mon[98901]: pgmap v1191: 119 pgs: 119 active+clean; 490 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 61 MiB/s wr, 970 op/s 2024-09-13T15:02:57.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:57 smithi096 ceph-mon[98901]: 1.1a scrub starts 2024-09-13T15:02:57.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:57 smithi096 ceph-mon[98901]: 1.1a scrub ok 2024-09-13T15:02:57.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:57 smithi096 ceph-mon[102928]: pgmap v1191: 119 pgs: 119 active+clean; 490 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 61 MiB/s wr, 970 op/s 2024-09-13T15:02:57.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:57 smithi096 ceph-mon[102928]: 1.1a scrub starts 2024-09-13T15:02:57.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:57 smithi096 ceph-mon[102928]: 1.1a scrub ok 2024-09-13T15:02:57.602 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:57.623 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:02:57.640 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:57.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:57.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:57 smithi204 ceph-mon[87071]: pgmap v1191: 119 pgs: 119 active+clean; 490 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 61 MiB/s wr, 970 op/s 2024-09-13T15:02:57.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:57 smithi204 ceph-mon[87071]: 1.1a scrub starts 2024-09-13T15:02:57.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:57 smithi204 ceph-mon[87071]: 1.1a scrub ok 2024-09-13T15:02:57.743 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:57.788 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:57.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:57.890 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:57.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.929 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:57.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:58.030 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:58.061 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:58.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:58.119 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:02:58.159 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 8 16 25566 25550 199.475 159.562 0.00472745 0.005008 2024-09-13T15:02:58.163 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:58.197 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:58.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:58.288 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:02:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:02:57] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-13T15:02:58.299 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:58.338 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:58.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:58.440 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:58.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.477 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:58.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:58.567 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:58 smithi096 ceph-mon[98901]: 1.8 scrub starts 2024-09-13T15:02:58.567 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:58 smithi096 ceph-mon[98901]: 1.8 scrub ok 2024-09-13T15:02:58.568 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:58 smithi096 ceph-mon[102928]: 1.8 scrub starts 2024-09-13T15:02:58.568 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:58 smithi096 ceph-mon[102928]: 1.8 scrub ok 2024-09-13T15:02:58.578 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:58.618 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:58.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:58.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:58 smithi204 ceph-mon[87071]: 1.8 scrub starts 2024-09-13T15:02:58.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:58 smithi204 ceph-mon[87071]: 1.8 scrub ok 2024-09-13T15:02:58.720 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:58.754 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:58.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:58.856 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:58.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.891 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:58.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:58.992 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:59.028 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:59.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:59.130 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:59.159 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 9 16 28003 27987 194.236 152.312 0.00571201 0.0051427 2024-09-13T15:02:59.163 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:59.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:59.265 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:59.299 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T15:02:59.302 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:59.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:59.404 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:59.439 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:59.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:59.490 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:59 smithi096 ceph-mon[98901]: pgmap v1192: 119 pgs: 119 active+clean; 1.2 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-13T15:02:59.491 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:59 smithi096 ceph-mon[98901]: 1.a scrub starts 2024-09-13T15:02:59.491 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:59 smithi096 ceph-mon[98901]: 1.a scrub ok 2024-09-13T15:02:59.491 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:59 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3437333311' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:02:59.491 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:02:59 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:02:59.491 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:59 smithi096 ceph-mon[102928]: pgmap v1192: 119 pgs: 119 active+clean; 1.2 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-13T15:02:59.491 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:59 smithi096 ceph-mon[102928]: 1.a scrub starts 2024-09-13T15:02:59.491 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:59 smithi096 ceph-mon[102928]: 1.a scrub ok 2024-09-13T15:02:59.491 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:59 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3437333311' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:02:59.492 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:02:59 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:02:59.541 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:59.573 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:02:59.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:59.674 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:59.713 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:59.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:59 smithi204 ceph-mon[87071]: pgmap v1192: 119 pgs: 119 active+clean; 1.2 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-13T15:02:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:59 smithi204 ceph-mon[87071]: 1.a scrub starts 2024-09-13T15:02:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:59 smithi204 ceph-mon[87071]: 1.a scrub ok 2024-09-13T15:02:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:59 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3437333311' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:02:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:02:59 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:02:59.815 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:59.848 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:59.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:59.950 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:59.988 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:02:59.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:00.090 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:00.121 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:00.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:00.159 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 10 16 30330 30314 189.357 145.438 0.00839623 0.00527597 2024-09-13T15:03:00.171 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T15:03:00.171 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:03:00.223 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:00.289 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:00.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:00.391 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:00.427 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:00.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:00 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:03:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:00 smithi204 ceph-mon[87071]: osdmap e361: 8 total, 8 up, 7 in 2024-09-13T15:03:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:03:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:03:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:03:00.529 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:00.553 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:00.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:00.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:00 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:03:00.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:00 smithi096 ceph-mon[98901]: osdmap e361: 8 total, 8 up, 7 in 2024-09-13T15:03:00.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:00 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:03:00.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:00 smithi096 ceph-mon[102928]: osdmap e361: 8 total, 8 up, 7 in 2024-09-13T15:03:00.655 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:00.685 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:00.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:00.787 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:00.826 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:00.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:00.928 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:00.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.967 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:00.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:01.068 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:01.107 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:01.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:01.159 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 11 16 32345 32329 183.593 125.938 0.00674046 0.005425 2024-09-13T15:03:01.209 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:01.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.248 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:01.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:01.349 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:01.382 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:01.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:01.483 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:01.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.519 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:01.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:01.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:01 smithi096 ceph-mon[98901]: pgmap v1194: 119 pgs: 119 active+clean; 1.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 144 MiB/s wr, 2.30k op/s 2024-09-13T15:03:01.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:01 smithi096 ceph-mon[102928]: pgmap v1194: 119 pgs: 119 active+clean; 1.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 144 MiB/s wr, 2.30k op/s 2024-09-13T15:03:01.620 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:01.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.650 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:01.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:01 smithi204 ceph-mon[87071]: pgmap v1194: 119 pgs: 119 active+clean; 1.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 144 MiB/s wr, 2.30k op/s 2024-09-13T15:03:01.751 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:01.793 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:01.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:01.895 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:01.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.920 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:01.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:02.021 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:02.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.054 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:02.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:02.082 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:03:02.083 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:03:02.155 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:02.159 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 12 16 34219 34203 178.056 117.125 0.00184318 0.00560807 2024-09-13T15:03:02.176 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:02.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:02.278 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:02.313 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:02.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:02.415 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:02.450 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:02.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:02.552 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:02.586 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:02.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:02.688 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:02.722 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:02.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:02.824 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:02.856 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:02.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:02.958 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:02.995 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:02.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:03.097 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:03.129 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:03.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:03.159 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 13 16 36227 36211 174.014 125.5 0.00986361 0.00574231 2024-09-13T15:03:03.232 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:03.269 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:03.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:03.371 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:03.407 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:03.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:03.509 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:03.538 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:03.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:03.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:03 smithi096 ceph-mon[98901]: pgmap v1195: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 173 MiB/s wr, 2.76k op/s 2024-09-13T15:03:03.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:03 smithi096 ceph-mon[102928]: pgmap v1195: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 173 MiB/s wr, 2.76k op/s 2024-09-13T15:03:03.640 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:03.672 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:03.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:03.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:03 smithi204 ceph-mon[87071]: pgmap v1195: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 173 MiB/s wr, 2.76k op/s 2024-09-13T15:03:03.773 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:03.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.806 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:03.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:03.871 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:03.871 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:03.907 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:03.945 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:03.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:04.047 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:04.084 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:04.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:04.160 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 14 16 38301 38285 170.833 129.625 0.00447377 0.0058483 2024-09-13T15:03:04.185 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:04.223 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:04.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:04.325 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:04.358 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:04.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:04.460 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:04.498 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:04.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:04.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:04 smithi096 ceph-mon[98901]: 1.17 scrub starts 2024-09-13T15:03:04.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:04 smithi096 ceph-mon[98901]: 1.17 scrub ok 2024-09-13T15:03:04.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:04 smithi096 ceph-mon[102928]: 1.17 scrub starts 2024-09-13T15:03:04.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:04 smithi096 ceph-mon[102928]: 1.17 scrub ok 2024-09-13T15:03:04.599 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:04.620 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:04.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:04.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:04 smithi204 ceph-mon[87071]: 1.17 scrub starts 2024-09-13T15:03:04.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:04 smithi204 ceph-mon[87071]: 1.17 scrub ok 2024-09-13T15:03:04.722 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:04.745 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:04.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:04.847 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:04.881 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:04.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:04.983 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:05.018 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:05.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:05.119 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:05.155 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:05.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:05.160 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 15 16 40081 40065 166.862 111.25 0.00426527 0.00598778 2024-09-13T15:03:05.258 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:05.295 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:05.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:05.304 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T15:03:05.397 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:05.431 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:05.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:05.512 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:05 smithi096 ceph-mon[98901]: 1.18 scrub starts 2024-09-13T15:03:05.512 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:05 smithi096 ceph-mon[98901]: 1.18 scrub ok 2024-09-13T15:03:05.512 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:05 smithi096 ceph-mon[98901]: pgmap v1196: 119 pgs: 119 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 171 MiB/s wr, 2.74k op/s 2024-09-13T15:03:05.513 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:05 smithi096 ceph-mon[102928]: 1.18 scrub starts 2024-09-13T15:03:05.513 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:05 smithi096 ceph-mon[102928]: 1.18 scrub ok 2024-09-13T15:03:05.513 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:05 smithi096 ceph-mon[102928]: pgmap v1196: 119 pgs: 119 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 171 MiB/s wr, 2.74k op/s 2024-09-13T15:03:05.533 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:05.568 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:05.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:05.670 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:05.705 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:05.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:05 smithi204 ceph-mon[87071]: 1.18 scrub starts 2024-09-13T15:03:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:05 smithi204 ceph-mon[87071]: 1.18 scrub ok 2024-09-13T15:03:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:05 smithi204 ceph-mon[87071]: pgmap v1196: 119 pgs: 119 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 171 MiB/s wr, 2.74k op/s 2024-09-13T15:03:05.808 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:05.838 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:05.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:05.940 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:05.978 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:05.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:06.080 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:06.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.113 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:06.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:06.160 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 16 16 42063 42047 164.175 123.875 0.00461141 0.00608653 2024-09-13T15:03:06.215 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:06.252 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:06.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:06.354 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:06.388 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:06.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:06.489 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:06.527 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:06.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:06.629 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:06.658 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:06.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:06.759 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:06.784 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:06.795 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:06.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:06.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:06.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:06.897 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:06.928 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:06.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:07.030 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:07.058 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:07.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:07.160 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:07.161 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 17 16 43980 43964 161.566 119.812 0.0107805 0.00618405 2024-09-13T15:03:07.193 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:07.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:07.295 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:07.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.339 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:07.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:07.440 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:07.477 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:07.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:07.552 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:07 smithi096 ceph-mon[98901]: pgmap v1197: 119 pgs: 119 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 166 MiB/s wr, 2.66k op/s 2024-09-13T15:03:07.552 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:07 smithi096 ceph-mon[102928]: pgmap v1197: 119 pgs: 119 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 166 MiB/s wr, 2.66k op/s 2024-09-13T15:03:07.579 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:07.616 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:07.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:07.717 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:07 smithi204 ceph-mon[87071]: pgmap v1197: 119 pgs: 119 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 166 MiB/s wr, 2.66k op/s 2024-09-13T15:03:07.746 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:07.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:07.848 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:07.885 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:07.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:07.986 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:08.017 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:08.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:08.119 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:08.160 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:08.160 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 18 16 45884 45868 159.201 119 0.00846645 0.00627664 2024-09-13T15:03:08.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:08.225 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:03:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:03:07] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-09-13T15:03:08.262 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:08.300 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:08.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:08.402 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:08.429 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:08.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:08.500 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:08 smithi096 ceph-mon[98901]: 1.1b scrub starts 2024-09-13T15:03:08.500 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:08 smithi096 ceph-mon[98901]: 1.1b scrub ok 2024-09-13T15:03:08.500 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:08 smithi096 ceph-mon[102928]: 1.1b scrub starts 2024-09-13T15:03:08.500 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:08 smithi096 ceph-mon[102928]: 1.1b scrub ok 2024-09-13T15:03:08.531 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:08.562 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:08.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:08.664 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:08.699 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:08.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:08.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:08 smithi204 ceph-mon[87071]: 1.1b scrub starts 2024-09-13T15:03:08.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:08 smithi204 ceph-mon[87071]: 1.1b scrub ok 2024-09-13T15:03:08.801 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:08.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.841 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:08.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:08.942 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:08.979 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:08.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:09.081 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:09.119 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:09.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:09.160 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 19 16 47685 47669 156.747 112.562 0.00506947 0.00637424 2024-09-13T15:03:09.221 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:09.255 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:09.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:09.357 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:09.374 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T15:03:09.392 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:09.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:09.440 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:09 smithi096 ceph-mon[98901]: pgmap v1198: 119 pgs: 119 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-09-13T15:03:09.440 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:09 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1239081797' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:09.441 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:09 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:09.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:09 smithi096 ceph-mon[102928]: pgmap v1198: 119 pgs: 119 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-09-13T15:03:09.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:09 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1239081797' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:09.441 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:09 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:09.493 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:09.530 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:09.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:09.632 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:09.666 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:09.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:09 smithi204 ceph-mon[87071]: pgmap v1198: 119 pgs: 119 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-09-13T15:03:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:09 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1239081797' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:09 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:09.769 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:09.808 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:09.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:09.910 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:09.948 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:09.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:10.011 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:10.050 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:10.080 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:10.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:10.161 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T15:03:10.160267+0000 min lat: 0.00132706 max lat: 0.121907 avg lat: 0.00646886 2024-09-13T15:03:10.161 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:03:10.161 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 20 16 49464 49448 154.469 111.188 0.0061895 0.00646886 2024-09-13T15:03:10.181 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:10.211 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:10.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:10.313 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:10.341 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:10.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:10.443 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:10 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:03:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:10 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T15:03:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:10 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:10.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:10 smithi204 ceph-mon[87071]: osdmap e362: 8 total, 8 up, 7 in 2024-09-13T15:03:10.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:03:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:03:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:03:10.472 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:10.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:10.574 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:10.614 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:10.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:10.716 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:10.747 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:10.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:10.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:10 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:03:10.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:10 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T15:03:10.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:10 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:10.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:10 smithi096 ceph-mon[98901]: osdmap e362: 8 total, 8 up, 7 in 2024-09-13T15:03:10.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:10 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:03:10.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:10 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T15:03:10.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:10 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:10.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:10 smithi096 ceph-mon[102928]: osdmap e362: 8 total, 8 up, 7 in 2024-09-13T15:03:10.850 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:10.890 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:10.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:10.992 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:11.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.035 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:11.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:11.136 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:11.160 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 21 16 51245 51229 152.414 111.312 0.00486103 0.00655557 2024-09-13T15:03:11.177 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:11.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:11.279 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:11.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.314 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:11.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:11.416 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:11.451 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:11.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:11.553 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:11.589 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:11.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:11.644 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:03:11.644 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:03:11.644 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:11 smithi096 ceph-mon[98901]: pgmap v1200: 119 pgs: 119 active+clean; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-09-13T15:03:11.645 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:11 smithi096 ceph-mon[102928]: pgmap v1200: 119 pgs: 119 active+clean; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-09-13T15:03:11.681 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:03:11.681 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:03:11.692 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:11.715 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:03:11.715 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:03:11.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:11 smithi204 ceph-mon[87071]: pgmap v1200: 119 pgs: 119 active+clean; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-09-13T15:03:11.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.729 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:11.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:11.750 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:03:11.750 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:03:11.789 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:03:11.790 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:03:11.825 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:03:11.825 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:03:11.829 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:11.856 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:03:11.856 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:03:11.861 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:11.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:11.878 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T15:03:11.880 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T15:03:11.889 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:03:11.889 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:03:11.963 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:12.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.001 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:12.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:12.102 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:12.131 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:12.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:12.161 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 22 16 52937 52921 150.293 105.75 0.00923575 0.00664872 2024-09-13T15:03:12.232 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:12.275 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:12.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:12.377 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:12.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.415 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:12.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:12.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:12.555 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:12.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:12.657 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:12.688 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:12.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:12.696 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:12 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3539030276' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:12.696 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:12 smithi096 ceph-mon[98901]: from='client.46412 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:03:12.696 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:12 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3539030276' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:12.696 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:12 smithi096 ceph-mon[102928]: from='client.46412 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:03:12.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:12 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3539030276' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:12.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:12 smithi204 ceph-mon[87071]: from='client.46412 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:03:12.738 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":1200,"stamp":"2024-09-13T15:03:10.174699+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2888374221,"num_objects":44028,"num_object_clones":0,"num_object_copies":132084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":44028,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":44907,"num_write_kb":2828623,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":109,"num_bytes_recovered":8404818,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":48132,"ondisk_log_size":48132,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14043248,"kb_used_data":8438712,"kb_used_omap":29,"kb_used_meta":5604450,"kb_avail":642172816,"statfs":{"total":671965249536,"available":657584963584,"internally_reserved":0,"allocated":8641241088,"data_stored":8623820649,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30654,"internal_metadata":5738956866},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1374093312,"num_objects":20967,"num_object_clones":0,"num_object_copies":62901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20967,"num_write_kb":1341888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":-525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002738"},"pg_stats":[{"pgid":"1.11","version":"82'100","reported_seq":327,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383953+0000","last_change":"2024-09-13T14:59:00.929470+0000","last_active":"2024-09-13T15:03:09.383953+0000","last_peered":"2024-09-13T15:03:09.383953+0000","last_clean":"2024-09-13T15:03:09.383953+0000","last_became_active":"2024-09-13T14:59:00.928726+0000","last_became_peered":"2024-09-13T14:59:00.928726+0000","last_unstale":"2024-09-13T15:03:09.383953+0000","last_undegraded":"2024-09-13T15:03:09.383953+0000","last_fullsized":"2024-09-13T15:03:09.383953+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.087994+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.087994+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:41.304122+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":167,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.302434+0000","last_change":"2024-09-13T14:59:00.935629+0000","last_active":"2024-09-13T15:02:59.302434+0000","last_peered":"2024-09-13T15:02:59.302434+0000","last_clean":"2024-09-13T15:02:59.302434+0000","last_became_active":"2024-09-13T14:59:00.935106+0000","last_became_peered":"2024-09-13T14:59:00.935106+0000","last_unstale":"2024-09-13T15:02:59.302434+0000","last_undegraded":"2024-09-13T15:02:59.302434+0000","last_fullsized":"2024-09-13T15:02:59.302434+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:12.481049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":327,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.241954+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T15:02:49.241954+0000","last_peered":"2024-09-13T15:02:49.241954+0000","last_clean":"2024-09-13T15:02:49.241954+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T15:02:49.241954+0000","last_undegraded":"2024-09-13T15:02:49.241954+0000","last_fullsized":"2024-09-13T15:02:49.241954+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":91,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.769334+0000","last_change":"2024-09-13T14:56:04.643672+0000","last_active":"2024-09-13T15:02:49.769334+0000","last_peered":"2024-09-13T15:02:49.769334+0000","last_clean":"2024-09-13T15:02:49.769334+0000","last_became_active":"2024-09-13T14:55:55.185995+0000","last_became_peered":"2024-09-13T14:55:55.185995+0000","last_unstale":"2024-09-13T15:02:49.769334+0000","last_undegraded":"2024-09-13T15:02:49.769334+0000","last_fullsized":"2024-09-13T15:02:49.769334+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.643614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.643614+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:36.495546+0000","scrub_duration":0.001441681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":334,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.770057+0000","last_change":"2024-09-13T14:59:00.926231+0000","last_active":"2024-09-13T15:02:49.770057+0000","last_peered":"2024-09-13T15:02:49.770057+0000","last_clean":"2024-09-13T15:02:49.770057+0000","last_became_active":"2024-09-13T14:59:00.925808+0000","last_became_peered":"2024-09-13T14:59:00.925808+0000","last_unstale":"2024-09-13T15:02:49.770057+0000","last_undegraded":"2024-09-13T15:02:49.770057+0000","last_fullsized":"2024-09-13T15:02:49.770057+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:39.672143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":618,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384469+0000","last_change":"2024-09-13T14:59:00.923848+0000","last_active":"2024-09-13T15:03:09.384469+0000","last_peered":"2024-09-13T15:03:09.384469+0000","last_clean":"2024-09-13T15:03:09.384469+0000","last_became_active":"2024-09-13T14:59:00.923522+0000","last_became_peered":"2024-09-13T14:59:00.923522+0000","last_unstale":"2024-09-13T15:03:09.384469+0000","last_undegraded":"2024-09-13T15:03:09.384469+0000","last_fullsized":"2024-09-13T15:03:09.384469+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:35:35.267527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":756,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301201+0000","last_change":"2024-09-13T14:59:00.930294+0000","last_active":"2024-09-13T15:02:59.301201+0000","last_peered":"2024-09-13T15:02:59.301201+0000","last_clean":"2024-09-13T15:02:59.301201+0000","last_became_active":"2024-09-13T14:59:00.929860+0000","last_became_peered":"2024-09-13T14:59:00.929860+0000","last_unstale":"2024-09-13T15:02:59.301201+0000","last_undegraded":"2024-09-13T15:02:59.301201+0000","last_fullsized":"2024-09-13T15:02:59.301201+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.240882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.240882+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:48:18.212524+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":612,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384229+0000","last_change":"2024-09-13T14:59:00.925448+0000","last_active":"2024-09-13T15:03:09.384229+0000","last_peered":"2024-09-13T15:03:09.384229+0000","last_clean":"2024-09-13T15:03:09.384229+0000","last_became_active":"2024-09-13T14:59:00.925152+0000","last_became_peered":"2024-09-13T14:59:00.925152+0000","last_unstale":"2024-09-13T15:03:09.384229+0000","last_undegraded":"2024-09-13T15:03:09.384229+0000","last_fullsized":"2024-09-13T15:03:09.384229+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:06.703642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":328,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.241853+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T15:02:49.241853+0000","last_peered":"2024-09-13T15:02:49.241853+0000","last_clean":"2024-09-13T15:02:49.241853+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T15:02:49.241853+0000","last_undegraded":"2024-09-13T15:02:49.241853+0000","last_fullsized":"2024-09-13T15:02:49.241853+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":91,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.769755+0000","last_change":"2024-09-13T14:56:03.605447+0000","last_active":"2024-09-13T15:02:49.769755+0000","last_peered":"2024-09-13T15:02:49.769755+0000","last_clean":"2024-09-13T15:02:49.769755+0000","last_became_active":"2024-09-13T14:55:55.218939+0000","last_became_peered":"2024-09-13T14:55:55.218939+0000","last_unstale":"2024-09-13T15:02:49.769755+0000","last_undegraded":"2024-09-13T15:02:49.769755+0000","last_fullsized":"2024-09-13T15:02:49.769755+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.605356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.605356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:12.467371+0000","scrub_duration":0.0020269620000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.242458+0000","last_change":"2024-09-13T14:59:00.925287+0000","last_active":"2024-09-13T15:02:49.242458+0000","last_peered":"2024-09-13T15:02:49.242458+0000","last_clean":"2024-09-13T15:02:49.242458+0000","last_became_active":"2024-09-13T14:59:00.923868+0000","last_became_peered":"2024-09-13T14:59:00.923868+0000","last_unstale":"2024-09-13T15:02:49.242458+0000","last_undegraded":"2024-09-13T15:02:49.242458+0000","last_fullsized":"2024-09-13T15:02:49.242458+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:28.106847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":155,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.321636+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T15:02:49.321636+0000","last_peered":"2024-09-13T15:02:49.321636+0000","last_clean":"2024-09-13T15:02:49.321636+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T15:02:49.321636+0000","last_undegraded":"2024-09-13T15:02:49.321636+0000","last_fullsized":"2024-09-13T15:02:49.321636+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"361'2643","reported_seq":2655,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.502224+0000","last_change":"2024-09-13T15:02:39.884666+0000","last_active":"2024-09-13T15:03:06.502224+0000","last_peered":"2024-09-13T15:03:06.502224+0000","last_clean":"2024-09-13T15:03:06.502224+0000","last_became_active":"2024-09-13T15:02:39.884349+0000","last_became_peered":"2024-09-13T15:02:39.884349+0000","last_unstale":"2024-09-13T15:03:06.502224+0000","last_undegraded":"2024-09-13T15:03:06.502224+0000","last_fullsized":"2024-09-13T15:03:06.502224+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":2643,"log_dups_size":0,"ondisk_log_size":2643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173211648,"num_objects":2643,"num_object_clones":0,"num_object_copies":7929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2643,"num_write_kb":169152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":111,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:53.820580+0000","last_change":"2024-09-13T15:02:53.820519+0000","last_active":"2024-09-13T15:02:53.820580+0000","last_peered":"2024-09-13T15:02:53.820580+0000","last_clean":"2024-09-13T15:02:53.820580+0000","last_became_active":"2024-09-13T14:59:53.222306+0000","last_became_peered":"2024-09-13T14:59:53.222306+0000","last_unstale":"2024-09-13T15:02:53.820580+0000","last_undegraded":"2024-09-13T15:02:53.820580+0000","last_fullsized":"2024-09-13T15:02:53.820580+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:53.820435+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:53.820435+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:40.108691+0000","scrub_duration":0.0030873739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":580,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.305203+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T15:02:59.305203+0000","last_peered":"2024-09-13T15:02:59.305203+0000","last_clean":"2024-09-13T15:02:59.305203+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T15:02:59.305203+0000","last_undegraded":"2024-09-13T15:02:59.305203+0000","last_fullsized":"2024-09-13T15:02:59.305203+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":347,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.769513+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T15:02:49.769513+0000","last_peered":"2024-09-13T15:02:49.769513+0000","last_clean":"2024-09-13T15:02:49.769513+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T15:02:49.769513+0000","last_undegraded":"2024-09-13T15:02:49.769513+0000","last_fullsized":"2024-09-13T15:02:49.769513+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"361'2660","reported_seq":2674,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.506886+0000","last_change":"2024-09-13T15:02:39.921810+0000","last_active":"2024-09-13T15:03:06.506886+0000","last_peered":"2024-09-13T15:03:06.506886+0000","last_clean":"2024-09-13T15:03:06.506886+0000","last_became_active":"2024-09-13T15:02:39.921510+0000","last_became_peered":"2024-09-13T15:02:39.921510+0000","last_unstale":"2024-09-13T15:03:06.506886+0000","last_undegraded":"2024-09-13T15:03:06.506886+0000","last_fullsized":"2024-09-13T15:03:06.506886+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":2660,"log_dups_size":0,"ondisk_log_size":2660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:53:53.564891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174325760,"num_objects":2660,"num_object_clones":0,"num_object_copies":7980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2660,"num_write_kb":170240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":541,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383205+0000","last_change":"2024-09-13T15:00:10.016332+0000","last_active":"2024-09-13T15:03:09.383205+0000","last_peered":"2024-09-13T15:03:09.383205+0000","last_clean":"2024-09-13T15:03:09.383205+0000","last_became_active":"2024-09-13T14:59:53.227343+0000","last_became_peered":"2024-09-13T14:59:53.227343+0000","last_unstale":"2024-09-13T15:03:09.383205+0000","last_undegraded":"2024-09-13T15:03:09.383205+0000","last_fullsized":"2024-09-13T15:03:09.383205+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.016260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.016260+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:26:31.269839+0000","scrub_duration":71,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":327,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.242394+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T15:02:49.242394+0000","last_peered":"2024-09-13T15:02:49.242394+0000","last_clean":"2024-09-13T15:02:49.242394+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T15:02:49.242394+0000","last_undegraded":"2024-09-13T15:02:49.242394+0000","last_fullsized":"2024-09-13T15:02:49.242394+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":392,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.346482+0000","last_change":"2024-09-13T14:59:00.929566+0000","last_active":"2024-09-13T15:02:49.346482+0000","last_peered":"2024-09-13T15:02:49.346482+0000","last_clean":"2024-09-13T15:02:49.346482+0000","last_became_active":"2024-09-13T14:59:00.929271+0000","last_became_peered":"2024-09-13T14:59:00.929271+0000","last_unstale":"2024-09-13T15:02:49.346482+0000","last_undegraded":"2024-09-13T15:02:49.346482+0000","last_fullsized":"2024-09-13T15:02:49.346482+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:25.655536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"361'2986","reported_seq":3011,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383047+0000","last_change":"2024-09-13T15:02:39.880701+0000","last_active":"2024-09-13T15:03:09.383047+0000","last_peered":"2024-09-13T15:03:09.383047+0000","last_clean":"2024-09-13T15:03:09.383047+0000","last_became_active":"2024-09-13T15:02:39.880319+0000","last_became_peered":"2024-09-13T15:02:39.880319+0000","last_unstale":"2024-09-13T15:03:09.383047+0000","last_undegraded":"2024-09-13T15:03:09.383047+0000","last_fullsized":"2024-09-13T15:03:09.383047+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":2986,"log_dups_size":0,"ondisk_log_size":2986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:25:44.594262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195690496,"num_objects":2986,"num_object_clones":0,"num_object_copies":8958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2986,"num_write_kb":191104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":194,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.322116+0000","last_change":"2024-09-13T14:56:08.889974+0000","last_active":"2024-09-13T15:02:49.322116+0000","last_peered":"2024-09-13T15:02:49.322116+0000","last_clean":"2024-09-13T15:02:49.322116+0000","last_became_active":"2024-09-13T14:55:55.185755+0000","last_became_peered":"2024-09-13T14:55:55.185755+0000","last_unstale":"2024-09-13T15:02:49.322116+0000","last_undegraded":"2024-09-13T15:02:49.322116+0000","last_fullsized":"2024-09-13T15:02:49.322116+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:08.889814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:08.889814+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:02.911564+0000","scrub_duration":0.0053731569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":337,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.242378+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T15:02:49.242378+0000","last_peered":"2024-09-13T15:02:49.242378+0000","last_clean":"2024-09-13T15:02:49.242378+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T15:02:49.242378+0000","last_undegraded":"2024-09-13T15:02:49.242378+0000","last_fullsized":"2024-09-13T15:02:49.242378+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":165,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.321683+0000","last_change":"2024-09-13T15:02:38.875727+0000","last_active":"2024-09-13T15:02:49.321683+0000","last_peered":"2024-09-13T15:02:49.321683+0000","last_clean":"2024-09-13T15:02:49.321683+0000","last_became_active":"2024-09-13T15:02:38.875372+0000","last_became_peered":"2024-09-13T15:02:38.875372+0000","last_unstale":"2024-09-13T15:02:49.321683+0000","last_undegraded":"2024-09-13T15:02:49.321683+0000","last_fullsized":"2024-09-13T15:02:49.321683+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:08.504810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"361'3036","reported_seq":3060,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.385039+0000","last_change":"2024-09-13T15:02:39.921711+0000","last_active":"2024-09-13T15:03:09.385039+0000","last_peered":"2024-09-13T15:03:09.385039+0000","last_clean":"2024-09-13T15:03:09.385039+0000","last_became_active":"2024-09-13T15:02:39.921415+0000","last_became_peered":"2024-09-13T15:02:39.921415+0000","last_unstale":"2024-09-13T15:03:09.385039+0000","last_undegraded":"2024-09-13T15:03:09.385039+0000","last_fullsized":"2024-09-13T15:03:09.385039+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3036,"log_dups_size":0,"ondisk_log_size":3036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:22.478391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198967296,"num_objects":3036,"num_object_clones":0,"num_object_copies":9108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3036,"num_write_kb":194304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":121,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:55.359129+0000","last_change":"2024-09-13T15:02:55.358839+0000","last_active":"2024-09-13T15:02:55.359129+0000","last_peered":"2024-09-13T15:02:55.359129+0000","last_clean":"2024-09-13T15:02:55.359129+0000","last_became_active":"2024-09-13T14:59:53.220175+0000","last_became_peered":"2024-09-13T14:59:53.220175+0000","last_unstale":"2024-09-13T15:02:55.359129+0000","last_undegraded":"2024-09-13T15:02:55.359129+0000","last_fullsized":"2024-09-13T15:02:55.359129+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.358699+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.358699+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:13:15.538248+0000","scrub_duration":0.0023994049999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":355,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.770017+0000","last_change":"2024-09-13T14:59:00.930126+0000","last_active":"2024-09-13T15:02:49.770017+0000","last_peered":"2024-09-13T15:02:49.770017+0000","last_clean":"2024-09-13T15:02:49.770017+0000","last_became_active":"2024-09-13T14:59:00.929803+0000","last_became_peered":"2024-09-13T14:59:00.929803+0000","last_unstale":"2024-09-13T15:02:49.770017+0000","last_undegraded":"2024-09-13T15:02:49.770017+0000","last_fullsized":"2024-09-13T15:02:49.770017+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:44.456770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":592,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383316+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T15:03:09.383316+0000","last_peered":"2024-09-13T15:03:09.383316+0000","last_clean":"2024-09-13T15:03:09.383316+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T15:03:09.383316+0000","last_undegraded":"2024-09-13T15:03:09.383316+0000","last_fullsized":"2024-09-13T15:03:09.383316+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"361'2917","reported_seq":2940,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:08.905816+0000","last_change":"2024-09-13T15:02:39.881732+0000","last_active":"2024-09-13T15:03:08.905816+0000","last_peered":"2024-09-13T15:03:08.905816+0000","last_clean":"2024-09-13T15:03:08.905816+0000","last_became_active":"2024-09-13T15:02:39.880853+0000","last_became_peered":"2024-09-13T15:02:39.880853+0000","last_unstale":"2024-09-13T15:03:08.905816+0000","last_undegraded":"2024-09-13T15:03:08.905816+0000","last_fullsized":"2024-09-13T15:03:08.905816+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":2917,"log_dups_size":0,"ondisk_log_size":2917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:19.523510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191168512,"num_objects":2917,"num_object_clones":0,"num_object_copies":8751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2917,"num_write_kb":186688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":538,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:50.810254+0000","last_change":"2024-09-13T15:02:50.810133+0000","last_active":"2024-09-13T15:02:50.810254+0000","last_peered":"2024-09-13T15:02:50.810254+0000","last_clean":"2024-09-13T15:02:50.810254+0000","last_became_active":"2024-09-13T14:59:53.234208+0000","last_became_peered":"2024-09-13T14:59:53.234208+0000","last_unstale":"2024-09-13T15:02:50.810254+0000","last_undegraded":"2024-09-13T15:02:50.810254+0000","last_fullsized":"2024-09-13T15:02:50.810254+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:50.810045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:50.810045+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:37:54.116248+0000","scrub_duration":0.0042318529999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":131,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.382730+0000","last_change":"2024-09-13T14:59:00.936078+0000","last_active":"2024-09-13T15:03:09.382730+0000","last_peered":"2024-09-13T15:03:09.382730+0000","last_clean":"2024-09-13T15:03:09.382730+0000","last_became_active":"2024-09-13T14:59:00.925997+0000","last_became_peered":"2024-09-13T14:59:00.925997+0000","last_unstale":"2024-09-13T15:03:09.382730+0000","last_undegraded":"2024-09-13T15:03:09.382730+0000","last_fullsized":"2024-09-13T15:03:09.382730+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:43.716136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":355,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.346970+0000","last_change":"2024-09-13T14:59:00.934706+0000","last_active":"2024-09-13T15:02:49.346970+0000","last_peered":"2024-09-13T15:02:49.346970+0000","last_clean":"2024-09-13T15:02:49.346970+0000","last_became_active":"2024-09-13T14:59:00.934367+0000","last_became_peered":"2024-09-13T14:59:00.934367+0000","last_unstale":"2024-09-13T15:02:49.346970+0000","last_undegraded":"2024-09-13T15:02:49.346970+0000","last_fullsized":"2024-09-13T15:02:49.346970+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:37:52.860543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"361'2713","reported_seq":2727,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.494446+0000","last_change":"2024-09-13T15:02:39.923388+0000","last_active":"2024-09-13T15:03:06.494446+0000","last_peered":"2024-09-13T15:03:06.494446+0000","last_clean":"2024-09-13T15:03:06.494446+0000","last_became_active":"2024-09-13T15:02:39.921428+0000","last_became_peered":"2024-09-13T15:02:39.921428+0000","last_unstale":"2024-09-13T15:03:06.494446+0000","last_undegraded":"2024-09-13T15:03:06.494446+0000","last_fullsized":"2024-09-13T15:03:06.494446+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":2713,"log_dups_size":0,"ondisk_log_size":2713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:55.689072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177799168,"num_objects":2713,"num_object_clones":0,"num_object_copies":8139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2713,"num_write_kb":173632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":302,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:57.138986+0000","last_change":"2024-09-13T15:02:57.138892+0000","last_active":"2024-09-13T15:02:57.138986+0000","last_peered":"2024-09-13T15:02:57.138986+0000","last_clean":"2024-09-13T15:02:57.138986+0000","last_became_active":"2024-09-13T14:59:53.223473+0000","last_became_peered":"2024-09-13T14:59:53.223473+0000","last_unstale":"2024-09-13T15:02:57.138986+0000","last_undegraded":"2024-09-13T15:02:57.138986+0000","last_fullsized":"2024-09-13T15:02:57.138986+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:57.138789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:57.138789+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:17.296460+0000","scrub_duration":0.0036861670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":145,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.321748+0000","last_change":"2024-09-13T14:48:32.200521+0000","last_active":"2024-09-13T15:02:49.321748+0000","last_peered":"2024-09-13T15:02:49.321748+0000","last_clean":"2024-09-13T15:02:49.321748+0000","last_became_active":"2024-09-13T14:48:32.200256+0000","last_became_peered":"2024-09-13T14:48:32.200256+0000","last_unstale":"2024-09-13T15:02:49.321748+0000","last_undegraded":"2024-09-13T15:02:49.321748+0000","last_fullsized":"2024-09-13T15:02:49.321748+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:38.899629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":342,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.347022+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T15:02:49.347022+0000","last_peered":"2024-09-13T15:02:49.347022+0000","last_clean":"2024-09-13T15:02:49.347022+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T15:02:49.347022+0000","last_undegraded":"2024-09-13T15:02:49.347022+0000","last_fullsized":"2024-09-13T15:02:49.347022+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"361'2643","reported_seq":2657,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.525008+0000","last_change":"2024-09-13T15:02:39.881185+0000","last_active":"2024-09-13T15:03:06.525008+0000","last_peered":"2024-09-13T15:03:06.525008+0000","last_clean":"2024-09-13T15:03:06.525008+0000","last_became_active":"2024-09-13T15:02:39.880841+0000","last_became_peered":"2024-09-13T15:02:39.880841+0000","last_unstale":"2024-09-13T15:03:06.525008+0000","last_undegraded":"2024-09-13T15:03:06.525008+0000","last_fullsized":"2024-09-13T15:03:06.525008+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":2643,"log_dups_size":0,"ondisk_log_size":2643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:49:37.338743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173211648,"num_objects":2643,"num_object_clones":0,"num_object_copies":7929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2643,"num_write_kb":169152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":772,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301362+0000","last_change":"2024-09-13T15:00:10.659910+0000","last_active":"2024-09-13T15:02:59.301362+0000","last_peered":"2024-09-13T15:02:59.301362+0000","last_clean":"2024-09-13T15:02:59.301362+0000","last_became_active":"2024-09-13T14:59:53.216739+0000","last_became_peered":"2024-09-13T14:59:53.216739+0000","last_unstale":"2024-09-13T15:02:59.301362+0000","last_undegraded":"2024-09-13T15:02:59.301362+0000","last_fullsized":"2024-09-13T15:02:59.301362+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.659838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.659838+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:05:29.052471+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":578,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383867+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T15:03:09.383867+0000","last_peered":"2024-09-13T15:03:09.383867+0000","last_clean":"2024-09-13T15:03:09.383867+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T15:03:09.383867+0000","last_undegraded":"2024-09-13T15:03:09.383867+0000","last_fullsized":"2024-09-13T15:03:09.383867+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":345,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.347214+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T15:02:49.347214+0000","last_peered":"2024-09-13T15:02:49.347214+0000","last_clean":"2024-09-13T15:02:49.347214+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T15:02:49.347214+0000","last_undegraded":"2024-09-13T15:02:49.347214+0000","last_fullsized":"2024-09-13T15:02:49.347214+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"361'2707","reported_seq":2720,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.539296+0000","last_change":"2024-09-13T15:02:39.881232+0000","last_active":"2024-09-13T15:03:06.539296+0000","last_peered":"2024-09-13T15:03:06.539296+0000","last_clean":"2024-09-13T15:03:06.539296+0000","last_became_active":"2024-09-13T15:02:39.880899+0000","last_became_peered":"2024-09-13T15:02:39.880899+0000","last_unstale":"2024-09-13T15:03:06.539296+0000","last_undegraded":"2024-09-13T15:03:06.539296+0000","last_fullsized":"2024-09-13T15:03:06.539296+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":2707,"log_dups_size":0,"ondisk_log_size":2707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:53:52.712556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177405952,"num_objects":2707,"num_object_clones":0,"num_object_copies":8121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2707,"num_write_kb":173248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":291,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:58.197499+0000","last_change":"2024-09-13T15:02:58.197427+0000","last_active":"2024-09-13T15:02:58.197499+0000","last_peered":"2024-09-13T15:02:58.197499+0000","last_clean":"2024-09-13T15:02:58.197499+0000","last_became_active":"2024-09-13T14:59:53.232987+0000","last_became_peered":"2024-09-13T14:59:53.232987+0000","last_unstale":"2024-09-13T15:02:58.197499+0000","last_undegraded":"2024-09-13T15:02:58.197499+0000","last_fullsized":"2024-09-13T15:02:58.197499+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:58.195731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:58.195731+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:51.565489+0000","scrub_duration":0.011077536000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":330,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301918+0000","last_change":"2024-09-13T14:59:00.931433+0000","last_active":"2024-09-13T15:02:59.301918+0000","last_peered":"2024-09-13T15:02:59.301918+0000","last_clean":"2024-09-13T15:02:59.301918+0000","last_became_active":"2024-09-13T14:59:00.928906+0000","last_became_peered":"2024-09-13T14:59:00.928906+0000","last_unstale":"2024-09-13T15:02:59.301918+0000","last_undegraded":"2024-09-13T15:02:59.301918+0000","last_fullsized":"2024-09-13T15:02:59.301918+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:12.786568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":584,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300985+0000","last_change":"2024-09-13T14:59:00.925277+0000","last_active":"2024-09-13T15:02:59.300985+0000","last_peered":"2024-09-13T15:02:59.300985+0000","last_clean":"2024-09-13T15:02:59.300985+0000","last_became_active":"2024-09-13T14:59:00.924952+0000","last_became_peered":"2024-09-13T14:59:00.924952+0000","last_unstale":"2024-09-13T15:02:59.300985+0000","last_undegraded":"2024-09-13T15:02:59.300985+0000","last_fullsized":"2024-09-13T15:02:59.300985+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:08.888187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"361'2645","reported_seq":2659,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.529003+0000","last_change":"2024-09-13T15:02:39.880337+0000","last_active":"2024-09-13T15:03:06.529003+0000","last_peered":"2024-09-13T15:03:06.529003+0000","last_clean":"2024-09-13T15:03:06.529003+0000","last_became_active":"2024-09-13T15:02:39.880044+0000","last_became_peered":"2024-09-13T15:02:39.880044+0000","last_unstale":"2024-09-13T15:03:06.529003+0000","last_undegraded":"2024-09-13T15:03:06.529003+0000","last_fullsized":"2024-09-13T15:03:06.529003+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":2645,"log_dups_size":0,"ondisk_log_size":2645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:38.582990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173342720,"num_objects":2645,"num_object_clones":0,"num_object_copies":7935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2645,"num_write_kb":169280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":509,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.241925+0000","last_change":"2024-09-13T14:55:58.568852+0000","last_active":"2024-09-13T15:02:49.241925+0000","last_peered":"2024-09-13T15:02:49.241925+0000","last_clean":"2024-09-13T15:02:49.241925+0000","last_became_active":"2024-09-13T14:55:51.169987+0000","last_became_peered":"2024-09-13T14:55:51.169987+0000","last_unstale":"2024-09-13T15:02:49.241925+0000","last_undegraded":"2024-09-13T15:02:49.241925+0000","last_fullsized":"2024-09-13T15:02:49.241925+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.568750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.568750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:56.273429+0000","scrub_duration":0.0074886340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":887,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.769968+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T15:02:49.769968+0000","last_peered":"2024-09-13T15:02:49.769968+0000","last_clean":"2024-09-13T15:02:49.769968+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T15:02:49.769968+0000","last_undegraded":"2024-09-13T15:02:49.769968+0000","last_fullsized":"2024-09-13T15:02:49.769968+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":181,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.769594+0000","last_change":"2024-09-13T14:59:00.930790+0000","last_active":"2024-09-13T15:02:49.769594+0000","last_peered":"2024-09-13T15:02:49.769594+0000","last_clean":"2024-09-13T15:02:49.769594+0000","last_became_active":"2024-09-13T14:59:00.930439+0000","last_became_peered":"2024-09-13T14:59:00.930439+0000","last_unstale":"2024-09-13T15:02:49.769594+0000","last_undegraded":"2024-09-13T15:02:49.769594+0000","last_fullsized":"2024-09-13T15:02:49.769594+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:15.636095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":145,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.322008+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T15:02:49.322008+0000","last_peered":"2024-09-13T15:02:49.322008+0000","last_clean":"2024-09-13T15:02:49.322008+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T15:02:49.322008+0000","last_undegraded":"2024-09-13T15:02:49.322008+0000","last_fullsized":"2024-09-13T15:02:49.322008+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"361'3030","reported_seq":3055,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383463+0000","last_change":"2024-09-13T15:02:39.921581+0000","last_active":"2024-09-13T15:03:09.383463+0000","last_peered":"2024-09-13T15:03:09.383463+0000","last_clean":"2024-09-13T15:03:09.383463+0000","last_became_active":"2024-09-13T15:02:39.921286+0000","last_became_peered":"2024-09-13T15:02:39.921286+0000","last_unstale":"2024-09-13T15:03:09.383463+0000","last_undegraded":"2024-09-13T15:03:09.383463+0000","last_fullsized":"2024-09-13T15:03:09.383463+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3030,"log_dups_size":0,"ondisk_log_size":3030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:54.996704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198574080,"num_objects":3030,"num_object_clones":0,"num_object_copies":9090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3030,"num_write_kb":193920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":743,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301138+0000","last_change":"2024-09-13T14:55:59.245841+0000","last_active":"2024-09-13T15:02:59.301138+0000","last_peered":"2024-09-13T15:02:59.301138+0000","last_clean":"2024-09-13T15:02:59.301138+0000","last_became_active":"2024-09-13T14:55:51.159487+0000","last_became_peered":"2024-09-13T14:55:51.159487+0000","last_unstale":"2024-09-13T15:02:59.301138+0000","last_undegraded":"2024-09-13T15:02:59.301138+0000","last_fullsized":"2024-09-13T15:02:59.301138+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.245213+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.245213+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:16.628694+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":572,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.241948+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T15:02:49.241948+0000","last_peered":"2024-09-13T15:02:49.241948+0000","last_clean":"2024-09-13T15:02:49.241948+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T15:02:49.241948+0000","last_undegraded":"2024-09-13T15:02:49.241948+0000","last_fullsized":"2024-09-13T15:02:49.241948+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":621,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384043+0000","last_change":"2024-09-13T14:59:00.928965+0000","last_active":"2024-09-13T15:03:09.384043+0000","last_peered":"2024-09-13T15:03:09.384043+0000","last_clean":"2024-09-13T15:03:09.384043+0000","last_became_active":"2024-09-13T14:59:00.928166+0000","last_became_peered":"2024-09-13T14:59:00.928166+0000","last_unstale":"2024-09-13T15:03:09.384043+0000","last_undegraded":"2024-09-13T15:03:09.384043+0000","last_fullsized":"2024-09-13T15:03:09.384043+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:34.878172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":617,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384250+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T15:03:09.384250+0000","last_peered":"2024-09-13T15:03:09.384250+0000","last_clean":"2024-09-13T15:03:09.384250+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T15:03:09.384250+0000","last_undegraded":"2024-09-13T15:03:09.384250+0000","last_fullsized":"2024-09-13T15:03:09.384250+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"361'2592","reported_seq":2606,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.507427+0000","last_change":"2024-09-13T15:02:39.921711+0000","last_active":"2024-09-13T15:03:06.507427+0000","last_peered":"2024-09-13T15:03:06.507427+0000","last_clean":"2024-09-13T15:03:06.507427+0000","last_became_active":"2024-09-13T15:02:39.921395+0000","last_became_peered":"2024-09-13T15:02:39.921395+0000","last_unstale":"2024-09-13T15:03:06.507427+0000","last_undegraded":"2024-09-13T15:03:06.507427+0000","last_fullsized":"2024-09-13T15:03:06.507427+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":2592,"log_dups_size":0,"ondisk_log_size":2592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:12.076600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169869312,"num_objects":2592,"num_object_clones":0,"num_object_copies":7776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2592,"num_write_kb":165888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":138,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.241869+0000","last_change":"2024-09-13T14:55:59.574827+0000","last_active":"2024-09-13T15:02:49.241869+0000","last_peered":"2024-09-13T15:02:49.241869+0000","last_clean":"2024-09-13T15:02:49.241869+0000","last_became_active":"2024-09-13T14:55:51.162903+0000","last_became_peered":"2024-09-13T14:55:51.162903+0000","last_unstale":"2024-09-13T15:02:49.241869+0000","last_undegraded":"2024-09-13T15:02:49.241869+0000","last_fullsized":"2024-09-13T15:02:49.241869+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.574773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.574773+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:39.074072+0000","scrub_duration":0.001553708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":542,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384128+0000","last_change":"2024-09-13T14:59:00.950349+0000","last_active":"2024-09-13T15:03:09.384128+0000","last_peered":"2024-09-13T15:03:09.384128+0000","last_clean":"2024-09-13T15:03:09.384128+0000","last_became_active":"2024-09-13T14:59:00.925703+0000","last_became_peered":"2024-09-13T14:59:00.925703+0000","last_unstale":"2024-09-13T15:03:09.384128+0000","last_undegraded":"2024-09-13T15:03:09.384128+0000","last_fullsized":"2024-09-13T15:03:09.384128+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:43.851359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":593,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.302053+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T15:02:59.302053+0000","last_peered":"2024-09-13T15:02:59.302053+0000","last_clean":"2024-09-13T15:02:59.302053+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T15:02:59.302053+0000","last_undegraded":"2024-09-13T15:02:59.302053+0000","last_fullsized":"2024-09-13T15:02:59.302053+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":611,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300931+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T15:02:59.300931+0000","last_peered":"2024-09-13T15:02:59.300931+0000","last_clean":"2024-09-13T15:02:59.300931+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T15:02:59.300931+0000","last_undegraded":"2024-09-13T15:02:59.300931+0000","last_fullsized":"2024-09-13T15:02:59.300931+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"361'2710","reported_seq":2724,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.513062+0000","last_change":"2024-09-13T15:02:39.882940+0000","last_active":"2024-09-13T15:03:06.513062+0000","last_peered":"2024-09-13T15:03:06.513062+0000","last_clean":"2024-09-13T15:03:06.513062+0000","last_became_active":"2024-09-13T15:02:39.882641+0000","last_became_peered":"2024-09-13T15:02:39.882641+0000","last_unstale":"2024-09-13T15:03:06.513062+0000","last_undegraded":"2024-09-13T15:03:06.513062+0000","last_fullsized":"2024-09-13T15:03:06.513062+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":2710,"log_dups_size":0,"ondisk_log_size":2710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:10.481971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177602560,"num_objects":2710,"num_object_clones":0,"num_object_copies":8130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2710,"num_write_kb":173440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":281,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.769556+0000","last_change":"2024-09-13T14:56:01.658240+0000","last_active":"2024-09-13T15:02:49.769556+0000","last_peered":"2024-09-13T15:02:49.769556+0000","last_clean":"2024-09-13T15:02:49.769556+0000","last_became_active":"2024-09-13T14:55:51.160480+0000","last_became_peered":"2024-09-13T14:55:51.160480+0000","last_unstale":"2024-09-13T15:02:49.769556+0000","last_undegraded":"2024-09-13T15:02:49.769556+0000","last_fullsized":"2024-09-13T15:02:49.769556+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.658143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.658143+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:19.262955+0000","scrub_duration":0.0038903079999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":987,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.769876+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T15:02:49.769876+0000","last_peered":"2024-09-13T15:02:49.769876+0000","last_clean":"2024-09-13T15:02:49.769876+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T15:02:49.769876+0000","last_undegraded":"2024-09-13T15:02:49.769876+0000","last_fullsized":"2024-09-13T15:02:49.769876+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":346,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.347052+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T15:02:49.347052+0000","last_peered":"2024-09-13T15:02:49.347052+0000","last_clean":"2024-09-13T15:02:49.347052+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T15:02:49.347052+0000","last_undegraded":"2024-09-13T15:02:49.347052+0000","last_fullsized":"2024-09-13T15:02:49.347052+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":617,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301485+0000","last_change":"2024-09-13T14:59:00.930713+0000","last_active":"2024-09-13T15:02:59.301485+0000","last_peered":"2024-09-13T15:02:59.301485+0000","last_clean":"2024-09-13T15:02:59.301485+0000","last_became_active":"2024-09-13T14:59:00.928260+0000","last_became_peered":"2024-09-13T14:59:00.928260+0000","last_unstale":"2024-09-13T15:02:59.301485+0000","last_undegraded":"2024-09-13T15:02:59.301485+0000","last_fullsized":"2024-09-13T15:02:59.301485+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:07.048853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"361'2652","reported_seq":2666,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.494469+0000","last_change":"2024-09-13T15:02:39.882990+0000","last_active":"2024-09-13T15:03:06.494469+0000","last_peered":"2024-09-13T15:03:06.494469+0000","last_clean":"2024-09-13T15:03:06.494469+0000","last_became_active":"2024-09-13T15:02:39.882663+0000","last_became_peered":"2024-09-13T15:02:39.882663+0000","last_unstale":"2024-09-13T15:03:06.494469+0000","last_undegraded":"2024-09-13T15:03:06.494469+0000","last_fullsized":"2024-09-13T15:03:06.494469+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":2652,"log_dups_size":0,"ondisk_log_size":2652,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:20.368697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173801472,"num_objects":2652,"num_object_clones":0,"num_object_copies":7956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2652,"num_write_kb":169728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":224,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.771693+0000","last_change":"2024-09-13T15:02:49.771643+0000","last_active":"2024-09-13T15:02:49.771693+0000","last_peered":"2024-09-13T15:02:49.771693+0000","last_clean":"2024-09-13T15:02:49.771693+0000","last_became_active":"2024-09-13T14:59:53.249243+0000","last_became_peered":"2024-09-13T14:59:53.249243+0000","last_unstale":"2024-09-13T15:02:49.771693+0000","last_undegraded":"2024-09-13T15:02:49.771693+0000","last_fullsized":"2024-09-13T15:02:49.771693+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:49.771572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:49.771572+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:11:27.741449+0000","scrub_duration":0.0035237329999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":352,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.770158+0000","last_change":"2024-09-13T14:59:00.931326+0000","last_active":"2024-09-13T15:02:49.770158+0000","last_peered":"2024-09-13T15:02:49.770158+0000","last_clean":"2024-09-13T15:02:49.770158+0000","last_became_active":"2024-09-13T14:59:00.931011+0000","last_became_peered":"2024-09-13T14:59:00.931011+0000","last_unstale":"2024-09-13T15:02:49.770158+0000","last_undegraded":"2024-09-13T15:02:49.770158+0000","last_fullsized":"2024-09-13T15:02:49.770158+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:24:41.833263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":1095,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.302270+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T15:02:59.302270+0000","last_peered":"2024-09-13T15:02:59.302270+0000","last_clean":"2024-09-13T15:02:59.302270+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T15:02:59.302270+0000","last_undegraded":"2024-09-13T15:02:59.302270+0000","last_fullsized":"2024-09-13T15:02:59.302270+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":567,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300786+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T15:02:59.300786+0000","last_peered":"2024-09-13T15:02:59.300786+0000","last_clean":"2024-09-13T15:02:59.300786+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T15:02:59.300786+0000","last_undegraded":"2024-09-13T15:02:59.300786+0000","last_fullsized":"2024-09-13T15:02:59.300786+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"361'2671","reported_seq":2684,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.531821+0000","last_change":"2024-09-13T15:02:39.881285+0000","last_active":"2024-09-13T15:03:06.531821+0000","last_peered":"2024-09-13T15:03:06.531821+0000","last_clean":"2024-09-13T15:03:06.531821+0000","last_became_active":"2024-09-13T15:02:39.880984+0000","last_became_peered":"2024-09-13T15:02:39.880984+0000","last_unstale":"2024-09-13T15:03:06.531821+0000","last_undegraded":"2024-09-13T15:03:06.531821+0000","last_fullsized":"2024-09-13T15:03:06.531821+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":2671,"log_dups_size":0,"ondisk_log_size":2671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:31.193802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175046656,"num_objects":2671,"num_object_clones":0,"num_object_copies":8013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2671,"num_write_kb":170944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":214,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:51.748718+0000","last_change":"2024-09-13T15:02:51.748425+0000","last_active":"2024-09-13T15:02:51.748718+0000","last_peered":"2024-09-13T15:02:51.748718+0000","last_clean":"2024-09-13T15:02:51.748718+0000","last_became_active":"2024-09-13T14:59:53.226288+0000","last_became_peered":"2024-09-13T14:59:53.226288+0000","last_unstale":"2024-09-13T15:02:51.748718+0000","last_undegraded":"2024-09-13T15:02:51.748718+0000","last_fullsized":"2024-09-13T15:02:51.748718+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:51.748121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:51.748121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:17:47.226084+0000","scrub_duration":0.0026525849999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":592,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.302171+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T15:02:59.302171+0000","last_peered":"2024-09-13T15:02:59.302171+0000","last_clean":"2024-09-13T15:02:59.302171+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T15:02:59.302171+0000","last_undegraded":"2024-09-13T15:02:59.302171+0000","last_fullsized":"2024-09-13T15:02:59.302171+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":574,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.242426+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T15:02:49.242426+0000","last_peered":"2024-09-13T15:02:49.242426+0000","last_clean":"2024-09-13T15:02:49.242426+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T15:02:49.242426+0000","last_undegraded":"2024-09-13T15:02:49.242426+0000","last_fullsized":"2024-09-13T15:02:49.242426+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":582,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383236+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T15:03:09.383236+0000","last_peered":"2024-09-13T15:03:09.383236+0000","last_clean":"2024-09-13T15:03:09.383236+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T15:03:09.383236+0000","last_undegraded":"2024-09-13T15:03:09.383236+0000","last_fullsized":"2024-09-13T15:03:09.383236+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"361'2695","reported_seq":2708,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.532632+0000","last_change":"2024-09-13T15:02:39.921655+0000","last_active":"2024-09-13T15:03:06.532632+0000","last_peered":"2024-09-13T15:03:06.532632+0000","last_clean":"2024-09-13T15:03:06.532632+0000","last_became_active":"2024-09-13T15:02:39.921331+0000","last_became_peered":"2024-09-13T15:02:39.921331+0000","last_unstale":"2024-09-13T15:03:06.532632+0000","last_undegraded":"2024-09-13T15:03:06.532632+0000","last_fullsized":"2024-09-13T15:03:06.532632+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":2695,"log_dups_size":0,"ondisk_log_size":2695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:48.004179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176619520,"num_objects":2695,"num_object_clones":0,"num_object_copies":8085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2695,"num_write_kb":172480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":332,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:52.361508+0000","last_change":"2024-09-13T15:02:52.361444+0000","last_active":"2024-09-13T15:02:52.361508+0000","last_peered":"2024-09-13T15:02:52.361508+0000","last_clean":"2024-09-13T15:02:52.361508+0000","last_became_active":"2024-09-13T14:59:53.220135+0000","last_became_peered":"2024-09-13T14:59:53.220135+0000","last_unstale":"2024-09-13T15:02:52.361508+0000","last_undegraded":"2024-09-13T15:02:52.361508+0000","last_fullsized":"2024-09-13T15:02:52.361508+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.361359+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.361359+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:35.003140+0000","scrub_duration":0.005845154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":82,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.242515+0000","last_change":"2024-09-13T14:59:00.936064+0000","last_active":"2024-09-13T15:02:49.242515+0000","last_peered":"2024-09-13T15:02:49.242515+0000","last_clean":"2024-09-13T15:02:49.242515+0000","last_became_active":"2024-09-13T14:59:00.932311+0000","last_became_peered":"2024-09-13T14:59:00.932311+0000","last_unstale":"2024-09-13T15:02:49.242515+0000","last_undegraded":"2024-09-13T15:02:49.242515+0000","last_fullsized":"2024-09-13T15:02:49.242515+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:16.456004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":950,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.302219+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T15:02:59.302219+0000","last_peered":"2024-09-13T15:02:59.302219+0000","last_clean":"2024-09-13T15:02:59.302219+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T15:02:59.302219+0000","last_undegraded":"2024-09-13T15:02:59.302219+0000","last_fullsized":"2024-09-13T15:02:59.302219+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":83,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.241426+0000","last_change":"2024-09-13T14:59:00.924324+0000","last_active":"2024-09-13T15:02:49.241426+0000","last_peered":"2024-09-13T15:02:49.241426+0000","last_clean":"2024-09-13T15:02:49.241426+0000","last_became_active":"2024-09-13T14:59:00.923798+0000","last_became_peered":"2024-09-13T14:59:00.923798+0000","last_unstale":"2024-09-13T15:02:49.241426+0000","last_undegraded":"2024-09-13T15:02:49.241426+0000","last_fullsized":"2024-09-13T15:02:49.241426+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:10.994877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"361'2700","reported_seq":2714,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.511185+0000","last_change":"2024-09-13T15:02:39.922368+0000","last_active":"2024-09-13T15:03:06.511185+0000","last_peered":"2024-09-13T15:03:06.511185+0000","last_clean":"2024-09-13T15:03:06.511185+0000","last_became_active":"2024-09-13T15:02:39.922106+0000","last_became_peered":"2024-09-13T15:02:39.922106+0000","last_unstale":"2024-09-13T15:03:06.511185+0000","last_undegraded":"2024-09-13T15:03:06.511185+0000","last_fullsized":"2024-09-13T15:03:06.511185+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":2700,"log_dups_size":0,"ondisk_log_size":2700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:13:52.416513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176947200,"num_objects":2700,"num_object_clones":0,"num_object_copies":8100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2700,"num_write_kb":172800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":69,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:54.193286+0000","last_change":"2024-09-13T15:02:54.193233+0000","last_active":"2024-09-13T15:02:54.193286+0000","last_peered":"2024-09-13T15:02:54.193286+0000","last_clean":"2024-09-13T15:02:54.193286+0000","last_became_active":"2024-09-13T14:59:00.925167+0000","last_became_peered":"2024-09-13T14:59:00.925167+0000","last_unstale":"2024-09-13T15:02:54.193286+0000","last_undegraded":"2024-09-13T15:02:54.193286+0000","last_fullsized":"2024-09-13T15:02:54.193286+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:54.193156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:54.193156+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:16.851666+0000","scrub_duration":0.0040069900000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":603,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383725+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T15:03:09.383725+0000","last_peered":"2024-09-13T15:03:09.383725+0000","last_clean":"2024-09-13T15:03:09.383725+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T15:03:09.383725+0000","last_undegraded":"2024-09-13T15:03:09.383725+0000","last_fullsized":"2024-09-13T15:03:09.383725+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1174,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.322217+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T15:02:49.322217+0000","last_peered":"2024-09-13T15:02:49.322217+0000","last_clean":"2024-09-13T15:02:49.322217+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T15:02:49.322217+0000","last_undegraded":"2024-09-13T15:02:49.322217+0000","last_fullsized":"2024-09-13T15:02:49.322217+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":328,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.346383+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T15:02:49.346383+0000","last_peered":"2024-09-13T15:02:49.346383+0000","last_clean":"2024-09-13T15:02:49.346383+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T15:02:49.346383+0000","last_undegraded":"2024-09-13T15:02:49.346383+0000","last_fullsized":"2024-09-13T15:02:49.346383+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"82'100","reported_seq":53,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.302028+0000","last_change":"2024-09-13T15:02:51.192587+0000","last_active":"2024-09-13T15:02:59.302028+0000","last_peered":"2024-09-13T15:02:59.302028+0000","last_clean":"2024-09-13T15:02:59.302028+0000","last_became_active":"2024-09-13T15:02:23.878033+0000","last_became_peered":"2024-09-13T15:02:23.878033+0000","last_unstale":"2024-09-13T15:02:59.302028+0000","last_undegraded":"2024-09-13T15:02:59.302028+0000","last_fullsized":"2024-09-13T15:02:59.302028+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:51.192503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:51.192503+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:25:55.606880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":345,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.346893+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T15:02:49.346893+0000","last_peered":"2024-09-13T15:02:49.346893+0000","last_clean":"2024-09-13T15:02:49.346893+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T15:02:49.346893+0000","last_undegraded":"2024-09-13T15:02:49.346893+0000","last_fullsized":"2024-09-13T15:02:49.346893+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":610,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383296+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T15:03:09.383296+0000","last_peered":"2024-09-13T15:03:09.383296+0000","last_clean":"2024-09-13T15:03:09.383296+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T15:03:09.383296+0000","last_undegraded":"2024-09-13T15:03:09.383296+0000","last_fullsized":"2024-09-13T15:03:09.383296+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.17","version":"82'100","reported_seq":173,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.108074+0000","last_change":"2024-09-13T15:03:03.108031+0000","last_active":"2024-09-13T15:03:03.108074+0000","last_peered":"2024-09-13T15:03:03.108074+0000","last_clean":"2024-09-13T15:03:03.108074+0000","last_became_active":"2024-09-13T15:02:23.883373+0000","last_became_peered":"2024-09-13T15:02:23.883373+0000","last_unstale":"2024-09-13T15:03:03.108074+0000","last_undegraded":"2024-09-13T15:03:03.108074+0000","last_fullsized":"2024-09-13T15:03:03.108074+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:03.107786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:03.107786+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:17:10.041468+0000","scrub_duration":0.0049024139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":591,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300689+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T15:02:59.300689+0000","last_peered":"2024-09-13T15:02:59.300689+0000","last_clean":"2024-09-13T15:02:59.300689+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T15:02:59.300689+0000","last_undegraded":"2024-09-13T15:02:59.300689+0000","last_fullsized":"2024-09-13T15:02:59.300689+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":351,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.242170+0000","last_change":"2024-09-13T14:59:00.929406+0000","last_active":"2024-09-13T15:02:49.242170+0000","last_peered":"2024-09-13T15:02:49.242170+0000","last_clean":"2024-09-13T15:02:49.242170+0000","last_became_active":"2024-09-13T14:59:00.928524+0000","last_became_peered":"2024-09-13T14:59:00.928524+0000","last_unstale":"2024-09-13T15:02:49.242170+0000","last_undegraded":"2024-09-13T15:02:49.242170+0000","last_fullsized":"2024-09-13T15:02:49.242170+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:10.609448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":172,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303796+0000","last_change":"2024-09-13T14:56:01.235655+0000","last_active":"2024-09-13T15:02:59.303796+0000","last_peered":"2024-09-13T15:02:59.303796+0000","last_clean":"2024-09-13T15:02:59.303796+0000","last_became_active":"2024-09-13T14:55:55.195703+0000","last_became_peered":"2024-09-13T14:55:55.195703+0000","last_unstale":"2024-09-13T15:02:59.303796+0000","last_undegraded":"2024-09-13T15:02:59.303796+0000","last_fullsized":"2024-09-13T15:02:59.303796+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.235497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.235497+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:23:21.640831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":325,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.770162+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T15:02:49.770162+0000","last_peered":"2024-09-13T15:02:49.770162+0000","last_clean":"2024-09-13T15:02:49.770162+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T15:02:49.770162+0000","last_undegraded":"2024-09-13T15:02:49.770162+0000","last_fullsized":"2024-09-13T15:02:49.770162+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":332,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.241651+0000","last_change":"2024-09-13T14:59:00.926586+0000","last_active":"2024-09-13T15:02:49.241651+0000","last_peered":"2024-09-13T15:02:49.241651+0000","last_clean":"2024-09-13T15:02:49.241651+0000","last_became_active":"2024-09-13T14:59:00.926206+0000","last_became_peered":"2024-09-13T14:59:00.926206+0000","last_unstale":"2024-09-13T15:02:49.241651+0000","last_undegraded":"2024-09-13T15:02:49.241651+0000","last_fullsized":"2024-09-13T15:02:49.241651+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:17:18.805856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.15","version":"82'100","reported_seq":29,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:55.870766+0000","last_change":"2024-09-13T15:02:55.870615+0000","last_active":"2024-09-13T15:02:55.870766+0000","last_peered":"2024-09-13T15:02:55.870766+0000","last_clean":"2024-09-13T15:02:55.870766+0000","last_became_active":"2024-09-13T15:02:23.883217+0000","last_became_peered":"2024-09-13T15:02:23.883217+0000","last_unstale":"2024-09-13T15:02:55.870766+0000","last_undegraded":"2024-09-13T15:02:55.870766+0000","last_fullsized":"2024-09-13T15:02:55.870766+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.870528+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.870528+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:10.023787+0000","scrub_duration":0.0047551850000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":597,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383095+0000","last_change":"2024-09-13T14:59:00.932503+0000","last_active":"2024-09-13T15:03:09.383095+0000","last_peered":"2024-09-13T15:03:09.383095+0000","last_clean":"2024-09-13T15:03:09.383095+0000","last_became_active":"2024-09-13T14:59:00.931358+0000","last_became_peered":"2024-09-13T14:59:00.931358+0000","last_unstale":"2024-09-13T15:03:09.383095+0000","last_undegraded":"2024-09-13T15:03:09.383095+0000","last_fullsized":"2024-09-13T15:03:09.383095+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:19.226182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":427,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300579+0000","last_change":"2024-09-13T14:59:00.927060+0000","last_active":"2024-09-13T15:02:59.300579+0000","last_peered":"2024-09-13T15:02:59.300579+0000","last_clean":"2024-09-13T15:02:59.300579+0000","last_became_active":"2024-09-13T14:59:00.925402+0000","last_became_peered":"2024-09-13T14:59:00.925402+0000","last_unstale":"2024-09-13T15:02:59.300579+0000","last_undegraded":"2024-09-13T15:02:59.300579+0000","last_fullsized":"2024-09-13T15:02:59.300579+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:31:08.336446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"82'100","reported_seq":29,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:56.319446+0000","last_change":"2024-09-13T15:02:56.319389+0000","last_active":"2024-09-13T15:02:56.319446+0000","last_peered":"2024-09-13T15:02:56.319446+0000","last_clean":"2024-09-13T15:02:56.319446+0000","last_became_active":"2024-09-13T15:02:23.876739+0000","last_became_peered":"2024-09-13T15:02:23.876739+0000","last_unstale":"2024-09-13T15:02:56.319446+0000","last_undegraded":"2024-09-13T15:02:56.319446+0000","last_fullsized":"2024-09-13T15:02:56.319446+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:56.319302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:56.319302+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:58.230158+0000","scrub_duration":0.0031182190000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":145,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.322127+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T15:02:49.322127+0000","last_peered":"2024-09-13T15:02:49.322127+0000","last_clean":"2024-09-13T15:02:49.322127+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T15:02:49.322127+0000","last_undegraded":"2024-09-13T15:02:49.322127+0000","last_fullsized":"2024-09-13T15:02:49.322127+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":187,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.382484+0000","last_change":"2024-09-13T14:59:00.924196+0000","last_active":"2024-09-13T15:03:09.382484+0000","last_peered":"2024-09-13T15:03:09.382484+0000","last_clean":"2024-09-13T15:03:09.382484+0000","last_became_active":"2024-09-13T14:59:00.923901+0000","last_became_peered":"2024-09-13T14:59:00.923901+0000","last_unstale":"2024-09-13T15:03:09.382484+0000","last_undegraded":"2024-09-13T15:03:09.382484+0000","last_fullsized":"2024-09-13T15:03:09.382484+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:01.913804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"82'100","reported_seq":601,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.769687+0000","last_change":"2024-09-13T15:02:23.883854+0000","last_active":"2024-09-13T15:02:49.769687+0000","last_peered":"2024-09-13T15:02:49.769687+0000","last_clean":"2024-09-13T15:02:49.769687+0000","last_became_active":"2024-09-13T15:02:23.883387+0000","last_became_peered":"2024-09-13T15:02:23.883387+0000","last_unstale":"2024-09-13T15:02:49.769687+0000","last_undegraded":"2024-09-13T15:02:49.769687+0000","last_fullsized":"2024-09-13T15:02:49.769687+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:05.698273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:05.698273+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":621,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384165+0000","last_change":"2024-09-13T14:59:00.924814+0000","last_active":"2024-09-13T15:03:09.384165+0000","last_peered":"2024-09-13T15:03:09.384165+0000","last_clean":"2024-09-13T15:03:09.384165+0000","last_became_active":"2024-09-13T14:59:00.924501+0000","last_became_peered":"2024-09-13T14:59:00.924501+0000","last_unstale":"2024-09-13T15:03:09.384165+0000","last_undegraded":"2024-09-13T15:03:09.384165+0000","last_fullsized":"2024-09-13T15:03:09.384165+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:11:15.382207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":583,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300594+0000","last_change":"2024-09-13T14:59:00.925431+0000","last_active":"2024-09-13T15:02:59.300594+0000","last_peered":"2024-09-13T15:02:59.300594+0000","last_clean":"2024-09-13T15:02:59.300594+0000","last_became_active":"2024-09-13T14:59:00.925023+0000","last_became_peered":"2024-09-13T14:59:00.925023+0000","last_unstale":"2024-09-13T15:02:59.300594+0000","last_undegraded":"2024-09-13T15:02:59.300594+0000","last_fullsized":"2024-09-13T15:02:59.300594+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:38:18.686253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"82'100","reported_seq":316,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:04.101951+0000","last_change":"2024-09-13T15:03:04.101898+0000","last_active":"2024-09-13T15:03:04.101951+0000","last_peered":"2024-09-13T15:03:04.101951+0000","last_clean":"2024-09-13T15:03:04.101951+0000","last_became_active":"2024-09-13T15:02:23.884406+0000","last_became_peered":"2024-09-13T15:02:23.884406+0000","last_unstale":"2024-09-13T15:03:04.101951+0000","last_undegraded":"2024-09-13T15:03:04.101951+0000","last_fullsized":"2024-09-13T15:03:04.101951+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:04.101802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:04.101802+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:34.459623+0000","scrub_duration":0.003875819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":135,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.346922+0000","last_change":"2024-09-13T14:59:00.927250+0000","last_active":"2024-09-13T15:02:49.346922+0000","last_peered":"2024-09-13T15:02:49.346922+0000","last_clean":"2024-09-13T15:02:49.346922+0000","last_became_active":"2024-09-13T14:59:00.926953+0000","last_became_peered":"2024-09-13T14:59:00.926953+0000","last_unstale":"2024-09-13T15:02:49.346922+0000","last_undegraded":"2024-09-13T15:02:49.346922+0000","last_fullsized":"2024-09-13T15:02:49.346922+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:58.306547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":344,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.346633+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T15:02:49.346633+0000","last_peered":"2024-09-13T15:02:49.346633+0000","last_clean":"2024-09-13T15:02:49.346633+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T15:02:49.346633+0000","last_undegraded":"2024-09-13T15:02:49.346633+0000","last_fullsized":"2024-09-13T15:02:49.346633+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"82'100","reported_seq":55,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383145+0000","last_change":"2024-09-13T15:02:52.429623+0000","last_active":"2024-09-13T15:03:09.383145+0000","last_peered":"2024-09-13T15:03:09.383145+0000","last_clean":"2024-09-13T15:03:09.383145+0000","last_became_active":"2024-09-13T15:02:23.880201+0000","last_became_peered":"2024-09-13T15:02:23.880201+0000","last_unstale":"2024-09-13T15:03:09.383145+0000","last_undegraded":"2024-09-13T15:03:09.383145+0000","last_fullsized":"2024-09-13T15:03:09.383145+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.429543+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.429543+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:43.145050+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":417,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.302517+0000","last_change":"2024-09-13T14:59:00.934412+0000","last_active":"2024-09-13T15:02:59.302517+0000","last_peered":"2024-09-13T15:02:59.302517+0000","last_clean":"2024-09-13T15:02:59.302517+0000","last_became_active":"2024-09-13T14:59:00.932986+0000","last_became_peered":"2024-09-13T14:59:00.932986+0000","last_unstale":"2024-09-13T15:02:59.302517+0000","last_undegraded":"2024-09-13T15:02:59.302517+0000","last_fullsized":"2024-09-13T15:02:59.302517+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:21:44.285324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":582,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.382827+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T15:03:09.382827+0000","last_peered":"2024-09-13T15:03:09.382827+0000","last_clean":"2024-09-13T15:03:09.382827+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T15:03:09.382827+0000","last_undegraded":"2024-09-13T15:03:09.382827+0000","last_fullsized":"2024-09-13T15:03:09.382827+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"82'100","reported_seq":29,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:55.680054+0000","last_change":"2024-09-13T15:02:55.679993+0000","last_active":"2024-09-13T15:02:55.680054+0000","last_peered":"2024-09-13T15:02:55.680054+0000","last_clean":"2024-09-13T15:02:55.680054+0000","last_became_active":"2024-09-13T15:02:23.878770+0000","last_became_peered":"2024-09-13T15:02:23.878770+0000","last_unstale":"2024-09-13T15:02:55.680054+0000","last_undegraded":"2024-09-13T15:02:55.680054+0000","last_fullsized":"2024-09-13T15:02:55.680054+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.679893+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.679893+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:25.717099+0000","scrub_duration":0.0050514899999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":352,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.346868+0000","last_change":"2024-09-13T14:59:00.929751+0000","last_active":"2024-09-13T15:02:49.346868+0000","last_peered":"2024-09-13T15:02:49.346868+0000","last_clean":"2024-09-13T15:02:49.346868+0000","last_became_active":"2024-09-13T14:59:00.929456+0000","last_became_peered":"2024-09-13T14:59:00.929456+0000","last_unstale":"2024-09-13T15:02:49.346868+0000","last_undegraded":"2024-09-13T15:02:49.346868+0000","last_fullsized":"2024-09-13T15:02:49.346868+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:11.462853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":327,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.769883+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T15:02:49.769883+0000","last_peered":"2024-09-13T15:02:49.769883+0000","last_clean":"2024-09-13T15:02:49.769883+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T15:02:49.769883+0000","last_undegraded":"2024-09-13T15:02:49.769883+0000","last_fullsized":"2024-09-13T15:02:49.769883+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":352,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.346603+0000","last_change":"2024-09-13T14:59:00.928898+0000","last_active":"2024-09-13T15:02:49.346603+0000","last_peered":"2024-09-13T15:02:49.346603+0000","last_clean":"2024-09-13T15:02:49.346603+0000","last_became_active":"2024-09-13T14:59:00.928585+0000","last_became_peered":"2024-09-13T14:59:00.928585+0000","last_unstale":"2024-09-13T15:02:49.346603+0000","last_undegraded":"2024-09-13T15:02:49.346603+0000","last_fullsized":"2024-09-13T15:02:49.346603+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:26:57.155149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":366,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.770053+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T15:02:49.770053+0000","last_peered":"2024-09-13T15:02:49.770053+0000","last_clean":"2024-09-13T15:02:49.770053+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T15:02:49.770053+0000","last_undegraded":"2024-09-13T15:02:49.770053+0000","last_fullsized":"2024-09-13T15:02:49.770053+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"82'100","reported_seq":19,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.241967+0000","last_change":"2024-09-13T15:02:23.879862+0000","last_active":"2024-09-13T15:02:49.241967+0000","last_peered":"2024-09-13T15:02:49.241967+0000","last_clean":"2024-09-13T15:02:49.241967+0000","last_became_active":"2024-09-13T15:02:23.879170+0000","last_became_peered":"2024-09-13T15:02:23.879170+0000","last_unstale":"2024-09-13T15:02:49.241967+0000","last_undegraded":"2024-09-13T15:02:49.241967+0000","last_fullsized":"2024-09-13T15:02:49.241967+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:06.945525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:06.945525+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:48.496118+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":153,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384043+0000","last_change":"2024-09-13T14:59:00.929835+0000","last_active":"2024-09-13T15:03:09.384043+0000","last_peered":"2024-09-13T15:03:09.384043+0000","last_clean":"2024-09-13T15:03:09.384043+0000","last_became_active":"2024-09-13T14:59:00.929514+0000","last_became_peered":"2024-09-13T14:59:00.929514+0000","last_unstale":"2024-09-13T15:03:09.384043+0000","last_undegraded":"2024-09-13T15:03:09.384043+0000","last_fullsized":"2024-09-13T15:03:09.384043+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:48.981240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":388,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.321990+0000","last_change":"2024-09-13T14:59:00.925895+0000","last_active":"2024-09-13T15:02:49.321990+0000","last_peered":"2024-09-13T15:02:49.321990+0000","last_clean":"2024-09-13T15:02:49.321990+0000","last_became_active":"2024-09-13T14:59:00.925564+0000","last_became_peered":"2024-09-13T14:59:00.925564+0000","last_unstale":"2024-09-13T15:02:49.321990+0000","last_undegraded":"2024-09-13T15:02:49.321990+0000","last_fullsized":"2024-09-13T15:02:49.321990+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:23.283927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1d","version":"82'100","reported_seq":55,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300923+0000","last_change":"2024-09-13T15:02:52.149390+0000","last_active":"2024-09-13T15:02:59.300923+0000","last_peered":"2024-09-13T15:02:59.300923+0000","last_clean":"2024-09-13T15:02:59.300923+0000","last_became_active":"2024-09-13T15:02:23.914809+0000","last_became_peered":"2024-09-13T15:02:23.914809+0000","last_unstale":"2024-09-13T15:02:59.300923+0000","last_undegraded":"2024-09-13T15:02:59.300923+0000","last_fullsized":"2024-09-13T15:02:59.300923+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.149290+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.149290+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:59.584474+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":348,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T15:02:49.346375+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T15:02:49.346375+0000","last_peered":"2024-09-13T15:02:49.346375+0000","last_clean":"2024-09-13T15:02:49.346375+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T15:02:49.346375+0000","last_undegraded":"2024-09-13T15:02:49.346375+0000","last_fullsized":"2024-09-13T15:02:49.346375+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":635,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.382997+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T15:03:09.382997+0000","last_peered":"2024-09-13T15:03:09.382997+0000","last_clean":"2024-09-13T15:03:09.382997+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T15:03:09.382997+0000","last_undegraded":"2024-09-13T15:03:09.382997+0000","last_fullsized":"2024-09-13T15:03:09.382997+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2883584000,"num_objects":44000,"num_object_clones":0,"num_object_copies":132000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":44000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44000,"num_write_kb":2816000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8842903552,"data_stored":8842903552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":44000,"ondisk_log_size":44000,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":16172,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397950,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1718328,"kb_used_data":837232,"kb_used_omap":1,"kb_used_meta":881086,"kb_avail":92026824,"statfs":{"total":95995035648,"available":94235467776,"internally_reserved":0,"allocated":857325568,"data_stored":854827041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":902232408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594179,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1471660,"kb_used_data":1165988,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":92273492,"statfs":{"total":95995035648,"available":94488055808,"internally_reserved":0,"allocated":1193971712,"data_stored":1191484501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790408,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1743336,"kb_used_data":846304,"kb_used_omap":1,"kb_used_meta":897022,"kb_avail":92001816,"statfs":{"total":95995035648,"available":94209859584,"internally_reserved":0,"allocated":866615296,"data_stored":864128622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":918550872},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986637,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2595304,"kb_used_data":1698080,"kb_used_omap":1,"kb_used_meta":897214,"kb_avail":91149848,"statfs":{"total":95995035648,"available":93337444352,"internally_reserved":0,"allocated":1738833920,"data_stored":1736346433,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":918747480},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":0,"seq":1438814044184,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762612,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2210728,"kb_used_data":1303584,"kb_used_omap":8,"kb_used_meta":907127,"kb_avail":91534424,"statfs":{"total":95995035648,"available":93731250176,"internally_reserved":0,"allocated":1334870016,"data_stored":1332383337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":928898506},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926138,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1615136,"kb_used_data":730840,"kb_used_omap":7,"kb_used_meta":884280,"kb_avail":92130016,"statfs":{"total":95995035648,"available":94341136384,"internally_reserved":0,"allocated":748380160,"data_stored":745900766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":905503433},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122378,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2688756,"kb_used_data":1856684,"kb_used_omap":7,"kb_used_meta":832056,"kb_avail":91056396,"statfs":{"total":95995035648,"available":93241749504,"internally_reserved":0,"allocated":1901244416,"data_stored":1898749949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":852025935},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1966669824,"data_stored":1966669824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":763822080,"data_stored":763822080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1382875136,"data_stored":1382875136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1751646208,"data_stored":1751646208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":872808448,"data_stored":872808448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1227292672,"data_stored":1227292672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":877789184,"data_stored":877789184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:03:12.746 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:03:12.790 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:12.825 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:12.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:12.927 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:12.973 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:12.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:13.076 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:13.106 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:13.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:13.161 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 23 16 54622 54606 148.337 105.312 0.0100983 0.00673695 2024-09-13T15:03:13.207 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:13.244 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:13.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:13.346 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:13.383 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:13.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:13.485 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:13.523 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:13.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:13.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:13 smithi096 ceph-mon[98901]: pgmap v1201: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 120 MiB/s wr, 1.91k op/s 2024-09-13T15:03:13.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:13 smithi096 ceph-mon[102928]: pgmap v1201: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 120 MiB/s wr, 1.91k op/s 2024-09-13T15:03:13.625 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:13.656 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:13.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:13.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:13 smithi204 ceph-mon[87071]: pgmap v1201: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 120 MiB/s wr, 1.91k op/s 2024-09-13T15:03:13.758 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:13.794 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:13.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:13.895 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:13.925 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:13.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:14.026 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:14.061 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:14.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:14.161 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 24 16 56269 56253 146.445 102.938 0.0109327 0.0068238 2024-09-13T15:03:14.163 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:14.196 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:14.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:14.298 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:14.330 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:14.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:14.432 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:14.467 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:14.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:14.569 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:14.607 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:14.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:14.709 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:14.740 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:14.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:14.842 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:14.884 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:14.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:14.985 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:15.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.023 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:15.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:15.124 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:15.161 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 25 16 57899 57883 144.659 101.875 0.00825236 0.00690771 2024-09-13T15:03:15.169 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:15.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:15.271 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:15.306 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:15.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:15.408 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:15.415 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:15.437 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T15:03:15.437 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T15:03:15.437 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 0 2024-09-13T15:03:15.437 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T15:03:15.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.438 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:15.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:15.539 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:15.572 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:15.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:15.674 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:15.711 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:15.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:15.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:15 smithi204 ceph-mon[87071]: pgmap v1202: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-13T15:03:15.813 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:15.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:15 smithi096 ceph-mon[98901]: pgmap v1202: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-13T15:03:15.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:15 smithi096 ceph-mon[102928]: pgmap v1202: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-13T15:03:15.847 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:15.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:15.949 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:15.981 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:15.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:16.083 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:16.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.122 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:16.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:16.162 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 26 16 59491 59475 142.922 99.5 0.00912534 0.00699278 2024-09-13T15:03:16.223 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:16.254 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:16.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:16.299 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T15:03:16.357 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:16.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.397 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:16.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:16.410 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:03:16.497 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:16.532 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:16.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:16.634 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:16.667 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:16.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:16.769 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:16.800 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:16.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:16.837 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:03:16.902 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:16.942 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:16.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:17.044 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:17.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.083 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:17.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:17.162 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 27 16 61028 61012 141.187 96.0625 0.00538418 0.00707765 2024-09-13T15:03:17.184 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:17.219 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:17.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:17.321 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:17.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.360 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:17.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:17.461 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:17.499 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:17.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:17.536 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:17 smithi096 ceph-mon[98901]: pgmap v1203: 119 pgs: 119 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 117 MiB/s wr, 1.86k op/s 2024-09-13T15:03:17.536 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:17 smithi096 ceph-mon[102928]: pgmap v1203: 119 pgs: 119 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 117 MiB/s wr, 1.86k op/s 2024-09-13T15:03:17.601 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:17.653 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:17.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.670 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:17.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:17.681 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:17.712 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:17.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:17 smithi204 ceph-mon[87071]: pgmap v1203: 119 pgs: 119 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 117 MiB/s wr, 1.86k op/s 2024-09-13T15:03:17.771 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:17.805 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:17.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:17.906 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:17.943 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:17.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:18.045 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:18.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:03:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:03:17] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-09-13T15:03:18.084 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:18.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:18.164 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 28 16 62543 62527 139.526 94.6875 0.00562201 0.00716211 2024-09-13T15:03:18.186 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:18.230 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:18.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:18.331 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:18.372 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:18.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:18.474 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:18.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.512 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:18.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:18.613 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:18.648 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:18.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:18.750 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:18.785 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:18.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:18.887 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:18.917 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:18.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:19.019 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:19.050 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:19.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:19.152 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:19.162 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 29 16 64091 64075 138.051 96.75 0.00685241 0.00723861 2024-09-13T15:03:19.182 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:19.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:19.284 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:19.320 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:19.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:19.422 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:19.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.458 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:19.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:19.558 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:19.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:19 smithi096 ceph-mon[98901]: pgmap v1204: 119 pgs: 119 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-09-13T15:03:19.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:19 smithi096 ceph-mon[102928]: pgmap v1204: 119 pgs: 119 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-09-13T15:03:19.599 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:19.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:19.643 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T15:03:19.644 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T15:03:19.701 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:19.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:19 smithi204 ceph-mon[87071]: pgmap v1204: 119 pgs: 119 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-09-13T15:03:19.742 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:19.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:19.844 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:19.877 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:19.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:19.979 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:20.013 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:20.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:20.116 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:20.154 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:20.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:20.162 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 30 16 65583 65567 136.558 93.25 0.00752243 0.00731776 2024-09-13T15:03:20.259 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:20.299 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:20.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:20.401 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:20.439 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:20.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:20.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:20 smithi204 ceph-mon[87071]: from='client.55894 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:03:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:03:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:03:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:03:20.541 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:20.579 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:20.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:20.681 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:20.717 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:20.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:20.819 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:20.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:20 smithi096 ceph-mon[98901]: from='client.55894 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:03:20.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:20 smithi096 ceph-mon[102928]: from='client.55894 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:03:20.849 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:20.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:20.951 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:20.988 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:20.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:21.090 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:21.126 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:21.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:21.162 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 31 16 67036 67020 135.082 90.8125 0.0181716 0.0073984 2024-09-13T15:03:21.228 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:21.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.257 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:21.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:21.358 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:21.388 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:21.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:21.489 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:21.517 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:21.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:21.619 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:21.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.649 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:21.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:21.750 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:21.777 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:21.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:21.879 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:21.907 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:21.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:22.009 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:22.045 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:22.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:22.147 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:22.162 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 32 16 68481 68465 133.683 90.3125 0.00770568 0.00747587 2024-09-13T15:03:22.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.183 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:22.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:21 smithi204 ceph-mon[87071]: pgmap v1205: 119 pgs: 119 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-09-13T15:03:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:03:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:22.283 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:22.319 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:22.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:22.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:21 smithi096 ceph-mon[98901]: pgmap v1205: 119 pgs: 119 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-09-13T15:03:22.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:03:22.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:21 smithi096 ceph-mon[102928]: pgmap v1205: 119 pgs: 119 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-09-13T15:03:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:03:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:22.420 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:22.441 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:22.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:22.543 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:22.574 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:22.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:22.675 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:22.711 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:22.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:22.789 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":1204,"stamp":"2024-09-13T15:03:18.177532+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3883014093,"num_objects":59205,"num_object_clones":0,"num_object_copies":177615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":59205,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":60084,"num_write_kb":3799951,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":109,"num_bytes_recovered":8404818,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":63309,"ondisk_log_size":63309,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16682976,"kb_used_data":11065960,"kb_used_omap":29,"kb_used_meta":5616930,"kb_avail":639533088,"statfs":{"total":671965249536,"available":654881882112,"internally_reserved":0,"allocated":11331543040,"data_stored":11314068078,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30654,"internal_metadata":5751736386},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,8,10,6,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1146355712,"num_objects":17492,"num_object_clones":0,"num_object_copies":52476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17492,"num_write_kb":1119488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003453"},"pg_stats":[{"pgid":"1.11","version":"82'100","reported_seq":327,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383953+0000","last_change":"2024-09-13T14:59:00.929470+0000","last_active":"2024-09-13T15:03:09.383953+0000","last_peered":"2024-09-13T15:03:09.383953+0000","last_clean":"2024-09-13T15:03:09.383953+0000","last_became_active":"2024-09-13T14:59:00.928726+0000","last_became_peered":"2024-09-13T14:59:00.928726+0000","last_unstale":"2024-09-13T15:03:09.383953+0000","last_undegraded":"2024-09-13T15:03:09.383953+0000","last_fullsized":"2024-09-13T15:03:09.383953+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.087994+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.087994+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:41.304122+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":169,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384071+0000","last_change":"2024-09-13T14:59:00.935629+0000","last_active":"2024-09-13T15:03:09.384071+0000","last_peered":"2024-09-13T15:03:09.384071+0000","last_clean":"2024-09-13T15:03:09.384071+0000","last_became_active":"2024-09-13T14:59:00.935106+0000","last_became_peered":"2024-09-13T14:59:00.935106+0000","last_unstale":"2024-09-13T15:03:09.384071+0000","last_undegraded":"2024-09-13T15:03:09.384071+0000","last_fullsized":"2024-09-13T15:03:09.384071+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:12.481049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":328,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301561+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T15:02:59.301561+0000","last_peered":"2024-09-13T15:02:59.301561+0000","last_clean":"2024-09-13T15:02:59.301561+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T15:02:59.301561+0000","last_undegraded":"2024-09-13T15:02:59.301561+0000","last_fullsized":"2024-09-13T15:02:59.301561+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":92,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303200+0000","last_change":"2024-09-13T14:56:04.643672+0000","last_active":"2024-09-13T15:02:59.303200+0000","last_peered":"2024-09-13T15:02:59.303200+0000","last_clean":"2024-09-13T15:02:59.303200+0000","last_became_active":"2024-09-13T14:55:55.185995+0000","last_became_peered":"2024-09-13T14:55:55.185995+0000","last_unstale":"2024-09-13T15:02:59.303200+0000","last_undegraded":"2024-09-13T15:02:59.303200+0000","last_fullsized":"2024-09-13T15:02:59.303200+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.643614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.643614+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:36.495546+0000","scrub_duration":0.001441681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":335,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303291+0000","last_change":"2024-09-13T14:59:00.926231+0000","last_active":"2024-09-13T15:02:59.303291+0000","last_peered":"2024-09-13T15:02:59.303291+0000","last_clean":"2024-09-13T15:02:59.303291+0000","last_became_active":"2024-09-13T14:59:00.925808+0000","last_became_peered":"2024-09-13T14:59:00.925808+0000","last_unstale":"2024-09-13T15:02:59.303291+0000","last_undegraded":"2024-09-13T15:02:59.303291+0000","last_fullsized":"2024-09-13T15:02:59.303291+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:39.672143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":618,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384469+0000","last_change":"2024-09-13T14:59:00.923848+0000","last_active":"2024-09-13T15:03:09.384469+0000","last_peered":"2024-09-13T15:03:09.384469+0000","last_clean":"2024-09-13T15:03:09.384469+0000","last_became_active":"2024-09-13T14:59:00.923522+0000","last_became_peered":"2024-09-13T14:59:00.923522+0000","last_unstale":"2024-09-13T15:03:09.384469+0000","last_undegraded":"2024-09-13T15:03:09.384469+0000","last_fullsized":"2024-09-13T15:03:09.384469+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:35:35.267527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":758,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383809+0000","last_change":"2024-09-13T14:59:00.930294+0000","last_active":"2024-09-13T15:03:09.383809+0000","last_peered":"2024-09-13T15:03:09.383809+0000","last_clean":"2024-09-13T15:03:09.383809+0000","last_became_active":"2024-09-13T14:59:00.929860+0000","last_became_peered":"2024-09-13T14:59:00.929860+0000","last_unstale":"2024-09-13T15:03:09.383809+0000","last_undegraded":"2024-09-13T15:03:09.383809+0000","last_fullsized":"2024-09-13T15:03:09.383809+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.240882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.240882+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:48:18.212524+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":612,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384229+0000","last_change":"2024-09-13T14:59:00.925448+0000","last_active":"2024-09-13T15:03:09.384229+0000","last_peered":"2024-09-13T15:03:09.384229+0000","last_clean":"2024-09-13T15:03:09.384229+0000","last_became_active":"2024-09-13T14:59:00.925152+0000","last_became_peered":"2024-09-13T14:59:00.925152+0000","last_unstale":"2024-09-13T15:03:09.384229+0000","last_undegraded":"2024-09-13T15:03:09.384229+0000","last_fullsized":"2024-09-13T15:03:09.384229+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:06.703642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":329,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301262+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T15:02:59.301262+0000","last_peered":"2024-09-13T15:02:59.301262+0000","last_clean":"2024-09-13T15:02:59.301262+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T15:02:59.301262+0000","last_undegraded":"2024-09-13T15:02:59.301262+0000","last_fullsized":"2024-09-13T15:02:59.301262+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":92,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303664+0000","last_change":"2024-09-13T14:56:03.605447+0000","last_active":"2024-09-13T15:02:59.303664+0000","last_peered":"2024-09-13T15:02:59.303664+0000","last_clean":"2024-09-13T15:02:59.303664+0000","last_became_active":"2024-09-13T14:55:55.218939+0000","last_became_peered":"2024-09-13T14:55:55.218939+0000","last_unstale":"2024-09-13T15:02:59.303664+0000","last_undegraded":"2024-09-13T15:02:59.303664+0000","last_fullsized":"2024-09-13T15:02:59.303664+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.605356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.605356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:12.467371+0000","scrub_duration":0.0020269620000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":84,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301366+0000","last_change":"2024-09-13T14:59:00.925287+0000","last_active":"2024-09-13T15:02:59.301366+0000","last_peered":"2024-09-13T15:02:59.301366+0000","last_clean":"2024-09-13T15:02:59.301366+0000","last_became_active":"2024-09-13T14:59:00.923868+0000","last_became_peered":"2024-09-13T14:59:00.923868+0000","last_unstale":"2024-09-13T15:02:59.301366+0000","last_undegraded":"2024-09-13T15:02:59.301366+0000","last_fullsized":"2024-09-13T15:02:59.301366+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:28.106847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":156,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301057+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T15:02:59.301057+0000","last_peered":"2024-09-13T15:02:59.301057+0000","last_clean":"2024-09-13T15:02:59.301057+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T15:02:59.301057+0000","last_undegraded":"2024-09-13T15:02:59.301057+0000","last_fullsized":"2024-09-13T15:02:59.301057+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"362'3709","reported_seq":3724,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:16.512905+0000","last_change":"2024-09-13T15:02:39.884666+0000","last_active":"2024-09-13T15:03:16.512905+0000","last_peered":"2024-09-13T15:03:16.512905+0000","last_clean":"2024-09-13T15:03:16.512905+0000","last_became_active":"2024-09-13T15:02:39.884349+0000","last_became_peered":"2024-09-13T15:02:39.884349+0000","last_unstale":"2024-09-13T15:03:16.512905+0000","last_undegraded":"2024-09-13T15:03:16.512905+0000","last_fullsized":"2024-09-13T15:03:16.512905+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3709,"log_dups_size":0,"ondisk_log_size":3709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243073024,"num_objects":3709,"num_object_clones":0,"num_object_copies":11127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3709,"num_write_kb":237376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":112,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300732+0000","last_change":"2024-09-13T15:02:53.820519+0000","last_active":"2024-09-13T15:02:59.300732+0000","last_peered":"2024-09-13T15:02:59.300732+0000","last_clean":"2024-09-13T15:02:59.300732+0000","last_became_active":"2024-09-13T14:59:53.222306+0000","last_became_peered":"2024-09-13T14:59:53.222306+0000","last_unstale":"2024-09-13T15:02:59.300732+0000","last_undegraded":"2024-09-13T15:02:59.300732+0000","last_fullsized":"2024-09-13T15:02:59.300732+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:53.820435+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:53.820435+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:40.108691+0000","scrub_duration":0.0030873739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":582,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.388028+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T15:03:09.388028+0000","last_peered":"2024-09-13T15:03:09.388028+0000","last_clean":"2024-09-13T15:03:09.388028+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T15:03:09.388028+0000","last_undegraded":"2024-09-13T15:03:09.388028+0000","last_fullsized":"2024-09-13T15:03:09.388028+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":348,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303568+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T15:02:59.303568+0000","last_peered":"2024-09-13T15:02:59.303568+0000","last_clean":"2024-09-13T15:02:59.303568+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T15:02:59.303568+0000","last_undegraded":"2024-09-13T15:02:59.303568+0000","last_fullsized":"2024-09-13T15:02:59.303568+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"362'3751","reported_seq":3765,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:16.541903+0000","last_change":"2024-09-13T15:02:39.921810+0000","last_active":"2024-09-13T15:03:16.541903+0000","last_peered":"2024-09-13T15:03:16.541903+0000","last_clean":"2024-09-13T15:03:16.541903+0000","last_became_active":"2024-09-13T15:02:39.921510+0000","last_became_peered":"2024-09-13T15:02:39.921510+0000","last_unstale":"2024-09-13T15:03:16.541903+0000","last_undegraded":"2024-09-13T15:03:16.541903+0000","last_fullsized":"2024-09-13T15:03:16.541903+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3751,"log_dups_size":0,"ondisk_log_size":3751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:53:53.564891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245825536,"num_objects":3751,"num_object_clones":0,"num_object_copies":11253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3751,"num_write_kb":240064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":541,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383205+0000","last_change":"2024-09-13T15:00:10.016332+0000","last_active":"2024-09-13T15:03:09.383205+0000","last_peered":"2024-09-13T15:03:09.383205+0000","last_clean":"2024-09-13T15:03:09.383205+0000","last_became_active":"2024-09-13T14:59:53.227343+0000","last_became_peered":"2024-09-13T14:59:53.227343+0000","last_unstale":"2024-09-13T15:03:09.383205+0000","last_undegraded":"2024-09-13T15:03:09.383205+0000","last_fullsized":"2024-09-13T15:03:09.383205+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.016260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.016260+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:26:31.269839+0000","scrub_duration":71,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":328,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301232+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T15:02:59.301232+0000","last_peered":"2024-09-13T15:02:59.301232+0000","last_clean":"2024-09-13T15:02:59.301232+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T15:02:59.301232+0000","last_undegraded":"2024-09-13T15:02:59.301232+0000","last_fullsized":"2024-09-13T15:02:59.301232+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":393,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298560+0000","last_change":"2024-09-13T14:59:00.929566+0000","last_active":"2024-09-13T15:02:59.298560+0000","last_peered":"2024-09-13T15:02:59.298560+0000","last_clean":"2024-09-13T15:02:59.298560+0000","last_became_active":"2024-09-13T14:59:00.929271+0000","last_became_peered":"2024-09-13T14:59:00.929271+0000","last_unstale":"2024-09-13T15:02:59.298560+0000","last_undegraded":"2024-09-13T15:02:59.298560+0000","last_fullsized":"2024-09-13T15:02:59.298560+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:25.655536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"362'3518","reported_seq":3543,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:14.369776+0000","last_change":"2024-09-13T15:02:39.880701+0000","last_active":"2024-09-13T15:03:14.369776+0000","last_peered":"2024-09-13T15:03:14.369776+0000","last_clean":"2024-09-13T15:03:14.369776+0000","last_became_active":"2024-09-13T15:02:39.880319+0000","last_became_peered":"2024-09-13T15:02:39.880319+0000","last_unstale":"2024-09-13T15:03:14.369776+0000","last_undegraded":"2024-09-13T15:03:14.369776+0000","last_fullsized":"2024-09-13T15:03:14.369776+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3518,"log_dups_size":0,"ondisk_log_size":3518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:25:44.594262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230555648,"num_objects":3518,"num_object_clones":0,"num_object_copies":10554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3518,"num_write_kb":225152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":195,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303721+0000","last_change":"2024-09-13T14:56:08.889974+0000","last_active":"2024-09-13T15:02:59.303721+0000","last_peered":"2024-09-13T15:02:59.303721+0000","last_clean":"2024-09-13T15:02:59.303721+0000","last_became_active":"2024-09-13T14:55:55.185755+0000","last_became_peered":"2024-09-13T14:55:55.185755+0000","last_unstale":"2024-09-13T15:02:59.303721+0000","last_undegraded":"2024-09-13T15:02:59.303721+0000","last_fullsized":"2024-09-13T15:02:59.303721+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:08.889814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:08.889814+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:02.911564+0000","scrub_duration":0.0053731569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":338,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301006+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T15:02:59.301006+0000","last_peered":"2024-09-13T15:02:59.301006+0000","last_clean":"2024-09-13T15:02:59.301006+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T15:02:59.301006+0000","last_undegraded":"2024-09-13T15:02:59.301006+0000","last_fullsized":"2024-09-13T15:02:59.301006+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":166,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300874+0000","last_change":"2024-09-13T15:02:38.875727+0000","last_active":"2024-09-13T15:02:59.300874+0000","last_peered":"2024-09-13T15:02:59.300874+0000","last_clean":"2024-09-13T15:02:59.300874+0000","last_became_active":"2024-09-13T15:02:38.875372+0000","last_became_peered":"2024-09-13T15:02:38.875372+0000","last_unstale":"2024-09-13T15:02:59.300874+0000","last_undegraded":"2024-09-13T15:02:59.300874+0000","last_fullsized":"2024-09-13T15:02:59.300874+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:08.504810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"362'3514","reported_seq":3539,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:14.376225+0000","last_change":"2024-09-13T15:02:39.921711+0000","last_active":"2024-09-13T15:03:14.376225+0000","last_peered":"2024-09-13T15:03:14.376225+0000","last_clean":"2024-09-13T15:03:14.376225+0000","last_became_active":"2024-09-13T15:02:39.921415+0000","last_became_peered":"2024-09-13T15:02:39.921415+0000","last_unstale":"2024-09-13T15:03:14.376225+0000","last_undegraded":"2024-09-13T15:03:14.376225+0000","last_fullsized":"2024-09-13T15:03:14.376225+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3514,"log_dups_size":0,"ondisk_log_size":3514,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:22.478391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230293504,"num_objects":3514,"num_object_clones":0,"num_object_copies":10542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3514,"num_write_kb":224896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":122,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.299943+0000","last_change":"2024-09-13T15:02:55.358839+0000","last_active":"2024-09-13T15:02:59.299943+0000","last_peered":"2024-09-13T15:02:59.299943+0000","last_clean":"2024-09-13T15:02:59.299943+0000","last_became_active":"2024-09-13T14:59:53.220175+0000","last_became_peered":"2024-09-13T14:59:53.220175+0000","last_unstale":"2024-09-13T15:02:59.299943+0000","last_undegraded":"2024-09-13T15:02:59.299943+0000","last_fullsized":"2024-09-13T15:02:59.299943+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.358699+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.358699+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:13:15.538248+0000","scrub_duration":0.0023994049999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":356,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303893+0000","last_change":"2024-09-13T14:59:00.930126+0000","last_active":"2024-09-13T15:02:59.303893+0000","last_peered":"2024-09-13T15:02:59.303893+0000","last_clean":"2024-09-13T15:02:59.303893+0000","last_became_active":"2024-09-13T14:59:00.929803+0000","last_became_peered":"2024-09-13T14:59:00.929803+0000","last_unstale":"2024-09-13T15:02:59.303893+0000","last_undegraded":"2024-09-13T15:02:59.303893+0000","last_fullsized":"2024-09-13T15:02:59.303893+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:44.456770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":592,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383316+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T15:03:09.383316+0000","last_peered":"2024-09-13T15:03:09.383316+0000","last_clean":"2024-09-13T15:03:09.383316+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T15:03:09.383316+0000","last_undegraded":"2024-09-13T15:03:09.383316+0000","last_fullsized":"2024-09-13T15:03:09.383316+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"362'3455","reported_seq":3477,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:13.921047+0000","last_change":"2024-09-13T15:02:39.881732+0000","last_active":"2024-09-13T15:03:13.921047+0000","last_peered":"2024-09-13T15:03:13.921047+0000","last_clean":"2024-09-13T15:03:13.921047+0000","last_became_active":"2024-09-13T15:02:39.880853+0000","last_became_peered":"2024-09-13T15:02:39.880853+0000","last_unstale":"2024-09-13T15:03:13.921047+0000","last_undegraded":"2024-09-13T15:03:13.921047+0000","last_fullsized":"2024-09-13T15:03:13.921047+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3455,"log_dups_size":0,"ondisk_log_size":3455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:19.523510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226426880,"num_objects":3455,"num_object_clones":0,"num_object_copies":10365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3455,"num_write_kb":221120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":539,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300644+0000","last_change":"2024-09-13T15:02:50.810133+0000","last_active":"2024-09-13T15:02:59.300644+0000","last_peered":"2024-09-13T15:02:59.300644+0000","last_clean":"2024-09-13T15:02:59.300644+0000","last_became_active":"2024-09-13T14:59:53.234208+0000","last_became_peered":"2024-09-13T14:59:53.234208+0000","last_unstale":"2024-09-13T15:02:59.300644+0000","last_undegraded":"2024-09-13T15:02:59.300644+0000","last_fullsized":"2024-09-13T15:02:59.300644+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:50.810045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:50.810045+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:37:54.116248+0000","scrub_duration":0.0042318529999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":131,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.382730+0000","last_change":"2024-09-13T14:59:00.936078+0000","last_active":"2024-09-13T15:03:09.382730+0000","last_peered":"2024-09-13T15:03:09.382730+0000","last_clean":"2024-09-13T15:03:09.382730+0000","last_became_active":"2024-09-13T14:59:00.925997+0000","last_became_peered":"2024-09-13T14:59:00.925997+0000","last_unstale":"2024-09-13T15:03:09.382730+0000","last_undegraded":"2024-09-13T15:03:09.382730+0000","last_fullsized":"2024-09-13T15:03:09.382730+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:43.716136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":356,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298632+0000","last_change":"2024-09-13T14:59:00.934706+0000","last_active":"2024-09-13T15:02:59.298632+0000","last_peered":"2024-09-13T15:02:59.298632+0000","last_clean":"2024-09-13T15:02:59.298632+0000","last_became_active":"2024-09-13T14:59:00.934367+0000","last_became_peered":"2024-09-13T14:59:00.934367+0000","last_unstale":"2024-09-13T15:02:59.298632+0000","last_undegraded":"2024-09-13T15:02:59.298632+0000","last_fullsized":"2024-09-13T15:02:59.298632+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:37:52.860543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"362'3804","reported_seq":3819,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:16.509314+0000","last_change":"2024-09-13T15:02:39.923388+0000","last_active":"2024-09-13T15:03:16.509314+0000","last_peered":"2024-09-13T15:03:16.509314+0000","last_clean":"2024-09-13T15:03:16.509314+0000","last_became_active":"2024-09-13T15:02:39.921428+0000","last_became_peered":"2024-09-13T15:02:39.921428+0000","last_unstale":"2024-09-13T15:03:16.509314+0000","last_undegraded":"2024-09-13T15:03:16.509314+0000","last_fullsized":"2024-09-13T15:03:16.509314+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3804,"log_dups_size":0,"ondisk_log_size":3804,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:55.689072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249298944,"num_objects":3804,"num_object_clones":0,"num_object_copies":11412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3804,"num_write_kb":243456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":303,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303293+0000","last_change":"2024-09-13T15:02:57.138892+0000","last_active":"2024-09-13T15:02:59.303293+0000","last_peered":"2024-09-13T15:02:59.303293+0000","last_clean":"2024-09-13T15:02:59.303293+0000","last_became_active":"2024-09-13T14:59:53.223473+0000","last_became_peered":"2024-09-13T14:59:53.223473+0000","last_unstale":"2024-09-13T15:02:59.303293+0000","last_undegraded":"2024-09-13T15:02:59.303293+0000","last_fullsized":"2024-09-13T15:02:59.303293+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:57.138789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:57.138789+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:17.296460+0000","scrub_duration":0.0036861670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":146,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301184+0000","last_change":"2024-09-13T14:48:32.200521+0000","last_active":"2024-09-13T15:02:59.301184+0000","last_peered":"2024-09-13T15:02:59.301184+0000","last_clean":"2024-09-13T15:02:59.301184+0000","last_became_active":"2024-09-13T14:48:32.200256+0000","last_became_peered":"2024-09-13T14:48:32.200256+0000","last_unstale":"2024-09-13T15:02:59.301184+0000","last_undegraded":"2024-09-13T15:02:59.301184+0000","last_fullsized":"2024-09-13T15:02:59.301184+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:38.899629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":343,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298784+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T15:02:59.298784+0000","last_peered":"2024-09-13T15:02:59.298784+0000","last_clean":"2024-09-13T15:02:59.298784+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T15:02:59.298784+0000","last_undegraded":"2024-09-13T15:02:59.298784+0000","last_fullsized":"2024-09-13T15:02:59.298784+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"362'3735","reported_seq":3750,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:16.531980+0000","last_change":"2024-09-13T15:02:39.881185+0000","last_active":"2024-09-13T15:03:16.531980+0000","last_peered":"2024-09-13T15:03:16.531980+0000","last_clean":"2024-09-13T15:03:16.531980+0000","last_became_active":"2024-09-13T15:02:39.880841+0000","last_became_peered":"2024-09-13T15:02:39.880841+0000","last_unstale":"2024-09-13T15:03:16.531980+0000","last_undegraded":"2024-09-13T15:03:16.531980+0000","last_fullsized":"2024-09-13T15:03:16.531980+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3735,"log_dups_size":0,"ondisk_log_size":3735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:49:37.338743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244776960,"num_objects":3735,"num_object_clones":0,"num_object_copies":11205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3735,"num_write_kb":239040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":774,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383945+0000","last_change":"2024-09-13T15:00:10.659910+0000","last_active":"2024-09-13T15:03:09.383945+0000","last_peered":"2024-09-13T15:03:09.383945+0000","last_clean":"2024-09-13T15:03:09.383945+0000","last_became_active":"2024-09-13T14:59:53.216739+0000","last_became_peered":"2024-09-13T14:59:53.216739+0000","last_unstale":"2024-09-13T15:03:09.383945+0000","last_undegraded":"2024-09-13T15:03:09.383945+0000","last_fullsized":"2024-09-13T15:03:09.383945+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.659838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.659838+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:05:29.052471+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":578,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383867+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T15:03:09.383867+0000","last_peered":"2024-09-13T15:03:09.383867+0000","last_clean":"2024-09-13T15:03:09.383867+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T15:03:09.383867+0000","last_undegraded":"2024-09-13T15:03:09.383867+0000","last_fullsized":"2024-09-13T15:03:09.383867+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":346,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.299155+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T15:02:59.299155+0000","last_peered":"2024-09-13T15:02:59.299155+0000","last_clean":"2024-09-13T15:02:59.299155+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T15:02:59.299155+0000","last_undegraded":"2024-09-13T15:02:59.299155+0000","last_fullsized":"2024-09-13T15:02:59.299155+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"362'3808","reported_seq":3822,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:16.542971+0000","last_change":"2024-09-13T15:02:39.881232+0000","last_active":"2024-09-13T15:03:16.542971+0000","last_peered":"2024-09-13T15:03:16.542971+0000","last_clean":"2024-09-13T15:03:16.542971+0000","last_became_active":"2024-09-13T15:02:39.880899+0000","last_became_peered":"2024-09-13T15:02:39.880899+0000","last_unstale":"2024-09-13T15:03:16.542971+0000","last_undegraded":"2024-09-13T15:03:16.542971+0000","last_fullsized":"2024-09-13T15:03:16.542971+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3808,"log_dups_size":0,"ondisk_log_size":3808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:53:52.712556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249561088,"num_objects":3808,"num_object_clones":0,"num_object_copies":11424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3808,"num_write_kb":243712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":292,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303492+0000","last_change":"2024-09-13T15:02:58.197427+0000","last_active":"2024-09-13T15:02:59.303492+0000","last_peered":"2024-09-13T15:02:59.303492+0000","last_clean":"2024-09-13T15:02:59.303492+0000","last_became_active":"2024-09-13T14:59:53.232987+0000","last_became_peered":"2024-09-13T14:59:53.232987+0000","last_unstale":"2024-09-13T15:02:59.303492+0000","last_undegraded":"2024-09-13T15:02:59.303492+0000","last_fullsized":"2024-09-13T15:02:59.303492+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:58.195731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:58.195731+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:51.565489+0000","scrub_duration":0.011077536000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":332,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383342+0000","last_change":"2024-09-13T14:59:00.931433+0000","last_active":"2024-09-13T15:03:09.383342+0000","last_peered":"2024-09-13T15:03:09.383342+0000","last_clean":"2024-09-13T15:03:09.383342+0000","last_became_active":"2024-09-13T14:59:00.928906+0000","last_became_peered":"2024-09-13T14:59:00.928906+0000","last_unstale":"2024-09-13T15:03:09.383342+0000","last_undegraded":"2024-09-13T15:03:09.383342+0000","last_fullsized":"2024-09-13T15:03:09.383342+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:12.786568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":586,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384579+0000","last_change":"2024-09-13T14:59:00.925277+0000","last_active":"2024-09-13T15:03:09.384579+0000","last_peered":"2024-09-13T15:03:09.384579+0000","last_clean":"2024-09-13T15:03:09.384579+0000","last_became_active":"2024-09-13T14:59:00.924952+0000","last_became_peered":"2024-09-13T14:59:00.924952+0000","last_unstale":"2024-09-13T15:03:09.384579+0000","last_undegraded":"2024-09-13T15:03:09.384579+0000","last_fullsized":"2024-09-13T15:03:09.384579+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:08.888187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"362'3688","reported_seq":3703,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:16.527481+0000","last_change":"2024-09-13T15:02:39.880337+0000","last_active":"2024-09-13T15:03:16.527481+0000","last_peered":"2024-09-13T15:03:16.527481+0000","last_clean":"2024-09-13T15:03:16.527481+0000","last_became_active":"2024-09-13T15:02:39.880044+0000","last_became_peered":"2024-09-13T15:02:39.880044+0000","last_unstale":"2024-09-13T15:03:16.527481+0000","last_undegraded":"2024-09-13T15:03:16.527481+0000","last_fullsized":"2024-09-13T15:03:16.527481+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3688,"log_dups_size":0,"ondisk_log_size":3688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:38.582990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241696768,"num_objects":3688,"num_object_clones":0,"num_object_copies":11064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3688,"num_write_kb":236032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":510,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301578+0000","last_change":"2024-09-13T14:55:58.568852+0000","last_active":"2024-09-13T15:02:59.301578+0000","last_peered":"2024-09-13T15:02:59.301578+0000","last_clean":"2024-09-13T15:02:59.301578+0000","last_became_active":"2024-09-13T14:55:51.169987+0000","last_became_peered":"2024-09-13T14:55:51.169987+0000","last_unstale":"2024-09-13T15:02:59.301578+0000","last_undegraded":"2024-09-13T15:02:59.301578+0000","last_fullsized":"2024-09-13T15:02:59.301578+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.568750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.568750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:56.273429+0000","scrub_duration":0.0074886340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":888,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303527+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T15:02:59.303527+0000","last_peered":"2024-09-13T15:02:59.303527+0000","last_clean":"2024-09-13T15:02:59.303527+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T15:02:59.303527+0000","last_undegraded":"2024-09-13T15:02:59.303527+0000","last_fullsized":"2024-09-13T15:02:59.303527+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":182,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303580+0000","last_change":"2024-09-13T14:59:00.930790+0000","last_active":"2024-09-13T15:02:59.303580+0000","last_peered":"2024-09-13T15:02:59.303580+0000","last_clean":"2024-09-13T15:02:59.303580+0000","last_became_active":"2024-09-13T14:59:00.930439+0000","last_became_peered":"2024-09-13T14:59:00.930439+0000","last_unstale":"2024-09-13T15:02:59.303580+0000","last_undegraded":"2024-09-13T15:02:59.303580+0000","last_fullsized":"2024-09-13T15:02:59.303580+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:15.636095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":146,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301039+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T15:02:59.301039+0000","last_peered":"2024-09-13T15:02:59.301039+0000","last_clean":"2024-09-13T15:02:59.301039+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T15:02:59.301039+0000","last_undegraded":"2024-09-13T15:02:59.301039+0000","last_fullsized":"2024-09-13T15:02:59.301039+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"362'3579","reported_seq":3604,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:14.375518+0000","last_change":"2024-09-13T15:02:39.921581+0000","last_active":"2024-09-13T15:03:14.375518+0000","last_peered":"2024-09-13T15:03:14.375518+0000","last_clean":"2024-09-13T15:03:14.375518+0000","last_became_active":"2024-09-13T15:02:39.921286+0000","last_became_peered":"2024-09-13T15:02:39.921286+0000","last_unstale":"2024-09-13T15:03:14.375518+0000","last_undegraded":"2024-09-13T15:03:14.375518+0000","last_fullsized":"2024-09-13T15:03:14.375518+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3579,"log_dups_size":0,"ondisk_log_size":3579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:54.996704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234553344,"num_objects":3579,"num_object_clones":0,"num_object_copies":10737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3579,"num_write_kb":229056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":745,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383673+0000","last_change":"2024-09-13T14:55:59.245841+0000","last_active":"2024-09-13T15:03:09.383673+0000","last_peered":"2024-09-13T15:03:09.383673+0000","last_clean":"2024-09-13T15:03:09.383673+0000","last_became_active":"2024-09-13T14:55:51.159487+0000","last_became_peered":"2024-09-13T14:55:51.159487+0000","last_unstale":"2024-09-13T15:03:09.383673+0000","last_undegraded":"2024-09-13T15:03:09.383673+0000","last_fullsized":"2024-09-13T15:03:09.383673+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.245213+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.245213+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:16.628694+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":573,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301324+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T15:02:59.301324+0000","last_peered":"2024-09-13T15:02:59.301324+0000","last_clean":"2024-09-13T15:02:59.301324+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T15:02:59.301324+0000","last_undegraded":"2024-09-13T15:02:59.301324+0000","last_fullsized":"2024-09-13T15:02:59.301324+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":621,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384043+0000","last_change":"2024-09-13T14:59:00.928965+0000","last_active":"2024-09-13T15:03:09.384043+0000","last_peered":"2024-09-13T15:03:09.384043+0000","last_clean":"2024-09-13T15:03:09.384043+0000","last_became_active":"2024-09-13T14:59:00.928166+0000","last_became_peered":"2024-09-13T14:59:00.928166+0000","last_unstale":"2024-09-13T15:03:09.384043+0000","last_undegraded":"2024-09-13T15:03:09.384043+0000","last_fullsized":"2024-09-13T15:03:09.384043+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:34.878172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":617,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384250+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T15:03:09.384250+0000","last_peered":"2024-09-13T15:03:09.384250+0000","last_clean":"2024-09-13T15:03:09.384250+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T15:03:09.384250+0000","last_undegraded":"2024-09-13T15:03:09.384250+0000","last_fullsized":"2024-09-13T15:03:09.384250+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"362'3690","reported_seq":3705,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:16.531762+0000","last_change":"2024-09-13T15:02:39.921711+0000","last_active":"2024-09-13T15:03:16.531762+0000","last_peered":"2024-09-13T15:03:16.531762+0000","last_clean":"2024-09-13T15:03:16.531762+0000","last_became_active":"2024-09-13T15:02:39.921395+0000","last_became_peered":"2024-09-13T15:02:39.921395+0000","last_unstale":"2024-09-13T15:03:16.531762+0000","last_undegraded":"2024-09-13T15:03:16.531762+0000","last_fullsized":"2024-09-13T15:03:16.531762+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3690,"log_dups_size":0,"ondisk_log_size":3690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:12.076600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241827840,"num_objects":3690,"num_object_clones":0,"num_object_copies":11070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3690,"num_write_kb":236160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":139,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300947+0000","last_change":"2024-09-13T14:55:59.574827+0000","last_active":"2024-09-13T15:02:59.300947+0000","last_peered":"2024-09-13T15:02:59.300947+0000","last_clean":"2024-09-13T15:02:59.300947+0000","last_became_active":"2024-09-13T14:55:51.162903+0000","last_became_peered":"2024-09-13T14:55:51.162903+0000","last_unstale":"2024-09-13T15:02:59.300947+0000","last_undegraded":"2024-09-13T15:02:59.300947+0000","last_fullsized":"2024-09-13T15:02:59.300947+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.574773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.574773+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:39.074072+0000","scrub_duration":0.001553708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":543,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384169+0000","last_change":"2024-09-13T14:59:00.950349+0000","last_active":"2024-09-13T15:03:09.384169+0000","last_peered":"2024-09-13T15:03:09.384169+0000","last_clean":"2024-09-13T15:03:09.384169+0000","last_became_active":"2024-09-13T14:59:00.925703+0000","last_became_peered":"2024-09-13T14:59:00.925703+0000","last_unstale":"2024-09-13T15:03:09.384169+0000","last_undegraded":"2024-09-13T15:03:09.384169+0000","last_fullsized":"2024-09-13T15:03:09.384169+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:43.851359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":595,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383456+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T15:03:09.383456+0000","last_peered":"2024-09-13T15:03:09.383456+0000","last_clean":"2024-09-13T15:03:09.383456+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T15:03:09.383456+0000","last_undegraded":"2024-09-13T15:03:09.383456+0000","last_fullsized":"2024-09-13T15:03:09.383456+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":613,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384496+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T15:03:09.384496+0000","last_peered":"2024-09-13T15:03:09.384496+0000","last_clean":"2024-09-13T15:03:09.384496+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T15:03:09.384496+0000","last_undegraded":"2024-09-13T15:03:09.384496+0000","last_fullsized":"2024-09-13T15:03:09.384496+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"362'3829","reported_seq":3844,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:16.540908+0000","last_change":"2024-09-13T15:02:39.882940+0000","last_active":"2024-09-13T15:03:16.540908+0000","last_peered":"2024-09-13T15:03:16.540908+0000","last_clean":"2024-09-13T15:03:16.540908+0000","last_became_active":"2024-09-13T15:02:39.882641+0000","last_became_peered":"2024-09-13T15:02:39.882641+0000","last_unstale":"2024-09-13T15:03:16.540908+0000","last_undegraded":"2024-09-13T15:03:16.540908+0000","last_fullsized":"2024-09-13T15:03:16.540908+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3829,"log_dups_size":0,"ondisk_log_size":3829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:10.481971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250937344,"num_objects":3829,"num_object_clones":0,"num_object_copies":11487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3829,"num_write_kb":245056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":282,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303185+0000","last_change":"2024-09-13T14:56:01.658240+0000","last_active":"2024-09-13T15:02:59.303185+0000","last_peered":"2024-09-13T15:02:59.303185+0000","last_clean":"2024-09-13T15:02:59.303185+0000","last_became_active":"2024-09-13T14:55:51.160480+0000","last_became_peered":"2024-09-13T14:55:51.160480+0000","last_unstale":"2024-09-13T15:02:59.303185+0000","last_undegraded":"2024-09-13T15:02:59.303185+0000","last_fullsized":"2024-09-13T15:02:59.303185+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.658143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.658143+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:19.262955+0000","scrub_duration":0.0038903079999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":988,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303957+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T15:02:59.303957+0000","last_peered":"2024-09-13T15:02:59.303957+0000","last_clean":"2024-09-13T15:02:59.303957+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T15:02:59.303957+0000","last_undegraded":"2024-09-13T15:02:59.303957+0000","last_fullsized":"2024-09-13T15:02:59.303957+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":347,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298513+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T15:02:59.298513+0000","last_peered":"2024-09-13T15:02:59.298513+0000","last_clean":"2024-09-13T15:02:59.298513+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T15:02:59.298513+0000","last_undegraded":"2024-09-13T15:02:59.298513+0000","last_fullsized":"2024-09-13T15:02:59.298513+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":619,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383748+0000","last_change":"2024-09-13T14:59:00.930713+0000","last_active":"2024-09-13T15:03:09.383748+0000","last_peered":"2024-09-13T15:03:09.383748+0000","last_clean":"2024-09-13T15:03:09.383748+0000","last_became_active":"2024-09-13T14:59:00.928260+0000","last_became_peered":"2024-09-13T14:59:00.928260+0000","last_unstale":"2024-09-13T15:03:09.383748+0000","last_undegraded":"2024-09-13T15:03:09.383748+0000","last_fullsized":"2024-09-13T15:03:09.383748+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:07.048853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"362'3752","reported_seq":3767,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:16.531232+0000","last_change":"2024-09-13T15:02:39.882990+0000","last_active":"2024-09-13T15:03:16.531232+0000","last_peered":"2024-09-13T15:03:16.531232+0000","last_clean":"2024-09-13T15:03:16.531232+0000","last_became_active":"2024-09-13T15:02:39.882663+0000","last_became_peered":"2024-09-13T15:02:39.882663+0000","last_unstale":"2024-09-13T15:03:16.531232+0000","last_undegraded":"2024-09-13T15:03:16.531232+0000","last_fullsized":"2024-09-13T15:03:16.531232+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3752,"log_dups_size":0,"ondisk_log_size":3752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:20.368697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245891072,"num_objects":3752,"num_object_clones":0,"num_object_copies":11256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3752,"num_write_kb":240128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":225,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303635+0000","last_change":"2024-09-13T15:02:49.771643+0000","last_active":"2024-09-13T15:02:59.303635+0000","last_peered":"2024-09-13T15:02:59.303635+0000","last_clean":"2024-09-13T15:02:59.303635+0000","last_became_active":"2024-09-13T14:59:53.249243+0000","last_became_peered":"2024-09-13T14:59:53.249243+0000","last_unstale":"2024-09-13T15:02:59.303635+0000","last_undegraded":"2024-09-13T15:02:59.303635+0000","last_fullsized":"2024-09-13T15:02:59.303635+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:49.771572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:49.771572+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:11:27.741449+0000","scrub_duration":0.0035237329999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":353,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303548+0000","last_change":"2024-09-13T14:59:00.931326+0000","last_active":"2024-09-13T15:02:59.303548+0000","last_peered":"2024-09-13T15:02:59.303548+0000","last_clean":"2024-09-13T15:02:59.303548+0000","last_became_active":"2024-09-13T14:59:00.931011+0000","last_became_peered":"2024-09-13T14:59:00.931011+0000","last_unstale":"2024-09-13T15:02:59.303548+0000","last_undegraded":"2024-09-13T15:02:59.303548+0000","last_fullsized":"2024-09-13T15:02:59.303548+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:24:41.833263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":1097,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383920+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T15:03:09.383920+0000","last_peered":"2024-09-13T15:03:09.383920+0000","last_clean":"2024-09-13T15:03:09.383920+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T15:03:09.383920+0000","last_undegraded":"2024-09-13T15:03:09.383920+0000","last_fullsized":"2024-09-13T15:03:09.383920+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":569,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383541+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T15:03:09.383541+0000","last_peered":"2024-09-13T15:03:09.383541+0000","last_clean":"2024-09-13T15:03:09.383541+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T15:03:09.383541+0000","last_undegraded":"2024-09-13T15:03:09.383541+0000","last_fullsized":"2024-09-13T15:03:09.383541+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"362'3759","reported_seq":3774,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:16.541789+0000","last_change":"2024-09-13T15:02:39.881285+0000","last_active":"2024-09-13T15:03:16.541789+0000","last_peered":"2024-09-13T15:03:16.541789+0000","last_clean":"2024-09-13T15:03:16.541789+0000","last_became_active":"2024-09-13T15:02:39.880984+0000","last_became_peered":"2024-09-13T15:02:39.880984+0000","last_unstale":"2024-09-13T15:03:16.541789+0000","last_undegraded":"2024-09-13T15:03:16.541789+0000","last_fullsized":"2024-09-13T15:03:16.541789+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3759,"log_dups_size":0,"ondisk_log_size":3759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:31.193802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246349824,"num_objects":3759,"num_object_clones":0,"num_object_copies":11277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3759,"num_write_kb":240576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":215,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300994+0000","last_change":"2024-09-13T15:02:51.748425+0000","last_active":"2024-09-13T15:02:59.300994+0000","last_peered":"2024-09-13T15:02:59.300994+0000","last_clean":"2024-09-13T15:02:59.300994+0000","last_became_active":"2024-09-13T14:59:53.226288+0000","last_became_peered":"2024-09-13T14:59:53.226288+0000","last_unstale":"2024-09-13T15:02:59.300994+0000","last_undegraded":"2024-09-13T15:02:59.300994+0000","last_fullsized":"2024-09-13T15:02:59.300994+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:51.748121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:51.748121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:17:47.226084+0000","scrub_duration":0.0026525849999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":594,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383832+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T15:03:09.383832+0000","last_peered":"2024-09-13T15:03:09.383832+0000","last_clean":"2024-09-13T15:03:09.383832+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T15:03:09.383832+0000","last_undegraded":"2024-09-13T15:03:09.383832+0000","last_fullsized":"2024-09-13T15:03:09.383832+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":575,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301269+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T15:02:59.301269+0000","last_peered":"2024-09-13T15:02:59.301269+0000","last_clean":"2024-09-13T15:02:59.301269+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T15:02:59.301269+0000","last_undegraded":"2024-09-13T15:02:59.301269+0000","last_fullsized":"2024-09-13T15:02:59.301269+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":582,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383236+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T15:03:09.383236+0000","last_peered":"2024-09-13T15:03:09.383236+0000","last_clean":"2024-09-13T15:03:09.383236+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T15:03:09.383236+0000","last_undegraded":"2024-09-13T15:03:09.383236+0000","last_fullsized":"2024-09-13T15:03:09.383236+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"362'3831","reported_seq":3846,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:16.507351+0000","last_change":"2024-09-13T15:02:39.921655+0000","last_active":"2024-09-13T15:03:16.507351+0000","last_peered":"2024-09-13T15:03:16.507351+0000","last_clean":"2024-09-13T15:03:16.507351+0000","last_became_active":"2024-09-13T15:02:39.921331+0000","last_became_peered":"2024-09-13T15:02:39.921331+0000","last_unstale":"2024-09-13T15:03:16.507351+0000","last_undegraded":"2024-09-13T15:03:16.507351+0000","last_fullsized":"2024-09-13T15:03:16.507351+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3831,"log_dups_size":0,"ondisk_log_size":3831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:48.004179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251068416,"num_objects":3831,"num_object_clones":0,"num_object_copies":11493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3831,"num_write_kb":245184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":333,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298498+0000","last_change":"2024-09-13T15:02:52.361444+0000","last_active":"2024-09-13T15:02:59.298498+0000","last_peered":"2024-09-13T15:02:59.298498+0000","last_clean":"2024-09-13T15:02:59.298498+0000","last_became_active":"2024-09-13T14:59:53.220135+0000","last_became_peered":"2024-09-13T14:59:53.220135+0000","last_unstale":"2024-09-13T15:02:59.298498+0000","last_undegraded":"2024-09-13T15:02:59.298498+0000","last_fullsized":"2024-09-13T15:02:59.298498+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.361359+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.361359+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:35.003140+0000","scrub_duration":0.005845154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":83,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301509+0000","last_change":"2024-09-13T14:59:00.936064+0000","last_active":"2024-09-13T15:02:59.301509+0000","last_peered":"2024-09-13T15:02:59.301509+0000","last_clean":"2024-09-13T15:02:59.301509+0000","last_became_active":"2024-09-13T14:59:00.932311+0000","last_became_peered":"2024-09-13T14:59:00.932311+0000","last_unstale":"2024-09-13T15:02:59.301509+0000","last_undegraded":"2024-09-13T15:02:59.301509+0000","last_fullsized":"2024-09-13T15:02:59.301509+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:16.456004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":952,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384298+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T15:03:09.384298+0000","last_peered":"2024-09-13T15:03:09.384298+0000","last_clean":"2024-09-13T15:03:09.384298+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T15:03:09.384298+0000","last_undegraded":"2024-09-13T15:03:09.384298+0000","last_fullsized":"2024-09-13T15:03:09.384298+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":84,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300375+0000","last_change":"2024-09-13T14:59:00.924324+0000","last_active":"2024-09-13T15:02:59.300375+0000","last_peered":"2024-09-13T15:02:59.300375+0000","last_clean":"2024-09-13T15:02:59.300375+0000","last_became_active":"2024-09-13T14:59:00.923798+0000","last_became_peered":"2024-09-13T14:59:00.923798+0000","last_unstale":"2024-09-13T15:02:59.300375+0000","last_undegraded":"2024-09-13T15:02:59.300375+0000","last_fullsized":"2024-09-13T15:02:59.300375+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:10.994877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"362'3755","reported_seq":3770,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:16.531939+0000","last_change":"2024-09-13T15:02:39.922368+0000","last_active":"2024-09-13T15:03:16.531939+0000","last_peered":"2024-09-13T15:03:16.531939+0000","last_clean":"2024-09-13T15:03:16.531939+0000","last_became_active":"2024-09-13T15:02:39.922106+0000","last_became_peered":"2024-09-13T15:02:39.922106+0000","last_unstale":"2024-09-13T15:03:16.531939+0000","last_undegraded":"2024-09-13T15:03:16.531939+0000","last_fullsized":"2024-09-13T15:03:16.531939+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":3755,"log_dups_size":0,"ondisk_log_size":3755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:13:52.416513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246087680,"num_objects":3755,"num_object_clones":0,"num_object_copies":11265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3755,"num_write_kb":240320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":70,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303721+0000","last_change":"2024-09-13T15:02:54.193233+0000","last_active":"2024-09-13T15:02:59.303721+0000","last_peered":"2024-09-13T15:02:59.303721+0000","last_clean":"2024-09-13T15:02:59.303721+0000","last_became_active":"2024-09-13T14:59:00.925167+0000","last_became_peered":"2024-09-13T14:59:00.925167+0000","last_unstale":"2024-09-13T15:02:59.303721+0000","last_undegraded":"2024-09-13T15:02:59.303721+0000","last_fullsized":"2024-09-13T15:02:59.303721+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:54.193156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:54.193156+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:16.851666+0000","scrub_duration":0.0040069900000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":603,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383725+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T15:03:09.383725+0000","last_peered":"2024-09-13T15:03:09.383725+0000","last_clean":"2024-09-13T15:03:09.383725+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T15:03:09.383725+0000","last_undegraded":"2024-09-13T15:03:09.383725+0000","last_fullsized":"2024-09-13T15:03:09.383725+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1175,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303972+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T15:02:59.303972+0000","last_peered":"2024-09-13T15:02:59.303972+0000","last_clean":"2024-09-13T15:02:59.303972+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T15:02:59.303972+0000","last_undegraded":"2024-09-13T15:02:59.303972+0000","last_fullsized":"2024-09-13T15:02:59.303972+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":329,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298065+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T15:02:59.298065+0000","last_peered":"2024-09-13T15:02:59.298065+0000","last_clean":"2024-09-13T15:02:59.298065+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T15:02:59.298065+0000","last_undegraded":"2024-09-13T15:02:59.298065+0000","last_fullsized":"2024-09-13T15:02:59.298065+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"82'100","reported_seq":55,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384160+0000","last_change":"2024-09-13T15:02:51.192587+0000","last_active":"2024-09-13T15:03:09.384160+0000","last_peered":"2024-09-13T15:03:09.384160+0000","last_clean":"2024-09-13T15:03:09.384160+0000","last_became_active":"2024-09-13T15:02:23.878033+0000","last_became_peered":"2024-09-13T15:02:23.878033+0000","last_unstale":"2024-09-13T15:03:09.384160+0000","last_undegraded":"2024-09-13T15:03:09.384160+0000","last_fullsized":"2024-09-13T15:03:09.384160+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:51.192503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:51.192503+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:25:55.606880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":346,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298395+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T15:02:59.298395+0000","last_peered":"2024-09-13T15:02:59.298395+0000","last_clean":"2024-09-13T15:02:59.298395+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T15:02:59.298395+0000","last_undegraded":"2024-09-13T15:02:59.298395+0000","last_fullsized":"2024-09-13T15:02:59.298395+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":610,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383296+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T15:03:09.383296+0000","last_peered":"2024-09-13T15:03:09.383296+0000","last_clean":"2024-09-13T15:03:09.383296+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T15:03:09.383296+0000","last_undegraded":"2024-09-13T15:03:09.383296+0000","last_fullsized":"2024-09-13T15:03:09.383296+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.17","version":"82'100","reported_seq":174,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.108097+0000","last_change":"2024-09-13T15:03:03.108031+0000","last_active":"2024-09-13T15:03:03.108097+0000","last_peered":"2024-09-13T15:03:03.108097+0000","last_clean":"2024-09-13T15:03:03.108097+0000","last_became_active":"2024-09-13T15:02:23.883373+0000","last_became_peered":"2024-09-13T15:02:23.883373+0000","last_unstale":"2024-09-13T15:03:03.108097+0000","last_undegraded":"2024-09-13T15:03:03.108097+0000","last_fullsized":"2024-09-13T15:03:03.108097+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:03.107786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:03.107786+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:17:10.041468+0000","scrub_duration":0.0049024139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":593,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383422+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T15:03:09.383422+0000","last_peered":"2024-09-13T15:03:09.383422+0000","last_clean":"2024-09-13T15:03:09.383422+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T15:03:09.383422+0000","last_undegraded":"2024-09-13T15:03:09.383422+0000","last_fullsized":"2024-09-13T15:03:09.383422+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":352,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300868+0000","last_change":"2024-09-13T14:59:00.929406+0000","last_active":"2024-09-13T15:02:59.300868+0000","last_peered":"2024-09-13T15:02:59.300868+0000","last_clean":"2024-09-13T15:02:59.300868+0000","last_became_active":"2024-09-13T14:59:00.928524+0000","last_became_peered":"2024-09-13T14:59:00.928524+0000","last_unstale":"2024-09-13T15:02:59.300868+0000","last_undegraded":"2024-09-13T15:02:59.300868+0000","last_fullsized":"2024-09-13T15:02:59.300868+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:10.609448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":174,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.387491+0000","last_change":"2024-09-13T14:56:01.235655+0000","last_active":"2024-09-13T15:03:09.387491+0000","last_peered":"2024-09-13T15:03:09.387491+0000","last_clean":"2024-09-13T15:03:09.387491+0000","last_became_active":"2024-09-13T14:55:55.195703+0000","last_became_peered":"2024-09-13T14:55:55.195703+0000","last_unstale":"2024-09-13T15:03:09.387491+0000","last_undegraded":"2024-09-13T15:03:09.387491+0000","last_fullsized":"2024-09-13T15:03:09.387491+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.235497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.235497+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:23:21.640831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":326,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303817+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T15:02:59.303817+0000","last_peered":"2024-09-13T15:02:59.303817+0000","last_clean":"2024-09-13T15:02:59.303817+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T15:02:59.303817+0000","last_undegraded":"2024-09-13T15:02:59.303817+0000","last_fullsized":"2024-09-13T15:02:59.303817+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":333,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300722+0000","last_change":"2024-09-13T14:59:00.926586+0000","last_active":"2024-09-13T15:02:59.300722+0000","last_peered":"2024-09-13T15:02:59.300722+0000","last_clean":"2024-09-13T15:02:59.300722+0000","last_became_active":"2024-09-13T14:59:00.926206+0000","last_became_peered":"2024-09-13T14:59:00.926206+0000","last_unstale":"2024-09-13T15:02:59.300722+0000","last_undegraded":"2024-09-13T15:02:59.300722+0000","last_fullsized":"2024-09-13T15:02:59.300722+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:17:18.805856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.15","version":"82'100","reported_seq":30,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301186+0000","last_change":"2024-09-13T15:02:55.870615+0000","last_active":"2024-09-13T15:02:59.301186+0000","last_peered":"2024-09-13T15:02:59.301186+0000","last_clean":"2024-09-13T15:02:59.301186+0000","last_became_active":"2024-09-13T15:02:23.883217+0000","last_became_peered":"2024-09-13T15:02:23.883217+0000","last_unstale":"2024-09-13T15:02:59.301186+0000","last_undegraded":"2024-09-13T15:02:59.301186+0000","last_fullsized":"2024-09-13T15:02:59.301186+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.870528+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.870528+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:10.023787+0000","scrub_duration":0.0047551850000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":597,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383095+0000","last_change":"2024-09-13T14:59:00.932503+0000","last_active":"2024-09-13T15:03:09.383095+0000","last_peered":"2024-09-13T15:03:09.383095+0000","last_clean":"2024-09-13T15:03:09.383095+0000","last_became_active":"2024-09-13T14:59:00.931358+0000","last_became_peered":"2024-09-13T14:59:00.931358+0000","last_unstale":"2024-09-13T15:03:09.383095+0000","last_undegraded":"2024-09-13T15:03:09.383095+0000","last_fullsized":"2024-09-13T15:03:09.383095+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:19.226182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":429,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383175+0000","last_change":"2024-09-13T14:59:00.927060+0000","last_active":"2024-09-13T15:03:09.383175+0000","last_peered":"2024-09-13T15:03:09.383175+0000","last_clean":"2024-09-13T15:03:09.383175+0000","last_became_active":"2024-09-13T14:59:00.925402+0000","last_became_peered":"2024-09-13T14:59:00.925402+0000","last_unstale":"2024-09-13T15:03:09.383175+0000","last_undegraded":"2024-09-13T15:03:09.383175+0000","last_fullsized":"2024-09-13T15:03:09.383175+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:31:08.336446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"82'100","reported_seq":30,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.299016+0000","last_change":"2024-09-13T15:02:56.319389+0000","last_active":"2024-09-13T15:02:59.299016+0000","last_peered":"2024-09-13T15:02:59.299016+0000","last_clean":"2024-09-13T15:02:59.299016+0000","last_became_active":"2024-09-13T15:02:23.876739+0000","last_became_peered":"2024-09-13T15:02:23.876739+0000","last_unstale":"2024-09-13T15:02:59.299016+0000","last_undegraded":"2024-09-13T15:02:59.299016+0000","last_fullsized":"2024-09-13T15:02:59.299016+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:56.319302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:56.319302+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:58.230158+0000","scrub_duration":0.0031182190000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":146,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303800+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T15:02:59.303800+0000","last_peered":"2024-09-13T15:02:59.303800+0000","last_clean":"2024-09-13T15:02:59.303800+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T15:02:59.303800+0000","last_undegraded":"2024-09-13T15:02:59.303800+0000","last_fullsized":"2024-09-13T15:02:59.303800+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":187,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.382484+0000","last_change":"2024-09-13T14:59:00.924196+0000","last_active":"2024-09-13T15:03:09.382484+0000","last_peered":"2024-09-13T15:03:09.382484+0000","last_clean":"2024-09-13T15:03:09.382484+0000","last_became_active":"2024-09-13T14:59:00.923901+0000","last_became_peered":"2024-09-13T14:59:00.923901+0000","last_unstale":"2024-09-13T15:03:09.382484+0000","last_undegraded":"2024-09-13T15:03:09.382484+0000","last_fullsized":"2024-09-13T15:03:09.382484+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:01.913804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"82'100","reported_seq":612,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:07.166878+0000","last_change":"2024-09-13T15:03:07.166808+0000","last_active":"2024-09-13T15:03:07.166878+0000","last_peered":"2024-09-13T15:03:07.166878+0000","last_clean":"2024-09-13T15:03:07.166878+0000","last_became_active":"2024-09-13T15:02:23.883387+0000","last_became_peered":"2024-09-13T15:02:23.883387+0000","last_unstale":"2024-09-13T15:03:07.166878+0000","last_undegraded":"2024-09-13T15:03:07.166878+0000","last_fullsized":"2024-09-13T15:03:07.166878+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:07.166034+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:07.166034+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:32:42.605595+0000","scrub_duration":0.01094734,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":621,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384165+0000","last_change":"2024-09-13T14:59:00.924814+0000","last_active":"2024-09-13T15:03:09.384165+0000","last_peered":"2024-09-13T15:03:09.384165+0000","last_clean":"2024-09-13T15:03:09.384165+0000","last_became_active":"2024-09-13T14:59:00.924501+0000","last_became_peered":"2024-09-13T14:59:00.924501+0000","last_unstale":"2024-09-13T15:03:09.384165+0000","last_undegraded":"2024-09-13T15:03:09.384165+0000","last_fullsized":"2024-09-13T15:03:09.384165+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:11:15.382207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":585,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383176+0000","last_change":"2024-09-13T14:59:00.925431+0000","last_active":"2024-09-13T15:03:09.383176+0000","last_peered":"2024-09-13T15:03:09.383176+0000","last_clean":"2024-09-13T15:03:09.383176+0000","last_became_active":"2024-09-13T14:59:00.925023+0000","last_became_peered":"2024-09-13T14:59:00.925023+0000","last_unstale":"2024-09-13T15:03:09.383176+0000","last_undegraded":"2024-09-13T15:03:09.383176+0000","last_fullsized":"2024-09-13T15:03:09.383176+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:38:18.686253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"82'100","reported_seq":317,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:04.101991+0000","last_change":"2024-09-13T15:03:04.101898+0000","last_active":"2024-09-13T15:03:04.101991+0000","last_peered":"2024-09-13T15:03:04.101991+0000","last_clean":"2024-09-13T15:03:04.101991+0000","last_became_active":"2024-09-13T15:02:23.884406+0000","last_became_peered":"2024-09-13T15:02:23.884406+0000","last_unstale":"2024-09-13T15:03:04.101991+0000","last_undegraded":"2024-09-13T15:03:04.101991+0000","last_fullsized":"2024-09-13T15:03:04.101991+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:04.101802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:04.101802+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:34.459623+0000","scrub_duration":0.003875819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":136,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298649+0000","last_change":"2024-09-13T14:59:00.927250+0000","last_active":"2024-09-13T15:02:59.298649+0000","last_peered":"2024-09-13T15:02:59.298649+0000","last_clean":"2024-09-13T15:02:59.298649+0000","last_became_active":"2024-09-13T14:59:00.926953+0000","last_became_peered":"2024-09-13T14:59:00.926953+0000","last_unstale":"2024-09-13T15:02:59.298649+0000","last_undegraded":"2024-09-13T15:02:59.298649+0000","last_fullsized":"2024-09-13T15:02:59.298649+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:58.306547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":345,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.299046+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T15:02:59.299046+0000","last_peered":"2024-09-13T15:02:59.299046+0000","last_clean":"2024-09-13T15:02:59.299046+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T15:02:59.299046+0000","last_undegraded":"2024-09-13T15:02:59.299046+0000","last_fullsized":"2024-09-13T15:02:59.299046+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"82'100","reported_seq":55,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383145+0000","last_change":"2024-09-13T15:02:52.429623+0000","last_active":"2024-09-13T15:03:09.383145+0000","last_peered":"2024-09-13T15:03:09.383145+0000","last_clean":"2024-09-13T15:03:09.383145+0000","last_became_active":"2024-09-13T15:02:23.880201+0000","last_became_peered":"2024-09-13T15:02:23.880201+0000","last_unstale":"2024-09-13T15:03:09.383145+0000","last_undegraded":"2024-09-13T15:03:09.383145+0000","last_fullsized":"2024-09-13T15:03:09.383145+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.429543+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.429543+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:43.145050+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":419,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384155+0000","last_change":"2024-09-13T14:59:00.934412+0000","last_active":"2024-09-13T15:03:09.384155+0000","last_peered":"2024-09-13T15:03:09.384155+0000","last_clean":"2024-09-13T15:03:09.384155+0000","last_became_active":"2024-09-13T14:59:00.932986+0000","last_became_peered":"2024-09-13T14:59:00.932986+0000","last_unstale":"2024-09-13T15:03:09.384155+0000","last_undegraded":"2024-09-13T15:03:09.384155+0000","last_fullsized":"2024-09-13T15:03:09.384155+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:21:44.285324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":582,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.382827+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T15:03:09.382827+0000","last_peered":"2024-09-13T15:03:09.382827+0000","last_clean":"2024-09-13T15:03:09.382827+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T15:03:09.382827+0000","last_undegraded":"2024-09-13T15:03:09.382827+0000","last_fullsized":"2024-09-13T15:03:09.382827+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"82'100","reported_seq":30,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300548+0000","last_change":"2024-09-13T15:02:55.679993+0000","last_active":"2024-09-13T15:02:59.300548+0000","last_peered":"2024-09-13T15:02:59.300548+0000","last_clean":"2024-09-13T15:02:59.300548+0000","last_became_active":"2024-09-13T15:02:23.878770+0000","last_became_peered":"2024-09-13T15:02:23.878770+0000","last_unstale":"2024-09-13T15:02:59.300548+0000","last_undegraded":"2024-09-13T15:02:59.300548+0000","last_fullsized":"2024-09-13T15:02:59.300548+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.679893+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.679893+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:25.717099+0000","scrub_duration":0.0050514899999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":353,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298332+0000","last_change":"2024-09-13T14:59:00.929751+0000","last_active":"2024-09-13T15:02:59.298332+0000","last_peered":"2024-09-13T15:02:59.298332+0000","last_clean":"2024-09-13T15:02:59.298332+0000","last_became_active":"2024-09-13T14:59:00.929456+0000","last_became_peered":"2024-09-13T14:59:00.929456+0000","last_unstale":"2024-09-13T15:02:59.298332+0000","last_undegraded":"2024-09-13T15:02:59.298332+0000","last_fullsized":"2024-09-13T15:02:59.298332+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:11.462853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":328,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303808+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T15:02:59.303808+0000","last_peered":"2024-09-13T15:02:59.303808+0000","last_clean":"2024-09-13T15:02:59.303808+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T15:02:59.303808+0000","last_undegraded":"2024-09-13T15:02:59.303808+0000","last_fullsized":"2024-09-13T15:02:59.303808+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":353,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298933+0000","last_change":"2024-09-13T14:59:00.928898+0000","last_active":"2024-09-13T15:02:59.298933+0000","last_peered":"2024-09-13T15:02:59.298933+0000","last_clean":"2024-09-13T15:02:59.298933+0000","last_became_active":"2024-09-13T14:59:00.928585+0000","last_became_peered":"2024-09-13T14:59:00.928585+0000","last_unstale":"2024-09-13T15:02:59.298933+0000","last_undegraded":"2024-09-13T15:02:59.298933+0000","last_fullsized":"2024-09-13T15:02:59.298933+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:26:57.155149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":367,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303370+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T15:02:59.303370+0000","last_peered":"2024-09-13T15:02:59.303370+0000","last_clean":"2024-09-13T15:02:59.303370+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T15:02:59.303370+0000","last_undegraded":"2024-09-13T15:02:59.303370+0000","last_fullsized":"2024-09-13T15:02:59.303370+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"82'100","reported_seq":20,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300672+0000","last_change":"2024-09-13T15:02:23.879862+0000","last_active":"2024-09-13T15:02:59.300672+0000","last_peered":"2024-09-13T15:02:59.300672+0000","last_clean":"2024-09-13T15:02:59.300672+0000","last_became_active":"2024-09-13T15:02:23.879170+0000","last_became_peered":"2024-09-13T15:02:23.879170+0000","last_unstale":"2024-09-13T15:02:59.300672+0000","last_undegraded":"2024-09-13T15:02:59.300672+0000","last_fullsized":"2024-09-13T15:02:59.300672+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:06.945525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:06.945525+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:48.496118+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":153,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384043+0000","last_change":"2024-09-13T14:59:00.929835+0000","last_active":"2024-09-13T15:03:09.384043+0000","last_peered":"2024-09-13T15:03:09.384043+0000","last_clean":"2024-09-13T15:03:09.384043+0000","last_became_active":"2024-09-13T14:59:00.929514+0000","last_became_peered":"2024-09-13T14:59:00.929514+0000","last_unstale":"2024-09-13T15:03:09.384043+0000","last_undegraded":"2024-09-13T15:03:09.384043+0000","last_fullsized":"2024-09-13T15:03:09.384043+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:48.981240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":389,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300857+0000","last_change":"2024-09-13T14:59:00.925895+0000","last_active":"2024-09-13T15:02:59.300857+0000","last_peered":"2024-09-13T15:02:59.300857+0000","last_clean":"2024-09-13T15:02:59.300857+0000","last_became_active":"2024-09-13T14:59:00.925564+0000","last_became_peered":"2024-09-13T14:59:00.925564+0000","last_unstale":"2024-09-13T15:02:59.300857+0000","last_undegraded":"2024-09-13T15:02:59.300857+0000","last_fullsized":"2024-09-13T15:02:59.300857+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:23.283927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1d","version":"82'100","reported_seq":57,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383654+0000","last_change":"2024-09-13T15:02:52.149390+0000","last_active":"2024-09-13T15:03:09.383654+0000","last_peered":"2024-09-13T15:03:09.383654+0000","last_clean":"2024-09-13T15:03:09.383654+0000","last_became_active":"2024-09-13T15:02:23.914809+0000","last_became_peered":"2024-09-13T15:02:23.914809+0000","last_unstale":"2024-09-13T15:03:09.383654+0000","last_undegraded":"2024-09-13T15:03:09.383654+0000","last_fullsized":"2024-09-13T15:03:09.383654+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.149290+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.149290+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:59.584474+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":349,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.297888+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T15:02:59.297888+0000","last_peered":"2024-09-13T15:02:59.297888+0000","last_clean":"2024-09-13T15:02:59.297888+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T15:02:59.297888+0000","last_undegraded":"2024-09-13T15:02:59.297888+0000","last_fullsized":"2024-09-13T15:02:59.297888+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":635,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.382997+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T15:03:09.382997+0000","last_peered":"2024-09-13T15:03:09.382997+0000","last_clean":"2024-09-13T15:03:09.382997+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T15:03:09.382997+0000","last_undegraded":"2024-09-13T15:03:09.382997+0000","last_fullsized":"2024-09-13T15:03:09.382997+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3878223872,"num_objects":59177,"num_object_clones":0,"num_object_copies":177531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":59177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":59177,"num_write_kb":3787328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11497504768,"data_stored":11497504768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":59177,"ondisk_log_size":59177,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":16172,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397952,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2071176,"kb_used_data":1185920,"kb_used_omap":1,"kb_used_meta":885246,"kb_avail":91673976,"statfs":{"total":95995035648,"available":93874151424,"internally_reserved":0,"allocated":1214382080,"data_stored":1211875764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":906492248},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594181,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1965372,"kb_used_data":1663860,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91779780,"statfs":{"total":95995035648,"available":93982494720,"internally_reserved":0,"allocated":1703792640,"data_stored":1701297640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790410,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2099832,"kb_used_data":1198640,"kb_used_omap":1,"kb_used_meta":901182,"kb_avail":91645320,"statfs":{"total":95995035648,"available":93844807680,"internally_reserved":0,"allocated":1227407360,"data_stored":1224912897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":922810712},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986639,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3293176,"kb_used_data":2395952,"kb_used_omap":1,"kb_used_meta":897214,"kb_avail":90451976,"statfs":{"total":95995035648,"available":92622823424,"internally_reserved":0,"allocated":2453454848,"data_stored":2450959572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":918747480},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,6],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":0,"seq":1438814044186,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762613,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2448952,"kb_used_data":1545968,"kb_used_omap":8,"kb_used_meta":902967,"kb_avail":91296200,"statfs":{"total":95995035648,"available":93487308800,"internally_reserved":0,"allocated":1583071232,"data_stored":1580576764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":924638666},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926139,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1763056,"kb_used_data":870440,"kb_used_omap":7,"kb_used_meta":892600,"kb_avail":91982096,"statfs":{"total":95995035648,"available":94189666304,"internally_reserved":0,"allocated":891330560,"data_stored":888843377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":914023113},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122379,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3041412,"kb_used_data":2205180,"kb_used_omap":7,"kb_used_meta":836216,"kb_avail":90703740,"statfs":{"total":95995035648,"available":92880629760,"internally_reserved":0,"allocated":2258104320,"data_stored":2255602064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":856285775},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2314731520,"data_stored":2314731520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":905248768,"data_stored":905248768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1625620480,"data_stored":1625620480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2463301632,"data_stored":2463301632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1230831616,"data_stored":1230831616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1726808064,"data_stored":1726808064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1230962688,"data_stored":1230962688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:03:22.794 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T15:03:22.794 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:03:22.795 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T15:03:22.795 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:03:22.813 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:22.848 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:22.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:22.950 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:22.987 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:22.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:23.088 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:23.123 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:23.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:23.162 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 33 16 69721 69705 131.981 77.5 0.00266402 0.00756733 2024-09-13T15:03:23.226 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:23.247 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:23.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:23.350 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:23.381 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:23.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:23.483 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:23.521 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:23.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:23.623 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:23.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.663 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:23.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:23.764 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:23.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.800 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:23.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:23.900 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:23.939 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:23.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:24.041 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:24.071 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:24.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:24.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:23 smithi096 ceph-mon[98901]: pgmap v1206: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-13T15:03:24.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:23 smithi096 ceph-mon[102928]: pgmap v1206: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-13T15:03:24.162 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 34 16 71000 70984 130.45 79.9375 0.0108742 0.00766151 2024-09-13T15:03:24.173 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:24.203 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:24.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:24.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:23 smithi204 ceph-mon[87071]: pgmap v1206: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-13T15:03:24.304 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:24.333 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:24.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:24.435 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:24.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.466 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:24.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:24.567 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:24.606 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:24.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:24.708 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:24.740 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:24.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:24.785 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T15:03:24.785 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T15:03:24.785 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T15:03:24.785 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T15:03:24.786 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T15:03:24.843 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:24.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.879 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:24.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:24.979 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:25.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.024 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:25.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:25.125 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:25.160 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:25.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:25.162 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 35 16 72354 72338 129.141 84.625 0.0106137 0.00773805 2024-09-13T15:03:25.262 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:25.297 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:25.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:25.398 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:25.430 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:25.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:25.532 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:25.564 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:25.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:25.666 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:25.699 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:25.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:25.800 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:25.832 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:25.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:25.934 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:25.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.973 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:25.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:26.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:25 smithi096 ceph-mon[98901]: pgmap v1207: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T15:03:26.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:25 smithi096 ceph-mon[102928]: pgmap v1207: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T15:03:26.074 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:26.108 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:26.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:26.110 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:03:26.111 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:03:26.140 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:03:26.140 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:03:26.163 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 36 16 73699 73683 127.889 84.0625 0.0128192 0.00781459 2024-09-13T15:03:26.182 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:03:26.182 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:03:26.210 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:26.214 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:03:26.214 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:03:26.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:25 smithi204 ceph-mon[87071]: pgmap v1207: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T15:03:26.242 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:03:26.243 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:03:26.246 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:26.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:26.273 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:03:26.273 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:03:26.310 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:03:26.311 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:03:26.343 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:03:26.343 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:03:26.347 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:26.387 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:26.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:26.489 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:26.528 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:26.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:26.630 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:26.666 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:26.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:26.768 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:26.786 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T15:03:26.807 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:26.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:26.909 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:26.941 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:26.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:27.043 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:27.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:26 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3849145995' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:27.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:26 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3849145995' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:27.077 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:27.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:27.163 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 37 16 75063 75047 126.736 85.25 0.00762164 0.00788609 2024-09-13T15:03:27.180 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:27.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:26 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3849145995' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:27.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.222 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:27.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:27.323 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:27.350 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:27.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:27.452 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:27.488 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:27.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:27.589 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:27.623 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:27.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:27.725 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:27.748 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:27.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:27.850 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:27.889 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:27.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:27.991 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:28.022 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:28.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:28.125 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:28.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.162 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:28.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:28.162 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 38 16 76392 76376 125.587 83.0625 0.00878259 0.00795828 2024-09-13T15:03:28.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:27 smithi204 ceph-mon[87071]: pgmap v1208: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-13T15:03:28.263 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:28.294 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:28.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:28.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:27 smithi096 ceph-mon[98901]: pgmap v1208: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-13T15:03:28.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:03:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:03:27] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.51.0" 2024-09-13T15:03:28.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:27 smithi096 ceph-mon[102928]: pgmap v1208: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-13T15:03:28.397 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:28.431 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:28.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:28.533 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:28.563 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:28.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:28.665 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:28.698 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:28.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:28.800 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:28.837 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:28.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:28.939 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:28.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.981 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:28.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:29.082 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:29.115 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:29.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:29.161 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:29.163 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 39 16 77722 77706 124.498 83.125 0.00546624 0.00802744 2024-09-13T15:03:29.217 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:29.248 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:29.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:29.350 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:29.392 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:29.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:29.494 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:29.534 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:29.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:29.636 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:29.663 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:29.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:29.765 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:29.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.803 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:29.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:29.904 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:29.941 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:29.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:29.976 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:29 smithi096 ceph-mon[98901]: pgmap v1209: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-09-13T15:03:29.976 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:29 smithi096 ceph-mon[102928]: pgmap v1209: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-09-13T15:03:30.003 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:30.043 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:30.086 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:30.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:30.164 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T15:03:30.163561+0000 min lat: 0.00132706 max lat: 0.137651 avg lat: 0.00809847 2024-09-13T15:03:30.164 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:03:30.164 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 40 16 79019 79003 123.41 81.0625 0.00611899 0.00809847 2024-09-13T15:03:30.187 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:30.213 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:30.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:30.215 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:29 smithi204 ceph-mon[87071]: pgmap v1209: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-09-13T15:03:30.315 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:30.354 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:30.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:30.456 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:03:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:03:30.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:03:30.485 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:30.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:30.587 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:30.616 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:30.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:30.718 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:30.746 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:30.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:30.848 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:30.886 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:30.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:30.988 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:31.021 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:31.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:31.026 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:03:31.123 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:31.155 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:31.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:31.164 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 41 16 80322 80306 122.386 81.4375 0.00742579 0.00816571 2024-09-13T15:03:31.257 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:31.298 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:31.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:31.399 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:31.436 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:31.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:31.538 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:31.570 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:31.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:31.672 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:31.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.707 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:31.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:31.808 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:31.839 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:31.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:31.903 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T15:03:31.941 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:31.971 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:31.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:32.073 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:32.107 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:32.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:32.164 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 42 16 81570 81554 121.329 78 0.00952375 0.00823646 2024-09-13T15:03:32.209 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:32.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:31 smithi204 ceph-mon[87071]: pgmap v1210: 119 pgs: 119 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T15:03:32.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:31 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3489335219' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:03:32.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:31 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:03:32.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.249 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:32.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:32.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:31 smithi096 ceph-mon[98901]: pgmap v1210: 119 pgs: 119 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T15:03:32.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:31 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3489335219' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:03:32.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:31 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:03:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:31 smithi096 ceph-mon[102928]: pgmap v1210: 119 pgs: 119 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T15:03:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:31 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3489335219' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:03:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:31 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:03:32.352 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:32.383 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:32.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:32.485 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:32.506 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:32.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:32.607 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:32.638 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:32.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:32.740 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:32.779 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:32.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:32.881 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:32.925 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:32.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:33.026 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:33.065 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:33.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:33.164 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 43 16 82652 82636 120.08 67.625 0.0184503 0.00832327 2024-09-13T15:03:33.167 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:33.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.204 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:33.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:33.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:32 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:33.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:32 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:03:33.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:32 smithi204 ceph-mon[87071]: osdmap e363: 8 total, 8 up, 7 in 2024-09-13T15:03:33.306 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:33.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:32 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:33.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:32 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:03:33.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:32 smithi096 ceph-mon[98901]: osdmap e363: 8 total, 8 up, 7 in 2024-09-13T15:03:33.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:32 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:33.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:32 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:03:33.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:32 smithi096 ceph-mon[102928]: osdmap e363: 8 total, 8 up, 7 in 2024-09-13T15:03:33.349 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:33.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:33.451 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:33.485 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:33.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:33.587 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:33.622 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:33.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:33.724 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:33.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.762 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:33.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:33.863 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:33.900 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:33.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:34.002 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:34.046 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:34.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:34.148 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:34.152 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:33 smithi096 ceph-mon[98901]: pgmap v1212: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T15:03:34.152 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:33 smithi096 ceph-mon[102928]: pgmap v1212: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T15:03:34.164 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 44 16 83905 83889 119.13 78.3125 0.011983 0.00838964 2024-09-13T15:03:34.193 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:34.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:34.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:33 smithi204 ceph-mon[87071]: pgmap v1212: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T15:03:34.294 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:34.324 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:34.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:34.426 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:34.460 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:34.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:34.562 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:34.594 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:34.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:34.696 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:34.733 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:34.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:34.783 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:34.834 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:34.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.885 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:34.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:34.986 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:35.015 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:34 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:03:35.015 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:34 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:03:35.028 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:35.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:35.129 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T15:03:35.131 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T15:03:35.131 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:35.164 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 45 16 85052 85036 118.076 71.6875 0.0165784 0.00846414 2024-09-13T15:03:35.177 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:35.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:35.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:34 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:03:35.280 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:35.338 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:35.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:35.439 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:35.477 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:35.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:35.579 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:35.615 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:35.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:35.717 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:35.748 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:35.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:35.850 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:35.883 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:35.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:35.984 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:35.986 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:03:35.986 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:03:36.019 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:36.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:36.121 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:36.153 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:36.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:36.164 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 46 15 86182 86167 117.046 70.6875 0.0168994 0.00853905 2024-09-13T15:03:36.196 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:03:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:35 smithi204 ceph-mon[87071]: pgmap v1213: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T15:03:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:03:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:36.224 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:35 smithi096 ceph-mon[98901]: pgmap v1213: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T15:03:36.224 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:03:36.225 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:36.225 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:35 smithi096 ceph-mon[102928]: pgmap v1213: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T15:03:36.225 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:03:36.225 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:36.255 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:36.303 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:36.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:36.405 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:36.436 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:36.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:36.538 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:36.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.575 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:36.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:36.666 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T15:03:36.675 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:36.711 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:36.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:36.813 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:36.855 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:36.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:36.934 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:36 smithi204 ceph-mon[87071]: from='client.55909 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:03:36.957 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:36.983 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:37.085 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:37.112 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:37.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:37.164 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 47 16 87313 87297 116.058 70.625 0.0117907 0.0086116 2024-09-13T15:03:37.214 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:37.262 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:37.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:37.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:36 smithi096 ceph-mon[98901]: from='client.55909 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:03:37.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:36 smithi096 ceph-mon[102928]: from='client.55909 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:03:37.364 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:37.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.405 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:37.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:37.406 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:37.411 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":1213,"stamp":"2024-09-13T15:03:34.181946+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5281159117,"num_objects":80539,"num_object_clones":0,"num_object_copies":241617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80539,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":81418,"num_write_kb":5165327,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":109,"num_bytes_recovered":8404818,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84643,"ondisk_log_size":84643,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":20904048,"kb_used_data":15261496,"kb_used_omap":29,"kb_used_meta":5642466,"kb_avail":635312016,"statfs":{"total":671965249536,"available":650559504384,"internally_reserved":0,"allocated":15627771904,"data_stored":15610289153,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30654,"internal_metadata":5777885250},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,0,5,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":871694336,"num_objects":13301,"num_object_clones":0,"num_object_copies":39903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13301,"num_write_kb":851264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002604"},"pg_stats":[{"pgid":"1.11","version":"82'100","reported_seq":327,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383953+0000","last_change":"2024-09-13T14:59:00.929470+0000","last_active":"2024-09-13T15:03:09.383953+0000","last_peered":"2024-09-13T15:03:09.383953+0000","last_clean":"2024-09-13T15:03:09.383953+0000","last_became_active":"2024-09-13T14:59:00.928726+0000","last_became_peered":"2024-09-13T14:59:00.928726+0000","last_unstale":"2024-09-13T15:03:09.383953+0000","last_undegraded":"2024-09-13T15:03:09.383953+0000","last_fullsized":"2024-09-13T15:03:09.383953+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.087994+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.087994+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:41.304122+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":171,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.909371+0000","last_change":"2024-09-13T14:59:00.935629+0000","last_active":"2024-09-13T15:03:31.909371+0000","last_peered":"2024-09-13T15:03:31.909371+0000","last_clean":"2024-09-13T15:03:31.909371+0000","last_became_active":"2024-09-13T14:59:00.935106+0000","last_became_peered":"2024-09-13T14:59:00.935106+0000","last_unstale":"2024-09-13T15:03:31.909371+0000","last_undegraded":"2024-09-13T15:03:31.909371+0000","last_fullsized":"2024-09-13T15:03:31.909371+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:12.481049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":328,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301561+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T15:02:59.301561+0000","last_peered":"2024-09-13T15:02:59.301561+0000","last_clean":"2024-09-13T15:02:59.301561+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T15:02:59.301561+0000","last_undegraded":"2024-09-13T15:02:59.301561+0000","last_fullsized":"2024-09-13T15:02:59.301561+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":92,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303200+0000","last_change":"2024-09-13T14:56:04.643672+0000","last_active":"2024-09-13T15:02:59.303200+0000","last_peered":"2024-09-13T15:02:59.303200+0000","last_clean":"2024-09-13T15:02:59.303200+0000","last_became_active":"2024-09-13T14:55:55.185995+0000","last_became_peered":"2024-09-13T14:55:55.185995+0000","last_unstale":"2024-09-13T15:02:59.303200+0000","last_undegraded":"2024-09-13T15:02:59.303200+0000","last_fullsized":"2024-09-13T15:02:59.303200+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.643614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.643614+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:36.495546+0000","scrub_duration":0.001441681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":335,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303291+0000","last_change":"2024-09-13T14:59:00.926231+0000","last_active":"2024-09-13T15:02:59.303291+0000","last_peered":"2024-09-13T15:02:59.303291+0000","last_clean":"2024-09-13T15:02:59.303291+0000","last_became_active":"2024-09-13T14:59:00.925808+0000","last_became_peered":"2024-09-13T14:59:00.925808+0000","last_unstale":"2024-09-13T15:02:59.303291+0000","last_undegraded":"2024-09-13T15:02:59.303291+0000","last_fullsized":"2024-09-13T15:02:59.303291+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:39.672143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":618,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384469+0000","last_change":"2024-09-13T14:59:00.923848+0000","last_active":"2024-09-13T15:03:09.384469+0000","last_peered":"2024-09-13T15:03:09.384469+0000","last_clean":"2024-09-13T15:03:09.384469+0000","last_became_active":"2024-09-13T14:59:00.923522+0000","last_became_peered":"2024-09-13T14:59:00.923522+0000","last_unstale":"2024-09-13T15:03:09.384469+0000","last_undegraded":"2024-09-13T15:03:09.384469+0000","last_fullsized":"2024-09-13T15:03:09.384469+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:35:35.267527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":760,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.909096+0000","last_change":"2024-09-13T14:59:00.930294+0000","last_active":"2024-09-13T15:03:31.909096+0000","last_peered":"2024-09-13T15:03:31.909096+0000","last_clean":"2024-09-13T15:03:31.909096+0000","last_became_active":"2024-09-13T14:59:00.929860+0000","last_became_peered":"2024-09-13T14:59:00.929860+0000","last_unstale":"2024-09-13T15:03:31.909096+0000","last_undegraded":"2024-09-13T15:03:31.909096+0000","last_fullsized":"2024-09-13T15:03:31.909096+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.240882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.240882+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:48:18.212524+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":612,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384229+0000","last_change":"2024-09-13T14:59:00.925448+0000","last_active":"2024-09-13T15:03:09.384229+0000","last_peered":"2024-09-13T15:03:09.384229+0000","last_clean":"2024-09-13T15:03:09.384229+0000","last_became_active":"2024-09-13T14:59:00.925152+0000","last_became_peered":"2024-09-13T14:59:00.925152+0000","last_unstale":"2024-09-13T15:03:09.384229+0000","last_undegraded":"2024-09-13T15:03:09.384229+0000","last_fullsized":"2024-09-13T15:03:09.384229+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:06.703642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":329,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301262+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T15:02:59.301262+0000","last_peered":"2024-09-13T15:02:59.301262+0000","last_clean":"2024-09-13T15:02:59.301262+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T15:02:59.301262+0000","last_undegraded":"2024-09-13T15:02:59.301262+0000","last_fullsized":"2024-09-13T15:02:59.301262+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":92,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303664+0000","last_change":"2024-09-13T14:56:03.605447+0000","last_active":"2024-09-13T15:02:59.303664+0000","last_peered":"2024-09-13T15:02:59.303664+0000","last_clean":"2024-09-13T15:02:59.303664+0000","last_became_active":"2024-09-13T14:55:55.218939+0000","last_became_peered":"2024-09-13T14:55:55.218939+0000","last_unstale":"2024-09-13T15:02:59.303664+0000","last_undegraded":"2024-09-13T15:02:59.303664+0000","last_fullsized":"2024-09-13T15:02:59.303664+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.605356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.605356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:12.467371+0000","scrub_duration":0.0020269620000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":84,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301366+0000","last_change":"2024-09-13T14:59:00.925287+0000","last_active":"2024-09-13T15:02:59.301366+0000","last_peered":"2024-09-13T15:02:59.301366+0000","last_clean":"2024-09-13T15:02:59.301366+0000","last_became_active":"2024-09-13T14:59:00.923868+0000","last_became_peered":"2024-09-13T14:59:00.923868+0000","last_unstale":"2024-09-13T15:02:59.301366+0000","last_undegraded":"2024-09-13T15:02:59.301366+0000","last_fullsized":"2024-09-13T15:02:59.301366+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:28.106847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":156,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301057+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T15:02:59.301057+0000","last_peered":"2024-09-13T15:02:59.301057+0000","last_clean":"2024-09-13T15:02:59.301057+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T15:02:59.301057+0000","last_undegraded":"2024-09-13T15:02:59.301057+0000","last_fullsized":"2024-09-13T15:02:59.301057+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"362'5058","reported_seq":5073,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.561550+0000","last_change":"2024-09-13T15:02:39.884666+0000","last_active":"2024-09-13T15:03:31.561550+0000","last_peered":"2024-09-13T15:03:31.561550+0000","last_clean":"2024-09-13T15:03:31.561550+0000","last_became_active":"2024-09-13T15:02:39.884349+0000","last_became_peered":"2024-09-13T15:02:39.884349+0000","last_unstale":"2024-09-13T15:03:31.561550+0000","last_undegraded":"2024-09-13T15:03:31.561550+0000","last_fullsized":"2024-09-13T15:03:31.561550+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5058,"log_dups_size":0,"ondisk_log_size":5058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331481088,"num_objects":5058,"num_object_clones":0,"num_object_copies":15174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5058,"num_write_kb":323712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":112,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300732+0000","last_change":"2024-09-13T15:02:53.820519+0000","last_active":"2024-09-13T15:02:59.300732+0000","last_peered":"2024-09-13T15:02:59.300732+0000","last_clean":"2024-09-13T15:02:59.300732+0000","last_became_active":"2024-09-13T14:59:53.222306+0000","last_became_peered":"2024-09-13T14:59:53.222306+0000","last_unstale":"2024-09-13T15:02:59.300732+0000","last_undegraded":"2024-09-13T15:02:59.300732+0000","last_fullsized":"2024-09-13T15:02:59.300732+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:53.820435+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:53.820435+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:40.108691+0000","scrub_duration":0.0030873739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":582,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.388028+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T15:03:09.388028+0000","last_peered":"2024-09-13T15:03:09.388028+0000","last_clean":"2024-09-13T15:03:09.388028+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T15:03:09.388028+0000","last_undegraded":"2024-09-13T15:03:09.388028+0000","last_fullsized":"2024-09-13T15:03:09.388028+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":348,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303568+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T15:02:59.303568+0000","last_peered":"2024-09-13T15:02:59.303568+0000","last_clean":"2024-09-13T15:02:59.303568+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T15:02:59.303568+0000","last_undegraded":"2024-09-13T15:02:59.303568+0000","last_fullsized":"2024-09-13T15:02:59.303568+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"362'5046","reported_seq":5061,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.487039+0000","last_change":"2024-09-13T15:02:39.921810+0000","last_active":"2024-09-13T15:03:31.487039+0000","last_peered":"2024-09-13T15:03:31.487039+0000","last_clean":"2024-09-13T15:03:31.487039+0000","last_became_active":"2024-09-13T15:02:39.921510+0000","last_became_peered":"2024-09-13T15:02:39.921510+0000","last_unstale":"2024-09-13T15:03:31.487039+0000","last_undegraded":"2024-09-13T15:03:31.487039+0000","last_fullsized":"2024-09-13T15:03:31.487039+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5046,"log_dups_size":0,"ondisk_log_size":5046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:53:53.564891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330694656,"num_objects":5046,"num_object_clones":0,"num_object_copies":15138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5046,"num_write_kb":322944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":541,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383205+0000","last_change":"2024-09-13T15:00:10.016332+0000","last_active":"2024-09-13T15:03:09.383205+0000","last_peered":"2024-09-13T15:03:09.383205+0000","last_clean":"2024-09-13T15:03:09.383205+0000","last_became_active":"2024-09-13T14:59:53.227343+0000","last_became_peered":"2024-09-13T14:59:53.227343+0000","last_unstale":"2024-09-13T15:03:09.383205+0000","last_undegraded":"2024-09-13T15:03:09.383205+0000","last_fullsized":"2024-09-13T15:03:09.383205+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.016260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.016260+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:26:31.269839+0000","scrub_duration":71,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":328,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301232+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T15:02:59.301232+0000","last_peered":"2024-09-13T15:02:59.301232+0000","last_clean":"2024-09-13T15:02:59.301232+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T15:02:59.301232+0000","last_undegraded":"2024-09-13T15:02:59.301232+0000","last_fullsized":"2024-09-13T15:02:59.301232+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":393,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298560+0000","last_change":"2024-09-13T14:59:00.929566+0000","last_active":"2024-09-13T15:02:59.298560+0000","last_peered":"2024-09-13T15:02:59.298560+0000","last_clean":"2024-09-13T15:02:59.298560+0000","last_became_active":"2024-09-13T14:59:00.929271+0000","last_became_peered":"2024-09-13T14:59:00.929271+0000","last_unstale":"2024-09-13T15:02:59.298560+0000","last_undegraded":"2024-09-13T15:02:59.298560+0000","last_fullsized":"2024-09-13T15:02:59.298560+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:25.655536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"362'4858","reported_seq":4883,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:29.384667+0000","last_change":"2024-09-13T15:02:39.880701+0000","last_active":"2024-09-13T15:03:29.384667+0000","last_peered":"2024-09-13T15:03:29.384667+0000","last_clean":"2024-09-13T15:03:29.384667+0000","last_became_active":"2024-09-13T15:02:39.880319+0000","last_became_peered":"2024-09-13T15:02:39.880319+0000","last_unstale":"2024-09-13T15:03:29.384667+0000","last_undegraded":"2024-09-13T15:03:29.384667+0000","last_fullsized":"2024-09-13T15:03:29.384667+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":4858,"log_dups_size":0,"ondisk_log_size":4858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:25:44.594262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318373888,"num_objects":4858,"num_object_clones":0,"num_object_copies":14574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4858,"num_write_kb":310912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":195,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303721+0000","last_change":"2024-09-13T14:56:08.889974+0000","last_active":"2024-09-13T15:02:59.303721+0000","last_peered":"2024-09-13T15:02:59.303721+0000","last_clean":"2024-09-13T15:02:59.303721+0000","last_became_active":"2024-09-13T14:55:55.185755+0000","last_became_peered":"2024-09-13T14:55:55.185755+0000","last_unstale":"2024-09-13T15:02:59.303721+0000","last_undegraded":"2024-09-13T15:02:59.303721+0000","last_fullsized":"2024-09-13T15:02:59.303721+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:08.889814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:08.889814+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:02.911564+0000","scrub_duration":0.0053731569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":338,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301006+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T15:02:59.301006+0000","last_peered":"2024-09-13T15:02:59.301006+0000","last_clean":"2024-09-13T15:02:59.301006+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T15:02:59.301006+0000","last_undegraded":"2024-09-13T15:02:59.301006+0000","last_fullsized":"2024-09-13T15:02:59.301006+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":166,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300874+0000","last_change":"2024-09-13T15:02:38.875727+0000","last_active":"2024-09-13T15:02:59.300874+0000","last_peered":"2024-09-13T15:02:59.300874+0000","last_clean":"2024-09-13T15:02:59.300874+0000","last_became_active":"2024-09-13T15:02:38.875372+0000","last_became_peered":"2024-09-13T15:02:38.875372+0000","last_unstale":"2024-09-13T15:02:59.300874+0000","last_undegraded":"2024-09-13T15:02:59.300874+0000","last_fullsized":"2024-09-13T15:02:59.300874+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:08.504810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"362'4847","reported_seq":4872,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:29.361486+0000","last_change":"2024-09-13T15:02:39.921711+0000","last_active":"2024-09-13T15:03:29.361486+0000","last_peered":"2024-09-13T15:03:29.361486+0000","last_clean":"2024-09-13T15:03:29.361486+0000","last_became_active":"2024-09-13T15:02:39.921415+0000","last_became_peered":"2024-09-13T15:02:39.921415+0000","last_unstale":"2024-09-13T15:03:29.361486+0000","last_undegraded":"2024-09-13T15:03:29.361486+0000","last_fullsized":"2024-09-13T15:03:29.361486+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":4847,"log_dups_size":0,"ondisk_log_size":4847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:22.478391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317652992,"num_objects":4847,"num_object_clones":0,"num_object_copies":14541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4847,"num_write_kb":310208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":122,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.299943+0000","last_change":"2024-09-13T15:02:55.358839+0000","last_active":"2024-09-13T15:02:59.299943+0000","last_peered":"2024-09-13T15:02:59.299943+0000","last_clean":"2024-09-13T15:02:59.299943+0000","last_became_active":"2024-09-13T14:59:53.220175+0000","last_became_peered":"2024-09-13T14:59:53.220175+0000","last_unstale":"2024-09-13T15:02:59.299943+0000","last_undegraded":"2024-09-13T15:02:59.299943+0000","last_fullsized":"2024-09-13T15:02:59.299943+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.358699+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.358699+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:13:15.538248+0000","scrub_duration":0.0023994049999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":356,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303893+0000","last_change":"2024-09-13T14:59:00.930126+0000","last_active":"2024-09-13T15:02:59.303893+0000","last_peered":"2024-09-13T15:02:59.303893+0000","last_clean":"2024-09-13T15:02:59.303893+0000","last_became_active":"2024-09-13T14:59:00.929803+0000","last_became_peered":"2024-09-13T14:59:00.929803+0000","last_unstale":"2024-09-13T15:02:59.303893+0000","last_undegraded":"2024-09-13T15:02:59.303893+0000","last_fullsized":"2024-09-13T15:02:59.303893+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:44.456770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":592,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383316+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T15:03:09.383316+0000","last_peered":"2024-09-13T15:03:09.383316+0000","last_clean":"2024-09-13T15:03:09.383316+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T15:03:09.383316+0000","last_undegraded":"2024-09-13T15:03:09.383316+0000","last_fullsized":"2024-09-13T15:03:09.383316+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"363'5145","reported_seq":5172,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:33.918450+0000","last_change":"2024-09-13T15:02:39.881732+0000","last_active":"2024-09-13T15:03:33.918450+0000","last_peered":"2024-09-13T15:03:33.918450+0000","last_clean":"2024-09-13T15:03:33.918450+0000","last_became_active":"2024-09-13T15:02:39.880853+0000","last_became_peered":"2024-09-13T15:02:39.880853+0000","last_unstale":"2024-09-13T15:03:33.918450+0000","last_undegraded":"2024-09-13T15:03:33.918450+0000","last_fullsized":"2024-09-13T15:03:33.918450+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5145,"log_dups_size":0,"ondisk_log_size":5145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:19.523510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337182720,"num_objects":5145,"num_object_clones":0,"num_object_copies":15435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5145,"num_write_kb":329280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":539,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300644+0000","last_change":"2024-09-13T15:02:50.810133+0000","last_active":"2024-09-13T15:02:59.300644+0000","last_peered":"2024-09-13T15:02:59.300644+0000","last_clean":"2024-09-13T15:02:59.300644+0000","last_became_active":"2024-09-13T14:59:53.234208+0000","last_became_peered":"2024-09-13T14:59:53.234208+0000","last_unstale":"2024-09-13T15:02:59.300644+0000","last_undegraded":"2024-09-13T15:02:59.300644+0000","last_fullsized":"2024-09-13T15:02:59.300644+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:50.810045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:50.810045+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:37:54.116248+0000","scrub_duration":0.0042318529999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":131,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.382730+0000","last_change":"2024-09-13T14:59:00.936078+0000","last_active":"2024-09-13T15:03:09.382730+0000","last_peered":"2024-09-13T15:03:09.382730+0000","last_clean":"2024-09-13T15:03:09.382730+0000","last_became_active":"2024-09-13T14:59:00.925997+0000","last_became_peered":"2024-09-13T14:59:00.925997+0000","last_unstale":"2024-09-13T15:03:09.382730+0000","last_undegraded":"2024-09-13T15:03:09.382730+0000","last_fullsized":"2024-09-13T15:03:09.382730+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:43.716136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":356,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298632+0000","last_change":"2024-09-13T14:59:00.934706+0000","last_active":"2024-09-13T15:02:59.298632+0000","last_peered":"2024-09-13T15:02:59.298632+0000","last_clean":"2024-09-13T15:02:59.298632+0000","last_became_active":"2024-09-13T14:59:00.934367+0000","last_became_peered":"2024-09-13T14:59:00.934367+0000","last_unstale":"2024-09-13T15:02:59.298632+0000","last_undegraded":"2024-09-13T15:02:59.298632+0000","last_fullsized":"2024-09-13T15:02:59.298632+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:37:52.860543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"362'5032","reported_seq":5047,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.553737+0000","last_change":"2024-09-13T15:02:39.923388+0000","last_active":"2024-09-13T15:03:31.553737+0000","last_peered":"2024-09-13T15:03:31.553737+0000","last_clean":"2024-09-13T15:03:31.553737+0000","last_became_active":"2024-09-13T15:02:39.921428+0000","last_became_peered":"2024-09-13T15:02:39.921428+0000","last_unstale":"2024-09-13T15:03:31.553737+0000","last_undegraded":"2024-09-13T15:03:31.553737+0000","last_fullsized":"2024-09-13T15:03:31.553737+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5032,"log_dups_size":0,"ondisk_log_size":5032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:55.689072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329777152,"num_objects":5032,"num_object_clones":0,"num_object_copies":15096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5032,"num_write_kb":322048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":303,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303293+0000","last_change":"2024-09-13T15:02:57.138892+0000","last_active":"2024-09-13T15:02:59.303293+0000","last_peered":"2024-09-13T15:02:59.303293+0000","last_clean":"2024-09-13T15:02:59.303293+0000","last_became_active":"2024-09-13T14:59:53.223473+0000","last_became_peered":"2024-09-13T14:59:53.223473+0000","last_unstale":"2024-09-13T15:02:59.303293+0000","last_undegraded":"2024-09-13T15:02:59.303293+0000","last_fullsized":"2024-09-13T15:02:59.303293+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:57.138789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:57.138789+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:17.296460+0000","scrub_duration":0.0036861670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":146,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301184+0000","last_change":"2024-09-13T14:48:32.200521+0000","last_active":"2024-09-13T15:02:59.301184+0000","last_peered":"2024-09-13T15:02:59.301184+0000","last_clean":"2024-09-13T15:02:59.301184+0000","last_became_active":"2024-09-13T14:48:32.200256+0000","last_became_peered":"2024-09-13T14:48:32.200256+0000","last_unstale":"2024-09-13T15:02:59.301184+0000","last_undegraded":"2024-09-13T15:02:59.301184+0000","last_fullsized":"2024-09-13T15:02:59.301184+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:38.899629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":343,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298784+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T15:02:59.298784+0000","last_peered":"2024-09-13T15:02:59.298784+0000","last_clean":"2024-09-13T15:02:59.298784+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T15:02:59.298784+0000","last_undegraded":"2024-09-13T15:02:59.298784+0000","last_fullsized":"2024-09-13T15:02:59.298784+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"362'5075","reported_seq":5090,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.508020+0000","last_change":"2024-09-13T15:02:39.881185+0000","last_active":"2024-09-13T15:03:31.508020+0000","last_peered":"2024-09-13T15:03:31.508020+0000","last_clean":"2024-09-13T15:03:31.508020+0000","last_became_active":"2024-09-13T15:02:39.880841+0000","last_became_peered":"2024-09-13T15:02:39.880841+0000","last_unstale":"2024-09-13T15:03:31.508020+0000","last_undegraded":"2024-09-13T15:03:31.508020+0000","last_fullsized":"2024-09-13T15:03:31.508020+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5075,"log_dups_size":0,"ondisk_log_size":5075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:49:37.338743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332595200,"num_objects":5075,"num_object_clones":0,"num_object_copies":15225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5075,"num_write_kb":324800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":776,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.909213+0000","last_change":"2024-09-13T15:00:10.659910+0000","last_active":"2024-09-13T15:03:31.909213+0000","last_peered":"2024-09-13T15:03:31.909213+0000","last_clean":"2024-09-13T15:03:31.909213+0000","last_became_active":"2024-09-13T14:59:53.216739+0000","last_became_peered":"2024-09-13T14:59:53.216739+0000","last_unstale":"2024-09-13T15:03:31.909213+0000","last_undegraded":"2024-09-13T15:03:31.909213+0000","last_fullsized":"2024-09-13T15:03:31.909213+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.659838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.659838+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:05:29.052471+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":578,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383867+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T15:03:09.383867+0000","last_peered":"2024-09-13T15:03:09.383867+0000","last_clean":"2024-09-13T15:03:09.383867+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T15:03:09.383867+0000","last_undegraded":"2024-09-13T15:03:09.383867+0000","last_fullsized":"2024-09-13T15:03:09.383867+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":346,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.299155+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T15:02:59.299155+0000","last_peered":"2024-09-13T15:02:59.299155+0000","last_clean":"2024-09-13T15:02:59.299155+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T15:02:59.299155+0000","last_undegraded":"2024-09-13T15:02:59.299155+0000","last_fullsized":"2024-09-13T15:02:59.299155+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"362'5101","reported_seq":5116,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.553030+0000","last_change":"2024-09-13T15:02:39.881232+0000","last_active":"2024-09-13T15:03:31.553030+0000","last_peered":"2024-09-13T15:03:31.553030+0000","last_clean":"2024-09-13T15:03:31.553030+0000","last_became_active":"2024-09-13T15:02:39.880899+0000","last_became_peered":"2024-09-13T15:02:39.880899+0000","last_unstale":"2024-09-13T15:03:31.553030+0000","last_undegraded":"2024-09-13T15:03:31.553030+0000","last_fullsized":"2024-09-13T15:03:31.553030+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5101,"log_dups_size":0,"ondisk_log_size":5101,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:53:52.712556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334299136,"num_objects":5101,"num_object_clones":0,"num_object_copies":15303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5101,"num_write_kb":326464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":292,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303492+0000","last_change":"2024-09-13T15:02:58.197427+0000","last_active":"2024-09-13T15:02:59.303492+0000","last_peered":"2024-09-13T15:02:59.303492+0000","last_clean":"2024-09-13T15:02:59.303492+0000","last_became_active":"2024-09-13T14:59:53.232987+0000","last_became_peered":"2024-09-13T14:59:53.232987+0000","last_unstale":"2024-09-13T15:02:59.303492+0000","last_undegraded":"2024-09-13T15:02:59.303492+0000","last_fullsized":"2024-09-13T15:02:59.303492+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:58.195731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:58.195731+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:51.565489+0000","scrub_duration":0.011077536000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":335,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.908899+0000","last_change":"2024-09-13T14:59:00.931433+0000","last_active":"2024-09-13T15:03:31.908899+0000","last_peered":"2024-09-13T15:03:31.908899+0000","last_clean":"2024-09-13T15:03:31.908899+0000","last_became_active":"2024-09-13T14:59:00.928906+0000","last_became_peered":"2024-09-13T14:59:00.928906+0000","last_unstale":"2024-09-13T15:03:31.908899+0000","last_undegraded":"2024-09-13T15:03:31.908899+0000","last_fullsized":"2024-09-13T15:03:31.908899+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:12.786568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":588,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.909525+0000","last_change":"2024-09-13T14:59:00.925277+0000","last_active":"2024-09-13T15:03:31.909525+0000","last_peered":"2024-09-13T15:03:31.909525+0000","last_clean":"2024-09-13T15:03:31.909525+0000","last_became_active":"2024-09-13T14:59:00.924952+0000","last_became_peered":"2024-09-13T14:59:00.924952+0000","last_unstale":"2024-09-13T15:03:31.909525+0000","last_undegraded":"2024-09-13T15:03:31.909525+0000","last_fullsized":"2024-09-13T15:03:31.909525+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:08.888187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"362'4986","reported_seq":5001,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.533431+0000","last_change":"2024-09-13T15:02:39.880337+0000","last_active":"2024-09-13T15:03:31.533431+0000","last_peered":"2024-09-13T15:03:31.533431+0000","last_clean":"2024-09-13T15:03:31.533431+0000","last_became_active":"2024-09-13T15:02:39.880044+0000","last_became_peered":"2024-09-13T15:02:39.880044+0000","last_unstale":"2024-09-13T15:03:31.533431+0000","last_undegraded":"2024-09-13T15:03:31.533431+0000","last_fullsized":"2024-09-13T15:03:31.533431+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":4986,"log_dups_size":0,"ondisk_log_size":4986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:38.582990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326762496,"num_objects":4986,"num_object_clones":0,"num_object_copies":14958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4986,"num_write_kb":319104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":510,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301578+0000","last_change":"2024-09-13T14:55:58.568852+0000","last_active":"2024-09-13T15:02:59.301578+0000","last_peered":"2024-09-13T15:02:59.301578+0000","last_clean":"2024-09-13T15:02:59.301578+0000","last_became_active":"2024-09-13T14:55:51.169987+0000","last_became_peered":"2024-09-13T14:55:51.169987+0000","last_unstale":"2024-09-13T15:02:59.301578+0000","last_undegraded":"2024-09-13T15:02:59.301578+0000","last_fullsized":"2024-09-13T15:02:59.301578+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.568750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.568750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:56.273429+0000","scrub_duration":0.0074886340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":888,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303527+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T15:02:59.303527+0000","last_peered":"2024-09-13T15:02:59.303527+0000","last_clean":"2024-09-13T15:02:59.303527+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T15:02:59.303527+0000","last_undegraded":"2024-09-13T15:02:59.303527+0000","last_fullsized":"2024-09-13T15:02:59.303527+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":182,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303580+0000","last_change":"2024-09-13T14:59:00.930790+0000","last_active":"2024-09-13T15:02:59.303580+0000","last_peered":"2024-09-13T15:02:59.303580+0000","last_clean":"2024-09-13T15:02:59.303580+0000","last_became_active":"2024-09-13T14:59:00.930439+0000","last_became_peered":"2024-09-13T14:59:00.930439+0000","last_unstale":"2024-09-13T15:02:59.303580+0000","last_undegraded":"2024-09-13T15:02:59.303580+0000","last_fullsized":"2024-09-13T15:02:59.303580+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:15.636095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":146,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301039+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T15:02:59.301039+0000","last_peered":"2024-09-13T15:02:59.301039+0000","last_clean":"2024-09-13T15:02:59.301039+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T15:02:59.301039+0000","last_undegraded":"2024-09-13T15:02:59.301039+0000","last_fullsized":"2024-09-13T15:02:59.301039+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"362'4900","reported_seq":4925,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:29.390888+0000","last_change":"2024-09-13T15:02:39.921581+0000","last_active":"2024-09-13T15:03:29.390888+0000","last_peered":"2024-09-13T15:03:29.390888+0000","last_clean":"2024-09-13T15:03:29.390888+0000","last_became_active":"2024-09-13T15:02:39.921286+0000","last_became_peered":"2024-09-13T15:02:39.921286+0000","last_unstale":"2024-09-13T15:03:29.390888+0000","last_undegraded":"2024-09-13T15:03:29.390888+0000","last_fullsized":"2024-09-13T15:03:29.390888+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":4900,"log_dups_size":0,"ondisk_log_size":4900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:54.996704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321126400,"num_objects":4900,"num_object_clones":0,"num_object_copies":14700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4900,"num_write_kb":313600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":748,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.908990+0000","last_change":"2024-09-13T14:55:59.245841+0000","last_active":"2024-09-13T15:03:31.908990+0000","last_peered":"2024-09-13T15:03:31.908990+0000","last_clean":"2024-09-13T15:03:31.908990+0000","last_became_active":"2024-09-13T14:55:51.159487+0000","last_became_peered":"2024-09-13T14:55:51.159487+0000","last_unstale":"2024-09-13T15:03:31.908990+0000","last_undegraded":"2024-09-13T15:03:31.908990+0000","last_fullsized":"2024-09-13T15:03:31.908990+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.245213+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.245213+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:16.628694+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":573,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301324+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T15:02:59.301324+0000","last_peered":"2024-09-13T15:02:59.301324+0000","last_clean":"2024-09-13T15:02:59.301324+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T15:02:59.301324+0000","last_undegraded":"2024-09-13T15:02:59.301324+0000","last_fullsized":"2024-09-13T15:02:59.301324+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":621,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384043+0000","last_change":"2024-09-13T14:59:00.928965+0000","last_active":"2024-09-13T15:03:09.384043+0000","last_peered":"2024-09-13T15:03:09.384043+0000","last_clean":"2024-09-13T15:03:09.384043+0000","last_became_active":"2024-09-13T14:59:00.928166+0000","last_became_peered":"2024-09-13T14:59:00.928166+0000","last_unstale":"2024-09-13T15:03:09.384043+0000","last_undegraded":"2024-09-13T15:03:09.384043+0000","last_fullsized":"2024-09-13T15:03:09.384043+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:34.878172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":617,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384250+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T15:03:09.384250+0000","last_peered":"2024-09-13T15:03:09.384250+0000","last_clean":"2024-09-13T15:03:09.384250+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T15:03:09.384250+0000","last_undegraded":"2024-09-13T15:03:09.384250+0000","last_fullsized":"2024-09-13T15:03:09.384250+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"362'4989","reported_seq":5004,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.550396+0000","last_change":"2024-09-13T15:02:39.921711+0000","last_active":"2024-09-13T15:03:31.550396+0000","last_peered":"2024-09-13T15:03:31.550396+0000","last_clean":"2024-09-13T15:03:31.550396+0000","last_became_active":"2024-09-13T15:02:39.921395+0000","last_became_peered":"2024-09-13T15:02:39.921395+0000","last_unstale":"2024-09-13T15:03:31.550396+0000","last_undegraded":"2024-09-13T15:03:31.550396+0000","last_fullsized":"2024-09-13T15:03:31.550396+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":4989,"log_dups_size":0,"ondisk_log_size":4989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:12.076600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326959104,"num_objects":4989,"num_object_clones":0,"num_object_copies":14967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4989,"num_write_kb":319296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":139,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300947+0000","last_change":"2024-09-13T14:55:59.574827+0000","last_active":"2024-09-13T15:02:59.300947+0000","last_peered":"2024-09-13T15:02:59.300947+0000","last_clean":"2024-09-13T15:02:59.300947+0000","last_became_active":"2024-09-13T14:55:51.162903+0000","last_became_peered":"2024-09-13T14:55:51.162903+0000","last_unstale":"2024-09-13T15:02:59.300947+0000","last_undegraded":"2024-09-13T15:02:59.300947+0000","last_fullsized":"2024-09-13T15:02:59.300947+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.574773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.574773+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:39.074072+0000","scrub_duration":0.001553708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":543,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384169+0000","last_change":"2024-09-13T14:59:00.950349+0000","last_active":"2024-09-13T15:03:09.384169+0000","last_peered":"2024-09-13T15:03:09.384169+0000","last_clean":"2024-09-13T15:03:09.384169+0000","last_became_active":"2024-09-13T14:59:00.925703+0000","last_became_peered":"2024-09-13T14:59:00.925703+0000","last_unstale":"2024-09-13T15:03:09.384169+0000","last_undegraded":"2024-09-13T15:03:09.384169+0000","last_fullsized":"2024-09-13T15:03:09.384169+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:43.851359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":597,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.908920+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T15:03:31.908920+0000","last_peered":"2024-09-13T15:03:31.908920+0000","last_clean":"2024-09-13T15:03:31.908920+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T15:03:31.908920+0000","last_undegraded":"2024-09-13T15:03:31.908920+0000","last_fullsized":"2024-09-13T15:03:31.908920+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":615,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.909340+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T15:03:31.909340+0000","last_peered":"2024-09-13T15:03:31.909340+0000","last_clean":"2024-09-13T15:03:31.909340+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T15:03:31.909340+0000","last_undegraded":"2024-09-13T15:03:31.909340+0000","last_fullsized":"2024-09-13T15:03:31.909340+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"362'5112","reported_seq":5127,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.567274+0000","last_change":"2024-09-13T15:02:39.882940+0000","last_active":"2024-09-13T15:03:31.567274+0000","last_peered":"2024-09-13T15:03:31.567274+0000","last_clean":"2024-09-13T15:03:31.567274+0000","last_became_active":"2024-09-13T15:02:39.882641+0000","last_became_peered":"2024-09-13T15:02:39.882641+0000","last_unstale":"2024-09-13T15:03:31.567274+0000","last_undegraded":"2024-09-13T15:03:31.567274+0000","last_fullsized":"2024-09-13T15:03:31.567274+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5112,"log_dups_size":0,"ondisk_log_size":5112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:10.481971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335020032,"num_objects":5112,"num_object_clones":0,"num_object_copies":15336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5112,"num_write_kb":327168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":282,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303185+0000","last_change":"2024-09-13T14:56:01.658240+0000","last_active":"2024-09-13T15:02:59.303185+0000","last_peered":"2024-09-13T15:02:59.303185+0000","last_clean":"2024-09-13T15:02:59.303185+0000","last_became_active":"2024-09-13T14:55:51.160480+0000","last_became_peered":"2024-09-13T14:55:51.160480+0000","last_unstale":"2024-09-13T15:02:59.303185+0000","last_undegraded":"2024-09-13T15:02:59.303185+0000","last_fullsized":"2024-09-13T15:02:59.303185+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.658143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.658143+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:19.262955+0000","scrub_duration":0.0038903079999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":988,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303957+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T15:02:59.303957+0000","last_peered":"2024-09-13T15:02:59.303957+0000","last_clean":"2024-09-13T15:02:59.303957+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T15:02:59.303957+0000","last_undegraded":"2024-09-13T15:02:59.303957+0000","last_fullsized":"2024-09-13T15:02:59.303957+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":347,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298513+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T15:02:59.298513+0000","last_peered":"2024-09-13T15:02:59.298513+0000","last_clean":"2024-09-13T15:02:59.298513+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T15:02:59.298513+0000","last_undegraded":"2024-09-13T15:02:59.298513+0000","last_fullsized":"2024-09-13T15:02:59.298513+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":621,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.909158+0000","last_change":"2024-09-13T14:59:00.930713+0000","last_active":"2024-09-13T15:03:31.909158+0000","last_peered":"2024-09-13T15:03:31.909158+0000","last_clean":"2024-09-13T15:03:31.909158+0000","last_became_active":"2024-09-13T14:59:00.928260+0000","last_became_peered":"2024-09-13T14:59:00.928260+0000","last_unstale":"2024-09-13T15:03:31.909158+0000","last_undegraded":"2024-09-13T15:03:31.909158+0000","last_fullsized":"2024-09-13T15:03:31.909158+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:07.048853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"362'5040","reported_seq":5055,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.528690+0000","last_change":"2024-09-13T15:02:39.882990+0000","last_active":"2024-09-13T15:03:31.528690+0000","last_peered":"2024-09-13T15:03:31.528690+0000","last_clean":"2024-09-13T15:03:31.528690+0000","last_became_active":"2024-09-13T15:02:39.882663+0000","last_became_peered":"2024-09-13T15:02:39.882663+0000","last_unstale":"2024-09-13T15:03:31.528690+0000","last_undegraded":"2024-09-13T15:03:31.528690+0000","last_fullsized":"2024-09-13T15:03:31.528690+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5040,"log_dups_size":0,"ondisk_log_size":5040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:20.368697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330301440,"num_objects":5040,"num_object_clones":0,"num_object_copies":15120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5040,"num_write_kb":322560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":225,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303635+0000","last_change":"2024-09-13T15:02:49.771643+0000","last_active":"2024-09-13T15:02:59.303635+0000","last_peered":"2024-09-13T15:02:59.303635+0000","last_clean":"2024-09-13T15:02:59.303635+0000","last_became_active":"2024-09-13T14:59:53.249243+0000","last_became_peered":"2024-09-13T14:59:53.249243+0000","last_unstale":"2024-09-13T15:02:59.303635+0000","last_undegraded":"2024-09-13T15:02:59.303635+0000","last_fullsized":"2024-09-13T15:02:59.303635+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:49.771572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:49.771572+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:11:27.741449+0000","scrub_duration":0.0035237329999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":353,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303548+0000","last_change":"2024-09-13T14:59:00.931326+0000","last_active":"2024-09-13T15:02:59.303548+0000","last_peered":"2024-09-13T15:02:59.303548+0000","last_clean":"2024-09-13T15:02:59.303548+0000","last_became_active":"2024-09-13T14:59:00.931011+0000","last_became_peered":"2024-09-13T14:59:00.931011+0000","last_unstale":"2024-09-13T15:02:59.303548+0000","last_undegraded":"2024-09-13T15:02:59.303548+0000","last_fullsized":"2024-09-13T15:02:59.303548+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:24:41.833263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":1099,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.909273+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T15:03:31.909273+0000","last_peered":"2024-09-13T15:03:31.909273+0000","last_clean":"2024-09-13T15:03:31.909273+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T15:03:31.909273+0000","last_undegraded":"2024-09-13T15:03:31.909273+0000","last_fullsized":"2024-09-13T15:03:31.909273+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":571,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.909556+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T15:03:31.909556+0000","last_peered":"2024-09-13T15:03:31.909556+0000","last_clean":"2024-09-13T15:03:31.909556+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T15:03:31.909556+0000","last_undegraded":"2024-09-13T15:03:31.909556+0000","last_fullsized":"2024-09-13T15:03:31.909556+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"362'5109","reported_seq":5124,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.569359+0000","last_change":"2024-09-13T15:02:39.881285+0000","last_active":"2024-09-13T15:03:31.569359+0000","last_peered":"2024-09-13T15:03:31.569359+0000","last_clean":"2024-09-13T15:03:31.569359+0000","last_became_active":"2024-09-13T15:02:39.880984+0000","last_became_peered":"2024-09-13T15:02:39.880984+0000","last_unstale":"2024-09-13T15:03:31.569359+0000","last_undegraded":"2024-09-13T15:03:31.569359+0000","last_fullsized":"2024-09-13T15:03:31.569359+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5109,"log_dups_size":0,"ondisk_log_size":5109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:31.193802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334823424,"num_objects":5109,"num_object_clones":0,"num_object_copies":15327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5109,"num_write_kb":326976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":215,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300994+0000","last_change":"2024-09-13T15:02:51.748425+0000","last_active":"2024-09-13T15:02:59.300994+0000","last_peered":"2024-09-13T15:02:59.300994+0000","last_clean":"2024-09-13T15:02:59.300994+0000","last_became_active":"2024-09-13T14:59:53.226288+0000","last_became_peered":"2024-09-13T14:59:53.226288+0000","last_unstale":"2024-09-13T15:02:59.300994+0000","last_undegraded":"2024-09-13T15:02:59.300994+0000","last_fullsized":"2024-09-13T15:02:59.300994+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:51.748121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:51.748121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:17:47.226084+0000","scrub_duration":0.0026525849999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":596,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.909257+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T15:03:31.909257+0000","last_peered":"2024-09-13T15:03:31.909257+0000","last_clean":"2024-09-13T15:03:31.909257+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T15:03:31.909257+0000","last_undegraded":"2024-09-13T15:03:31.909257+0000","last_fullsized":"2024-09-13T15:03:31.909257+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":575,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301269+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T15:02:59.301269+0000","last_peered":"2024-09-13T15:02:59.301269+0000","last_clean":"2024-09-13T15:02:59.301269+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T15:02:59.301269+0000","last_undegraded":"2024-09-13T15:02:59.301269+0000","last_fullsized":"2024-09-13T15:02:59.301269+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":582,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383236+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T15:03:09.383236+0000","last_peered":"2024-09-13T15:03:09.383236+0000","last_clean":"2024-09-13T15:03:09.383236+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T15:03:09.383236+0000","last_undegraded":"2024-09-13T15:03:09.383236+0000","last_fullsized":"2024-09-13T15:03:09.383236+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"362'5144","reported_seq":5158,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.561130+0000","last_change":"2024-09-13T15:02:39.921655+0000","last_active":"2024-09-13T15:03:31.561130+0000","last_peered":"2024-09-13T15:03:31.561130+0000","last_clean":"2024-09-13T15:03:31.561130+0000","last_became_active":"2024-09-13T15:02:39.921331+0000","last_became_peered":"2024-09-13T15:02:39.921331+0000","last_unstale":"2024-09-13T15:03:31.561130+0000","last_undegraded":"2024-09-13T15:03:31.561130+0000","last_fullsized":"2024-09-13T15:03:31.561130+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5144,"log_dups_size":0,"ondisk_log_size":5144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:48.004179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337117184,"num_objects":5144,"num_object_clones":0,"num_object_copies":15432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5144,"num_write_kb":329216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":333,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298498+0000","last_change":"2024-09-13T15:02:52.361444+0000","last_active":"2024-09-13T15:02:59.298498+0000","last_peered":"2024-09-13T15:02:59.298498+0000","last_clean":"2024-09-13T15:02:59.298498+0000","last_became_active":"2024-09-13T14:59:53.220135+0000","last_became_peered":"2024-09-13T14:59:53.220135+0000","last_unstale":"2024-09-13T15:02:59.298498+0000","last_undegraded":"2024-09-13T15:02:59.298498+0000","last_fullsized":"2024-09-13T15:02:59.298498+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.361359+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.361359+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:35.003140+0000","scrub_duration":0.005845154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":83,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301509+0000","last_change":"2024-09-13T14:59:00.936064+0000","last_active":"2024-09-13T15:02:59.301509+0000","last_peered":"2024-09-13T15:02:59.301509+0000","last_clean":"2024-09-13T15:02:59.301509+0000","last_became_active":"2024-09-13T14:59:00.932311+0000","last_became_peered":"2024-09-13T14:59:00.932311+0000","last_unstale":"2024-09-13T15:02:59.301509+0000","last_undegraded":"2024-09-13T15:02:59.301509+0000","last_fullsized":"2024-09-13T15:02:59.301509+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:16.456004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":954,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.909634+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T15:03:31.909634+0000","last_peered":"2024-09-13T15:03:31.909634+0000","last_clean":"2024-09-13T15:03:31.909634+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T15:03:31.909634+0000","last_undegraded":"2024-09-13T15:03:31.909634+0000","last_fullsized":"2024-09-13T15:03:31.909634+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":84,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300375+0000","last_change":"2024-09-13T14:59:00.924324+0000","last_active":"2024-09-13T15:02:59.300375+0000","last_peered":"2024-09-13T15:02:59.300375+0000","last_clean":"2024-09-13T15:02:59.300375+0000","last_became_active":"2024-09-13T14:59:00.923798+0000","last_became_peered":"2024-09-13T14:59:00.923798+0000","last_unstale":"2024-09-13T15:02:59.300375+0000","last_undegraded":"2024-09-13T15:02:59.300375+0000","last_fullsized":"2024-09-13T15:02:59.300375+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:10.994877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"362'5069","reported_seq":5083,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.525480+0000","last_change":"2024-09-13T15:02:39.922368+0000","last_active":"2024-09-13T15:03:31.525480+0000","last_peered":"2024-09-13T15:03:31.525480+0000","last_clean":"2024-09-13T15:03:31.525480+0000","last_became_active":"2024-09-13T15:02:39.922106+0000","last_became_peered":"2024-09-13T15:02:39.922106+0000","last_unstale":"2024-09-13T15:03:31.525480+0000","last_undegraded":"2024-09-13T15:03:31.525480+0000","last_fullsized":"2024-09-13T15:03:31.525480+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5069,"log_dups_size":0,"ondisk_log_size":5069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:13:52.416513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332201984,"num_objects":5069,"num_object_clones":0,"num_object_copies":15207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5069,"num_write_kb":324416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":70,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303721+0000","last_change":"2024-09-13T15:02:54.193233+0000","last_active":"2024-09-13T15:02:59.303721+0000","last_peered":"2024-09-13T15:02:59.303721+0000","last_clean":"2024-09-13T15:02:59.303721+0000","last_became_active":"2024-09-13T14:59:00.925167+0000","last_became_peered":"2024-09-13T14:59:00.925167+0000","last_unstale":"2024-09-13T15:02:59.303721+0000","last_undegraded":"2024-09-13T15:02:59.303721+0000","last_fullsized":"2024-09-13T15:02:59.303721+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:54.193156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:54.193156+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:16.851666+0000","scrub_duration":0.0040069900000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":603,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383725+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T15:03:09.383725+0000","last_peered":"2024-09-13T15:03:09.383725+0000","last_clean":"2024-09-13T15:03:09.383725+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T15:03:09.383725+0000","last_undegraded":"2024-09-13T15:03:09.383725+0000","last_fullsized":"2024-09-13T15:03:09.383725+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1175,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303972+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T15:02:59.303972+0000","last_peered":"2024-09-13T15:02:59.303972+0000","last_clean":"2024-09-13T15:02:59.303972+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T15:02:59.303972+0000","last_undegraded":"2024-09-13T15:02:59.303972+0000","last_fullsized":"2024-09-13T15:02:59.303972+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":329,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298065+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T15:02:59.298065+0000","last_peered":"2024-09-13T15:02:59.298065+0000","last_clean":"2024-09-13T15:02:59.298065+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T15:02:59.298065+0000","last_undegraded":"2024-09-13T15:02:59.298065+0000","last_fullsized":"2024-09-13T15:02:59.298065+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"82'100","reported_seq":57,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.909522+0000","last_change":"2024-09-13T15:02:51.192587+0000","last_active":"2024-09-13T15:03:31.909522+0000","last_peered":"2024-09-13T15:03:31.909522+0000","last_clean":"2024-09-13T15:03:31.909522+0000","last_became_active":"2024-09-13T15:02:23.878033+0000","last_became_peered":"2024-09-13T15:02:23.878033+0000","last_unstale":"2024-09-13T15:03:31.909522+0000","last_undegraded":"2024-09-13T15:03:31.909522+0000","last_fullsized":"2024-09-13T15:03:31.909522+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:51.192503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:51.192503+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:25:55.606880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":346,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298395+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T15:02:59.298395+0000","last_peered":"2024-09-13T15:02:59.298395+0000","last_clean":"2024-09-13T15:02:59.298395+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T15:02:59.298395+0000","last_undegraded":"2024-09-13T15:02:59.298395+0000","last_fullsized":"2024-09-13T15:02:59.298395+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":610,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383296+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T15:03:09.383296+0000","last_peered":"2024-09-13T15:03:09.383296+0000","last_clean":"2024-09-13T15:03:09.383296+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T15:03:09.383296+0000","last_undegraded":"2024-09-13T15:03:09.383296+0000","last_fullsized":"2024-09-13T15:03:09.383296+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.17","version":"82'100","reported_seq":174,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.108097+0000","last_change":"2024-09-13T15:03:03.108031+0000","last_active":"2024-09-13T15:03:03.108097+0000","last_peered":"2024-09-13T15:03:03.108097+0000","last_clean":"2024-09-13T15:03:03.108097+0000","last_became_active":"2024-09-13T15:02:23.883373+0000","last_became_peered":"2024-09-13T15:02:23.883373+0000","last_unstale":"2024-09-13T15:03:03.108097+0000","last_undegraded":"2024-09-13T15:03:03.108097+0000","last_fullsized":"2024-09-13T15:03:03.108097+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:03.107786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:03.107786+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:17:10.041468+0000","scrub_duration":0.0049024139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":595,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.909145+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T15:03:31.909145+0000","last_peered":"2024-09-13T15:03:31.909145+0000","last_clean":"2024-09-13T15:03:31.909145+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T15:03:31.909145+0000","last_undegraded":"2024-09-13T15:03:31.909145+0000","last_fullsized":"2024-09-13T15:03:31.909145+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":352,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300868+0000","last_change":"2024-09-13T14:59:00.929406+0000","last_active":"2024-09-13T15:02:59.300868+0000","last_peered":"2024-09-13T15:02:59.300868+0000","last_clean":"2024-09-13T15:02:59.300868+0000","last_became_active":"2024-09-13T14:59:00.928524+0000","last_became_peered":"2024-09-13T14:59:00.928524+0000","last_unstale":"2024-09-13T15:02:59.300868+0000","last_undegraded":"2024-09-13T15:02:59.300868+0000","last_fullsized":"2024-09-13T15:02:59.300868+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:10.609448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":174,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.387491+0000","last_change":"2024-09-13T14:56:01.235655+0000","last_active":"2024-09-13T15:03:09.387491+0000","last_peered":"2024-09-13T15:03:09.387491+0000","last_clean":"2024-09-13T15:03:09.387491+0000","last_became_active":"2024-09-13T14:55:55.195703+0000","last_became_peered":"2024-09-13T14:55:55.195703+0000","last_unstale":"2024-09-13T15:03:09.387491+0000","last_undegraded":"2024-09-13T15:03:09.387491+0000","last_fullsized":"2024-09-13T15:03:09.387491+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.235497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.235497+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:23:21.640831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":326,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303817+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T15:02:59.303817+0000","last_peered":"2024-09-13T15:02:59.303817+0000","last_clean":"2024-09-13T15:02:59.303817+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T15:02:59.303817+0000","last_undegraded":"2024-09-13T15:02:59.303817+0000","last_fullsized":"2024-09-13T15:02:59.303817+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":333,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300722+0000","last_change":"2024-09-13T14:59:00.926586+0000","last_active":"2024-09-13T15:02:59.300722+0000","last_peered":"2024-09-13T15:02:59.300722+0000","last_clean":"2024-09-13T15:02:59.300722+0000","last_became_active":"2024-09-13T14:59:00.926206+0000","last_became_peered":"2024-09-13T14:59:00.926206+0000","last_unstale":"2024-09-13T15:02:59.300722+0000","last_undegraded":"2024-09-13T15:02:59.300722+0000","last_fullsized":"2024-09-13T15:02:59.300722+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:17:18.805856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.15","version":"82'100","reported_seq":30,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.301186+0000","last_change":"2024-09-13T15:02:55.870615+0000","last_active":"2024-09-13T15:02:59.301186+0000","last_peered":"2024-09-13T15:02:59.301186+0000","last_clean":"2024-09-13T15:02:59.301186+0000","last_became_active":"2024-09-13T15:02:23.883217+0000","last_became_peered":"2024-09-13T15:02:23.883217+0000","last_unstale":"2024-09-13T15:02:59.301186+0000","last_undegraded":"2024-09-13T15:02:59.301186+0000","last_fullsized":"2024-09-13T15:02:59.301186+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.870528+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.870528+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:10.023787+0000","scrub_duration":0.0047551850000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":597,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383095+0000","last_change":"2024-09-13T14:59:00.932503+0000","last_active":"2024-09-13T15:03:09.383095+0000","last_peered":"2024-09-13T15:03:09.383095+0000","last_clean":"2024-09-13T15:03:09.383095+0000","last_became_active":"2024-09-13T14:59:00.931358+0000","last_became_peered":"2024-09-13T14:59:00.931358+0000","last_unstale":"2024-09-13T15:03:09.383095+0000","last_undegraded":"2024-09-13T15:03:09.383095+0000","last_fullsized":"2024-09-13T15:03:09.383095+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:19.226182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":431,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.908794+0000","last_change":"2024-09-13T14:59:00.927060+0000","last_active":"2024-09-13T15:03:31.908794+0000","last_peered":"2024-09-13T15:03:31.908794+0000","last_clean":"2024-09-13T15:03:31.908794+0000","last_became_active":"2024-09-13T14:59:00.925402+0000","last_became_peered":"2024-09-13T14:59:00.925402+0000","last_unstale":"2024-09-13T15:03:31.908794+0000","last_undegraded":"2024-09-13T15:03:31.908794+0000","last_fullsized":"2024-09-13T15:03:31.908794+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:31:08.336446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"82'100","reported_seq":30,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.299016+0000","last_change":"2024-09-13T15:02:56.319389+0000","last_active":"2024-09-13T15:02:59.299016+0000","last_peered":"2024-09-13T15:02:59.299016+0000","last_clean":"2024-09-13T15:02:59.299016+0000","last_became_active":"2024-09-13T15:02:23.876739+0000","last_became_peered":"2024-09-13T15:02:23.876739+0000","last_unstale":"2024-09-13T15:02:59.299016+0000","last_undegraded":"2024-09-13T15:02:59.299016+0000","last_fullsized":"2024-09-13T15:02:59.299016+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:56.319302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:56.319302+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:58.230158+0000","scrub_duration":0.0031182190000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":146,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303800+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T15:02:59.303800+0000","last_peered":"2024-09-13T15:02:59.303800+0000","last_clean":"2024-09-13T15:02:59.303800+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T15:02:59.303800+0000","last_undegraded":"2024-09-13T15:02:59.303800+0000","last_fullsized":"2024-09-13T15:02:59.303800+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":187,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.382484+0000","last_change":"2024-09-13T14:59:00.924196+0000","last_active":"2024-09-13T15:03:09.382484+0000","last_peered":"2024-09-13T15:03:09.382484+0000","last_clean":"2024-09-13T15:03:09.382484+0000","last_became_active":"2024-09-13T14:59:00.923901+0000","last_became_peered":"2024-09-13T14:59:00.923901+0000","last_unstale":"2024-09-13T15:03:09.382484+0000","last_undegraded":"2024-09-13T15:03:09.382484+0000","last_fullsized":"2024-09-13T15:03:09.382484+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:01.913804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"82'100","reported_seq":612,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:07.166878+0000","last_change":"2024-09-13T15:03:07.166808+0000","last_active":"2024-09-13T15:03:07.166878+0000","last_peered":"2024-09-13T15:03:07.166878+0000","last_clean":"2024-09-13T15:03:07.166878+0000","last_became_active":"2024-09-13T15:02:23.883387+0000","last_became_peered":"2024-09-13T15:02:23.883387+0000","last_unstale":"2024-09-13T15:03:07.166878+0000","last_undegraded":"2024-09-13T15:03:07.166878+0000","last_fullsized":"2024-09-13T15:03:07.166878+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:07.166034+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:07.166034+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:32:42.605595+0000","scrub_duration":0.01094734,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":621,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384165+0000","last_change":"2024-09-13T14:59:00.924814+0000","last_active":"2024-09-13T15:03:09.384165+0000","last_peered":"2024-09-13T15:03:09.384165+0000","last_clean":"2024-09-13T15:03:09.384165+0000","last_became_active":"2024-09-13T14:59:00.924501+0000","last_became_peered":"2024-09-13T14:59:00.924501+0000","last_unstale":"2024-09-13T15:03:09.384165+0000","last_undegraded":"2024-09-13T15:03:09.384165+0000","last_fullsized":"2024-09-13T15:03:09.384165+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:11:15.382207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":587,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.908835+0000","last_change":"2024-09-13T14:59:00.925431+0000","last_active":"2024-09-13T15:03:31.908835+0000","last_peered":"2024-09-13T15:03:31.908835+0000","last_clean":"2024-09-13T15:03:31.908835+0000","last_became_active":"2024-09-13T14:59:00.925023+0000","last_became_peered":"2024-09-13T14:59:00.925023+0000","last_unstale":"2024-09-13T15:03:31.908835+0000","last_undegraded":"2024-09-13T15:03:31.908835+0000","last_fullsized":"2024-09-13T15:03:31.908835+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:38:18.686253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"82'100","reported_seq":317,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:03:04.101991+0000","last_change":"2024-09-13T15:03:04.101898+0000","last_active":"2024-09-13T15:03:04.101991+0000","last_peered":"2024-09-13T15:03:04.101991+0000","last_clean":"2024-09-13T15:03:04.101991+0000","last_became_active":"2024-09-13T15:02:23.884406+0000","last_became_peered":"2024-09-13T15:02:23.884406+0000","last_unstale":"2024-09-13T15:03:04.101991+0000","last_undegraded":"2024-09-13T15:03:04.101991+0000","last_fullsized":"2024-09-13T15:03:04.101991+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:04.101802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:04.101802+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:34.459623+0000","scrub_duration":0.003875819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":136,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298649+0000","last_change":"2024-09-13T14:59:00.927250+0000","last_active":"2024-09-13T15:02:59.298649+0000","last_peered":"2024-09-13T15:02:59.298649+0000","last_clean":"2024-09-13T15:02:59.298649+0000","last_became_active":"2024-09-13T14:59:00.926953+0000","last_became_peered":"2024-09-13T14:59:00.926953+0000","last_unstale":"2024-09-13T15:02:59.298649+0000","last_undegraded":"2024-09-13T15:02:59.298649+0000","last_fullsized":"2024-09-13T15:02:59.298649+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:58.306547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":345,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.299046+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T15:02:59.299046+0000","last_peered":"2024-09-13T15:02:59.299046+0000","last_clean":"2024-09-13T15:02:59.299046+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T15:02:59.299046+0000","last_undegraded":"2024-09-13T15:02:59.299046+0000","last_fullsized":"2024-09-13T15:02:59.299046+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"82'100","reported_seq":55,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383145+0000","last_change":"2024-09-13T15:02:52.429623+0000","last_active":"2024-09-13T15:03:09.383145+0000","last_peered":"2024-09-13T15:03:09.383145+0000","last_clean":"2024-09-13T15:03:09.383145+0000","last_became_active":"2024-09-13T15:02:23.880201+0000","last_became_peered":"2024-09-13T15:02:23.880201+0000","last_unstale":"2024-09-13T15:03:09.383145+0000","last_undegraded":"2024-09-13T15:03:09.383145+0000","last_fullsized":"2024-09-13T15:03:09.383145+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.429543+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.429543+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:43.145050+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":421,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.909486+0000","last_change":"2024-09-13T14:59:00.934412+0000","last_active":"2024-09-13T15:03:31.909486+0000","last_peered":"2024-09-13T15:03:31.909486+0000","last_clean":"2024-09-13T15:03:31.909486+0000","last_became_active":"2024-09-13T14:59:00.932986+0000","last_became_peered":"2024-09-13T14:59:00.932986+0000","last_unstale":"2024-09-13T15:03:31.909486+0000","last_undegraded":"2024-09-13T15:03:31.909486+0000","last_fullsized":"2024-09-13T15:03:31.909486+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:21:44.285324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":583,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.383187+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T15:03:09.383187+0000","last_peered":"2024-09-13T15:03:09.383187+0000","last_clean":"2024-09-13T15:03:09.383187+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T15:03:09.383187+0000","last_undegraded":"2024-09-13T15:03:09.383187+0000","last_fullsized":"2024-09-13T15:03:09.383187+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"82'100","reported_seq":30,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300548+0000","last_change":"2024-09-13T15:02:55.679993+0000","last_active":"2024-09-13T15:02:59.300548+0000","last_peered":"2024-09-13T15:02:59.300548+0000","last_clean":"2024-09-13T15:02:59.300548+0000","last_became_active":"2024-09-13T15:02:23.878770+0000","last_became_peered":"2024-09-13T15:02:23.878770+0000","last_unstale":"2024-09-13T15:02:59.300548+0000","last_undegraded":"2024-09-13T15:02:59.300548+0000","last_fullsized":"2024-09-13T15:02:59.300548+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.679893+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.679893+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:25.717099+0000","scrub_duration":0.0050514899999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":353,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298332+0000","last_change":"2024-09-13T14:59:00.929751+0000","last_active":"2024-09-13T15:02:59.298332+0000","last_peered":"2024-09-13T15:02:59.298332+0000","last_clean":"2024-09-13T15:02:59.298332+0000","last_became_active":"2024-09-13T14:59:00.929456+0000","last_became_peered":"2024-09-13T14:59:00.929456+0000","last_unstale":"2024-09-13T15:02:59.298332+0000","last_undegraded":"2024-09-13T15:02:59.298332+0000","last_fullsized":"2024-09-13T15:02:59.298332+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:11.462853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":328,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303808+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T15:02:59.303808+0000","last_peered":"2024-09-13T15:02:59.303808+0000","last_clean":"2024-09-13T15:02:59.303808+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T15:02:59.303808+0000","last_undegraded":"2024-09-13T15:02:59.303808+0000","last_fullsized":"2024-09-13T15:02:59.303808+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":353,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.298933+0000","last_change":"2024-09-13T14:59:00.928898+0000","last_active":"2024-09-13T15:02:59.298933+0000","last_peered":"2024-09-13T15:02:59.298933+0000","last_clean":"2024-09-13T15:02:59.298933+0000","last_became_active":"2024-09-13T14:59:00.928585+0000","last_became_peered":"2024-09-13T14:59:00.928585+0000","last_unstale":"2024-09-13T15:02:59.298933+0000","last_undegraded":"2024-09-13T15:02:59.298933+0000","last_fullsized":"2024-09-13T15:02:59.298933+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:26:57.155149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":367,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.303370+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T15:02:59.303370+0000","last_peered":"2024-09-13T15:02:59.303370+0000","last_clean":"2024-09-13T15:02:59.303370+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T15:02:59.303370+0000","last_undegraded":"2024-09-13T15:02:59.303370+0000","last_fullsized":"2024-09-13T15:02:59.303370+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"82'100","reported_seq":20,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300672+0000","last_change":"2024-09-13T15:02:23.879862+0000","last_active":"2024-09-13T15:02:59.300672+0000","last_peered":"2024-09-13T15:02:59.300672+0000","last_clean":"2024-09-13T15:02:59.300672+0000","last_became_active":"2024-09-13T15:02:23.879170+0000","last_became_peered":"2024-09-13T15:02:23.879170+0000","last_unstale":"2024-09-13T15:02:59.300672+0000","last_undegraded":"2024-09-13T15:02:59.300672+0000","last_fullsized":"2024-09-13T15:02:59.300672+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:06.945525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:06.945525+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:48.496118+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":153,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.384043+0000","last_change":"2024-09-13T14:59:00.929835+0000","last_active":"2024-09-13T15:03:09.384043+0000","last_peered":"2024-09-13T15:03:09.384043+0000","last_clean":"2024-09-13T15:03:09.384043+0000","last_became_active":"2024-09-13T14:59:00.929514+0000","last_became_peered":"2024-09-13T14:59:00.929514+0000","last_unstale":"2024-09-13T15:03:09.384043+0000","last_undegraded":"2024-09-13T15:03:09.384043+0000","last_fullsized":"2024-09-13T15:03:09.384043+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:48.981240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":389,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.300857+0000","last_change":"2024-09-13T14:59:00.925895+0000","last_active":"2024-09-13T15:02:59.300857+0000","last_peered":"2024-09-13T15:02:59.300857+0000","last_clean":"2024-09-13T15:02:59.300857+0000","last_became_active":"2024-09-13T14:59:00.925564+0000","last_became_peered":"2024-09-13T14:59:00.925564+0000","last_unstale":"2024-09-13T15:02:59.300857+0000","last_undegraded":"2024-09-13T15:02:59.300857+0000","last_fullsized":"2024-09-13T15:02:59.300857+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:23.283927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1d","version":"82'100","reported_seq":59,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T15:03:31.909182+0000","last_change":"2024-09-13T15:02:52.149390+0000","last_active":"2024-09-13T15:03:31.909182+0000","last_peered":"2024-09-13T15:03:31.909182+0000","last_clean":"2024-09-13T15:03:31.909182+0000","last_became_active":"2024-09-13T15:02:23.914809+0000","last_became_peered":"2024-09-13T15:02:23.914809+0000","last_unstale":"2024-09-13T15:03:31.909182+0000","last_undegraded":"2024-09-13T15:03:31.909182+0000","last_fullsized":"2024-09-13T15:03:31.909182+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.149290+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.149290+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:59.584474+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":349,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.297888+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T15:02:59.297888+0000","last_peered":"2024-09-13T15:02:59.297888+0000","last_clean":"2024-09-13T15:02:59.297888+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T15:02:59.297888+0000","last_undegraded":"2024-09-13T15:02:59.297888+0000","last_fullsized":"2024-09-13T15:02:59.297888+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":635,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T15:03:09.382997+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T15:03:09.382997+0000","last_peered":"2024-09-13T15:03:09.382997+0000","last_clean":"2024-09-13T15:03:09.382997+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T15:03:09.382997+0000","last_undegraded":"2024-09-13T15:03:09.382997+0000","last_fullsized":"2024-09-13T15:03:09.382997+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5276368896,"num_objects":80511,"num_object_clones":0,"num_object_copies":241533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":80511,"num_write_kb":5152704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15737946112,"data_stored":15737946112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":80511,"ondisk_log_size":80511,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":16172,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397955,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2490184,"kb_used_data":1596608,"kb_used_omap":1,"kb_used_meta":893566,"kb_avail":91254968,"statfs":{"total":95995035648,"available":93445087232,"internally_reserved":0,"allocated":1634926592,"data_stored":1632420276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915011928},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594184,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2558076,"kb_used_data":2256564,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91187076,"statfs":{"total":95995035648,"available":93375565824,"internally_reserved":0,"allocated":2310721536,"data_stored":2308226536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,5,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790413,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2504952,"kb_used_data":1616240,"kb_used_omap":1,"kb_used_meta":888702,"kb_avail":91240200,"statfs":{"total":95995035648,"available":93429964800,"internally_reserved":0,"allocated":1655029760,"data_stored":1652535297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910031192},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986642,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4118200,"kb_used_data":3229296,"kb_used_omap":1,"kb_used_meta":888894,"kb_avail":89626952,"statfs":{"total":95995035648,"available":91777998848,"internally_reserved":0,"allocated":3306799104,"data_stored":3304303828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910227800},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":0,"seq":1438814044189,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762616,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3083448,"kb_used_data":2156336,"kb_used_omap":8,"kb_used_meta":927095,"kb_avail":90661704,"statfs":{"total":95995035648,"available":92837584896,"internally_reserved":0,"allocated":2208088064,"data_stored":2205593596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":949345738},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926143,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2226432,"kb_used_data":1337976,"kb_used_omap":7,"kb_used_meta":888440,"kb_avail":91518720,"statfs":{"total":95995035648,"available":93715169280,"internally_reserved":0,"allocated":1370087424,"data_stored":1367592452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":909763273},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122382,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3922756,"kb_used_data":3068476,"kb_used_omap":7,"kb_used_meta":854264,"kb_avail":89822396,"statfs":{"total":95995035648,"available":91978133504,"internally_reserved":0,"allocated":3142119424,"data_stored":3139617168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":874766927},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3190554624,"data_stored":3190554624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1361969152,"data_stored":1361969152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2234318848,"data_stored":2234318848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3316252672,"data_stored":3316252672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1653997568,"data_stored":1653997568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2329411584,"data_stored":2329411584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1651441664,"data_stored":1651441664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:03:37.416 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 31 2024-09-13T15:03:37.506 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:37.544 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:37.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:37.646 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:37.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.692 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:37.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:37.793 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:37.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.831 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:37.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:37.932 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:37.964 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:37.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:38.066 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:38.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.109 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:38.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:38.165 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 48 16 88433 88417 115.099 70 0.0152607 0.0086837 2024-09-13T15:03:38.202 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:37 smithi096 ceph-mon[98901]: pgmap v1214: 119 pgs: 119 active+clean; 5.0 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T15:03:38.202 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:03:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:03:37] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-09-13T15:03:38.203 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:37 smithi096 ceph-mon[102928]: pgmap v1214: 119 pgs: 119 active+clean; 5.0 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T15:03:38.210 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:37 smithi204 ceph-mon[87071]: pgmap v1214: 119 pgs: 119 active+clean; 5.0 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T15:03:38.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.248 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:38.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:38.349 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:38.388 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:38.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:38.490 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:38.533 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:38.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:38.635 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:38.689 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:38.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:38.790 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:38.820 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:38.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:38.923 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:38.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.963 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:38.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:39.064 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:39.093 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:39.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:39.165 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 49 16 89574 89558 114.205 71.3125 0.0312667 0.00875159 2024-09-13T15:03:39.195 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:39.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.224 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:39.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:39.325 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:39.360 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:39.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:39.462 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:39.503 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:39.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:39.605 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:39.633 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:39.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:39.735 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:39.764 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:39.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:39.866 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:39.900 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:39.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:40.002 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:40.032 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:40.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:40.134 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:40.165 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 50 16 90711 90695 113.342 71.0625 0.0222572 0.00881775 2024-09-13T15:03:40.182 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:40.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:40.215 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:39 smithi204 ceph-mon[87071]: pgmap v1215: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T15:03:40.216 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:03:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:03:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:03:40.223 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:39 smithi096 ceph-mon[98901]: pgmap v1215: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T15:03:40.224 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:39 smithi096 ceph-mon[102928]: pgmap v1215: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T15:03:40.234 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:40.284 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:40.321 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:40.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:40.423 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:40.461 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:40.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:40.563 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:40.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.742 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:40.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:40.843 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:40.878 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:40.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:40.980 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:41.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.016 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:41.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:41.027 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:41.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:40 smithi096 ceph-mon[98901]: pgmap v1216: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T15:03:41.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:40 smithi096 ceph-mon[102928]: pgmap v1216: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T15:03:41.077 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:03:41.077 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:03:41.107 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:03:41.108 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:03:41.117 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:41.130 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:03:41.130 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:03:41.153 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:03:41.154 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:03:41.155 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:41.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:41.165 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 51 16 91845 91829 112.51 70.875 0.0178968 0.00888356 2024-09-13T15:03:41.184 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:03:41.185 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:03:41.217 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:03:41.218 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:03:41.247 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:03:41.247 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:03:41.257 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:41.279 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:03:41.280 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:03:41.297 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:41.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:41.399 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:41.431 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:41.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:41.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:40 smithi204 ceph-mon[87071]: pgmap v1216: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T15:03:41.533 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:41.580 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:41.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:41.684 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:41.712 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:41.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:41.814 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:41.843 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:41.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:41.945 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:41.992 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:41.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:42.094 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:42.115 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:42.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:42.165 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 52 16 92867 92851 111.574 63.875 0.00454945 0.00895797 2024-09-13T15:03:42.216 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:42.254 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:42.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:42.309 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:41 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1434805498' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:42.310 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:03:42.310 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:41 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:03:42.310 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:41 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1434805498' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:42.310 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:03:42.310 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:41 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:03:42.356 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:42.382 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:42.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:42.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:41 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1434805498' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:42.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:03:42.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:41 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:03:42.484 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:42.516 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:42.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:42.619 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:42.649 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:42.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:42.750 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:42.783 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:42.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:42.885 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:42.922 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:42.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:43.024 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:43.063 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:43.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:43.164 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:43.166 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 53 16 94067 94051 110.884 75 0.00468311 0.00901399 2024-09-13T15:03:43.197 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:43.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:43.299 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:43.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:43 smithi096 ceph-mon[98901]: pgmap v1217: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T15:03:43.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:43 smithi096 ceph-mon[102928]: pgmap v1217: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T15:03:43.341 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:43.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:43.443 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:43.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:43 smithi204 ceph-mon[87071]: pgmap v1217: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T15:03:43.485 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:43.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:43.587 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:43.625 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:43.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:43.727 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:43.764 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:43.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:43.866 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:43.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.902 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:43.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:44.003 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:44.019 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T15:03:44.044 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:44.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:44.146 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:44.165 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 54 16 95227 95211 110.173 72.5 0.0119375 0.00907172 2024-09-13T15:03:44.182 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:44.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:44.284 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:44.319 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:44.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:44.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:44 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/699580459' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:44.322 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:44 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/699580459' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:44.347 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:03:44.421 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:44.453 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:44.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:44.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:44 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/699580459' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:44.554 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:44.596 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:44.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:44.698 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:44.740 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:44.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:44.841 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:44.880 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:44.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:44.982 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:45.011 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:45.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:45.112 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:45.147 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:45.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:45.166 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 55 16 96404 96388 109.507 73.5625 0.0161977 0.00912706 2024-09-13T15:03:45.249 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:45.283 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:45.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:45.285 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:45 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:45.285 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:45 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/699580459' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:45.285 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:45 smithi096 ceph-mon[98901]: osdmap e364: 8 total, 8 up, 7 in 2024-09-13T15:03:45.285 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:45 smithi096 ceph-mon[98901]: pgmap v1219: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T15:03:45.285 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:45 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:45.285 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:45 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/699580459' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:45.285 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:45 smithi096 ceph-mon[102928]: osdmap e364: 8 total, 8 up, 7 in 2024-09-13T15:03:45.285 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:45 smithi096 ceph-mon[102928]: pgmap v1219: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T15:03:45.385 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:45.417 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:45.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:45 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:45 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/699580459' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:45 smithi204 ceph-mon[87071]: osdmap e364: 8 total, 8 up, 7 in 2024-09-13T15:03:45.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:45 smithi204 ceph-mon[87071]: pgmap v1219: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T15:03:45.520 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:45.558 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:45.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:45.660 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:45.695 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:45.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:45.796 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:45.832 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:45.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:45.934 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:45.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.975 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:45.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:46.077 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:46.116 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:46.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:46.152 INFO:teuthology.orchestra.run.smithi096.stderr:set pool 1 pgp_num to 31 2024-09-13T15:03:46.168 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 56 16 97534 97518 108.809 70.625 0.0130679 0.00918619 2024-09-13T15:03:46.218 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:46.252 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:46.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:46.354 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:46.391 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:46.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:46.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:46 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3446235169' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T15:03:46.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:46 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T15:03:46.492 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:46.502 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:46 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3446235169' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T15:03:46.502 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:46 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T15:03:46.504 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:46 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3446235169' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T15:03:46.504 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:46 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T15:03:46.527 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:46.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:46.629 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:46.664 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:46.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:46.765 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:46.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.797 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:46.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:46.900 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:46.933 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:46.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:47.035 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:47.077 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:47.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:47.100 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:03:47.168 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 57 16 98730 98714 108.211 74.75 0.0167097 0.00923655 2024-09-13T15:03:47.179 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:47.219 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:47.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:47.321 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:47.367 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:47.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:47.468 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:47.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:47 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-13T15:03:47.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:47 smithi204 ceph-mon[87071]: osdmap e365: 8 total, 8 up, 7 in 2024-09-13T15:03:47.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:47 smithi204 ceph-mon[87071]: pgmap v1221: 119 pgs: 119 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 55 MiB/s wr, 877 op/s 2024-09-13T15:03:47.501 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:47.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:47.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:47 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-13T15:03:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:47 smithi096 ceph-mon[98901]: osdmap e365: 8 total, 8 up, 7 in 2024-09-13T15:03:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:47 smithi096 ceph-mon[98901]: pgmap v1221: 119 pgs: 119 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 55 MiB/s wr, 877 op/s 2024-09-13T15:03:47.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:47 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-13T15:03:47.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:47 smithi096 ceph-mon[102928]: osdmap e365: 8 total, 8 up, 7 in 2024-09-13T15:03:47.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:47 smithi096 ceph-mon[102928]: pgmap v1221: 119 pgs: 119 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 55 MiB/s wr, 877 op/s 2024-09-13T15:03:47.604 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:47.641 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:47.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:47.710 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:47.742 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:47.784 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:47.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:47.886 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:47.925 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:47.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:48.027 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:48.074 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:48.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:48.168 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 58 16 99836 99820 107.537 69.125 0.0124124 0.00929465 2024-09-13T15:03:48.176 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:48.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.212 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:48.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:48.261 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:03:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:03:47] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-09-13T15:03:48.313 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:48.351 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:48.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:48.453 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:48.489 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:48.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:48.591 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:48.631 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:48.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:48.733 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:48.769 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:48.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:48.871 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:48.913 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:48.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:48.986 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:49.015 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:49.020 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T15:03:49.021 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T15:03:49.021 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.021 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.36552.0:100799 10.3 e365/355 10:c23266c0:::benchmark_data_smithi096_288650_object100798:head v 365'6323, mlcod=365'6320)", 2024-09-13T15:03:49.021 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:03:49.011363+0000", 2024-09-13T15:03:49.021 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0061296279999999998, 2024-09-13T15:03:49.021 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0061554909999999999, 2024-09-13T15:03:49.021 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:03:49.021 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:03:49.021 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T15:03:49.021 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:03:49.021 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.022 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:03:49.022 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.011363+0000", 2024-09-13T15:03:49.022 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:03:49.022 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.022 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.022 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:03:49.022 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.011363+0000", 2024-09-13T15:03:49.022 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:03:49.022 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.022 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.022 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:03:49.022 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.011364+0000", 2024-09-13T15:03:49.022 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.5999999999999991e-07 2024-09-13T15:03:49.022 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.023 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.023 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:03:49.023 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.011393+0000", 2024-09-13T15:03:49.023 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.9824000000000001e-05 2024-09-13T15:03:49.023 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.023 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.023 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:03:49.023 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.011406+0000", 2024-09-13T15:03:49.023 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2975e-05 2024-09-13T15:03:49.023 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.023 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.023 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:03:49.023 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.011415+0000", 2024-09-13T15:03:49.023 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.208e-06 2024-09-13T15:03:49.023 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.024 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.024 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:03:49.024 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.013198+0000", 2024-09-13T15:03:49.024 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001782881 2024-09-13T15:03:49.024 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.024 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.024 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:03:49.024 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.013221+0000", 2024-09-13T15:03:49.024 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3003e-05 2024-09-13T15:03:49.024 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:03:49.024 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:03:49.024 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:03:49.024 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.024 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.025 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.36552.0:100787 10.5 e365/355 10:aee9760d:::benchmark_data_smithi096_288650_object100786:head v 365'6242, mlcod=365'6239)", 2024-09-13T15:03:49.025 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:03:49.015017+0000", 2024-09-13T15:03:49.025 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0024757239999999999, 2024-09-13T15:03:49.025 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.002562791, 2024-09-13T15:03:49.025 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:03:49.025 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:03:49.025 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T15:03:49.025 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:03:49.025 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.025 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:03:49.025 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.015017+0000", 2024-09-13T15:03:49.025 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:03:49.025 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.025 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.025 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:03:49.026 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.015017+0000", 2024-09-13T15:03:49.026 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:03:49.026 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.026 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.026 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:03:49.026 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.015018+0000", 2024-09-13T15:03:49.026 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.78e-07 2024-09-13T15:03:49.026 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.026 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.026 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:03:49.026 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.015040+0000", 2024-09-13T15:03:49.026 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.2237000000000001e-05 2024-09-13T15:03:49.026 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.026 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.026 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:03:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.015053+0000", 2024-09-13T15:03:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2775000000000001e-05 2024-09-13T15:03:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:03:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.015061+0000", 2024-09-13T15:03:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9829999999999996e-06 2024-09-13T15:03:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:03:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.015136+0000", 2024-09-13T15:03:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.5498000000000001e-05 2024-09-13T15:03:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:03:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.015155+0000", 2024-09-13T15:03:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8955999999999999e-05 2024-09-13T15:03:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:03:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:03:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:03:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.36552.0:100802 10.3 e365/355 10:cd3dad01:::benchmark_data_smithi096_288650_object100801:head v 365'6324, mlcod=365'6320)", 2024-09-13T15:03:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:03:49.016591+0000", 2024-09-13T15:03:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.00090154799999999995, 2024-09-13T15:03:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001039648, 2024-09-13T15:03:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:03:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:03:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T15:03:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:03:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:03:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.016591+0000", 2024-09-13T15:03:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:03:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:03:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.016591+0000", 2024-09-13T15:03:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:03:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:03:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.016592+0000", 2024-09-13T15:03:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6000000000000002e-07 2024-09-13T15:03:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:03:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.016668+0000", 2024-09-13T15:03:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.6842999999999996e-05 2024-09-13T15:03:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:03:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.016680+0000", 2024-09-13T15:03:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1086e-05 2024-09-13T15:03:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.031 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:03:49.031 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.016688+0000", 2024-09-13T15:03:49.031 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0530000000000007e-06 2024-09-13T15:03:49.031 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.031 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.031 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:03:49.031 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.016857+0000", 2024-09-13T15:03:49.031 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00016958899999999999 2024-09-13T15:03:49.031 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.031 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.032 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:03:49.032 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.016875+0000", 2024-09-13T15:03:49.032 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8219e-05 2024-09-13T15:03:49.032 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:03:49.032 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:03:49.032 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:03:49.032 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.032 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.032 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.36552.0:100803 10.f e365/355 10:fa2872a8:::benchmark_data_smithi096_288650_object100802:head v 365'6313, mlcod=365'6310)", 2024-09-13T15:03:49.032 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:03:49.016641+0000", 2024-09-13T15:03:49.032 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.00085152999999999997, 2024-09-13T15:03:49.032 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00103765, 2024-09-13T15:03:49.033 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:03:49.033 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:03:49.033 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T15:03:49.033 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:03:49.033 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.033 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:03:49.033 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.016641+0000", 2024-09-13T15:03:49.033 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:03:49.033 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.033 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.033 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:03:49.033 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.016641+0000", 2024-09-13T15:03:49.033 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:03:49.034 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.034 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.034 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:03:49.034 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.016642+0000", 2024-09-13T15:03:49.034 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.4099999999999998e-07 2024-09-13T15:03:49.034 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.034 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.034 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:03:49.034 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.016698+0000", 2024-09-13T15:03:49.034 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.613e-05 2024-09-13T15:03:49.034 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.034 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.034 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:03:49.034 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.016706+0000", 2024-09-13T15:03:49.035 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5450000000000003e-06 2024-09-13T15:03:49.035 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.035 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.035 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:03:49.035 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.016713+0000", 2024-09-13T15:03:49.035 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3389999999999998e-06 2024-09-13T15:03:49.035 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.035 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.035 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:03:49.035 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.016998+0000", 2024-09-13T15:03:49.035 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00028521500000000001 2024-09-13T15:03:49.035 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:03:49.035 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:03:49.035 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:03:49.036 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:03:49.017015+0000", 2024-09-13T15:03:49.036 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7640999999999999e-05 2024-09-13T15:03:49.036 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:03:49.036 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:03:49.036 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:03:49.036 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:03:49.036 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2024-09-13T15:03:49.036 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 4 2024-09-13T15:03:49.036 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T15:03:49.055 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:49.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:49.157 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:49.168 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 59 16 100973 100957 106.919 71.0625 0.038789 0.00934789 2024-09-13T15:03:49.199 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:49.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:49.300 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:49.336 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:49.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:49.438 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:49.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.478 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:49.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:49.574 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:49 smithi096 ceph-mon[98901]: pgmap v1222: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T15:03:49.575 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:49 smithi096 ceph-mon[102928]: pgmap v1222: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T15:03:49.578 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:49.617 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:49.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:49.718 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:49.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:49 smithi204 ceph-mon[87071]: pgmap v1222: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T15:03:49.750 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:49.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:49.852 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:49.884 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:49.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:49.986 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:50.018 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:50.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:50.120 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:50.164 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:50.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:50.168 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T15:03:50.168120+0000 min lat: 0.00132706 max lat: 0.187508 avg lat: 0.00939589 2024-09-13T15:03:50.168 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:03:50.168 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 60 16 102157 102141 106.37 74 0.0160133 0.00939589 2024-09-13T15:03:50.227 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T15:03:50.266 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:50.305 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:50.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:50.407 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:50.443 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:50.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:03:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:03:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:03:50.545 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:50.581 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:50.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:50.683 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:50.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.713 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:50.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:50.814 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:50.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.853 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:50.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:50.954 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:50.990 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:50.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:51.038 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:03:51.039 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:03:51.064 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:03:51.065 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:03:51.093 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:51.098 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:03:51.098 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:03:51.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.132 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:51.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:51.133 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:03:51.133 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:03:51.165 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:03:51.165 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:03:51.168 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 61 16 103316 103300 105.814 72.4375 0.00841024 0.00944615 2024-09-13T15:03:51.202 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:03:51.202 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:03:51.232 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:51.235 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:03:51.236 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:03:51.267 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:51.267 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:03:51.267 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:03:51.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:51.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:51 smithi096 ceph-mon[98901]: pgmap v1223: 119 pgs: 119 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 56 MiB/s wr, 894 op/s 2024-09-13T15:03:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:51 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1229233151' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:51 smithi096 ceph-mon[102928]: pgmap v1223: 119 pgs: 119 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 56 MiB/s wr, 894 op/s 2024-09-13T15:03:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:51.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:51 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1229233151' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:51.369 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:51.396 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:51.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:51.497 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:51.535 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:51.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:51.637 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:51.669 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:51.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:51 smithi204 ceph-mon[87071]: pgmap v1223: 119 pgs: 119 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 56 MiB/s wr, 894 op/s 2024-09-13T15:03:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:51 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1229233151' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:51.771 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:51.802 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:51.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:51.904 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:51.936 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:51.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:52.038 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:52.078 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:52.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:52.168 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 62 16 104498 104482 105.299 73.875 0.0102715 0.00949203 2024-09-13T15:03:52.180 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:52.215 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:52.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:52.316 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:52.360 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:52.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:52.462 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:52.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.496 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:52.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:52.597 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:52.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.645 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:52.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:52.713 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 6] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:03:52.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T15:03:52.745 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:52.797 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:52.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:52.899 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:52.951 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:52.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:53.053 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:53.090 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:53.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:53.169 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 63 16 105612 105596 104.732 69.625 0.00895813 0.00954369 2024-09-13T15:03:53.191 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:53.221 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:53.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:53.323 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:53.376 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:53.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:53.478 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:53.516 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:53.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:53 smithi096 ceph-mon[98901]: pgmap v1224: 119 pgs: 119 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-09-13T15:03:53.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:53 smithi096 ceph-mon[102928]: pgmap v1224: 119 pgs: 119 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-09-13T15:03:53.617 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:53.661 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:53.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:53 smithi204 ceph-mon[87071]: pgmap v1224: 119 pgs: 119 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-09-13T15:03:53.763 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:53.809 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:53.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:53.910 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:53.949 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:53.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:54.051 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:54.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.083 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:54.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:54.170 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 64 16 106787 106771 104.241 73.4375 0.0085553 0.00958791 2024-09-13T15:03:54.185 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:54.228 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:54.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:54.330 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:54.367 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:54.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:54.403 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:54.470 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:54.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.509 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:54.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:54.610 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:54.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.649 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:54.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:54.659 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:03:54.750 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:54.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.790 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:54.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:54.891 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:54.935 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:54.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:55.037 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:55.070 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:55.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:55.170 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 65 16 107907 107891 103.715 70 0.00483671 0.00963732 2024-09-13T15:03:55.173 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:55.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.210 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:55.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:55.262 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T15:03:55.311 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:55.344 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:55.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:55.446 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:55.475 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:55.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:55.551 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:55 smithi096 ceph-mon[98901]: pgmap v1225: 119 pgs: 119 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T15:03:55.552 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:55 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1996930263' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:03:55.552 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:55 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:03:55.553 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:55 smithi096 ceph-mon[102928]: pgmap v1225: 119 pgs: 119 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T15:03:55.553 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:55 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1996930263' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:03:55.553 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:55 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:03:55.577 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:55.612 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:55.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:55.714 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:55 smithi204 ceph-mon[87071]: pgmap v1225: 119 pgs: 119 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T15:03:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:55 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1996930263' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:03:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:55 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:03:55.747 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:55.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:55.849 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:55.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.877 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:55.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:55.978 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:56.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.014 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:56.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:56.115 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:56.154 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:56.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:56.170 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 66 16 109055 109039 103.23 71.75 0.00442224 0.00968238 2024-09-13T15:03:56.255 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:56.289 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:56.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:56.393 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:56.422 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:56.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:56.524 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:56.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.566 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:56.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:56.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:56 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:56.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:56 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:03:56.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:56 smithi096 ceph-mon[98901]: osdmap e366: 8 total, 8 up, 7 in 2024-09-13T15:03:56.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:56 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:56.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:56 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:03:56.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:56 smithi096 ceph-mon[102928]: osdmap e366: 8 total, 8 up, 7 in 2024-09-13T15:03:56.667 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:56.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.698 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:56.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:56.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:56 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:56.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:56 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:03:56.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:56 smithi204 ceph-mon[87071]: osdmap e366: 8 total, 8 up, 7 in 2024-09-13T15:03:56.799 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:56.835 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:56.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:56.937 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:56.972 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:56.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:57.074 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:57.112 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:57.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:57.170 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 67 16 110193 110177 102.751 71.125 0.00501918 0.00972776 2024-09-13T15:03:57.214 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:57.247 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:57.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:57.349 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:57.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.391 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:57.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:57.492 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:57.530 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:57.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:57.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:57 smithi096 ceph-mon[98901]: pgmap v1227: 119 pgs: 119 active+clean; 6.4 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T15:03:57.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:57 smithi096 ceph-mon[102928]: pgmap v1227: 119 pgs: 119 active+clean; 6.4 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T15:03:57.631 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:57.656 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:57.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:57.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:57 smithi204 ceph-mon[87071]: pgmap v1227: 119 pgs: 119 active+clean; 6.4 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T15:03:57.758 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:57.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.795 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:57.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:57.897 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:57.918 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:57.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:58.020 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:58.061 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:58.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:58.163 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:58.174 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 68 16 111224 111208 102.188 64.4375 0.0120106 0.00978134 2024-09-13T15:03:58.192 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:58.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:58.266 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:03:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:03:57] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-09-13T15:03:58.294 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:58.329 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:58.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:58.432 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:58.461 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:58.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:58.563 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:58.594 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:58.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:58.696 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:58.723 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:58.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:58.825 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:58.861 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:58.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:58.963 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:59.003 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:59.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:59.105 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:59.145 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:59.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:59.170 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 69 16 112373 112357 101.747 71.8125 0.00853239 0.00982372 2024-09-13T15:03:59.247 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:59.284 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:59.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:59.386 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:59.418 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:59.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:59.519 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:59.562 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:03:59.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:03:59 smithi096 ceph-mon[98901]: pgmap v1228: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T15:03:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:03:59 smithi096 ceph-mon[102928]: pgmap v1228: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T15:03:59.664 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:59.692 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:59.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:03:59 smithi204 ceph-mon[87071]: pgmap v1228: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T15:03:59.793 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:59.821 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:59.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:59.923 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:59.952 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:03:59.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:00.054 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:00.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.103 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:00.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:00.170 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 70 16 113508 113492 101.307 70.9375 0.00809052 0.00986639 2024-09-13T15:04:00.204 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:00.242 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:00.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:00.344 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:00.377 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:00.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:04:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:04:00.215Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:04:00.480 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:00.512 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:00.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:00.614 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:00.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.650 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:00.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:00.751 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:00.783 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:00.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:00.885 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:00.886 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:00.918 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:00.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:01.020 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:01.064 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:01.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:01.151 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:04:01.151 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:04:01.165 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:01.170 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 71 16 114629 114613 100.867 70.0625 0.0149639 0.00990958 2024-09-13T15:04:01.204 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:01.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:01.306 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:01.345 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:01.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:01.447 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:01.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.492 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:01.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:01.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:01.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.637 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:01.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:01.738 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:01.772 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:01.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:01.874 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:01.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.895 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:01.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:01.996 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:02.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.038 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:02.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:02.139 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:02.168 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:02.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:02.170 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 72 16 115698 115682 100.394 66.8125 0.00892042 0.00995664 2024-09-13T15:04:02.198 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:01 smithi096 ceph-mon[98901]: pgmap v1229: 119 pgs: 119 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T15:04:02.198 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:01 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:04:02.198 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:01 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:04:02.199 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:01 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:04:02.199 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:01 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:04:02.199 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:01 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:04:02.199 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:01 smithi096 ceph-mon[102928]: pgmap v1229: 119 pgs: 119 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T15:04:02.199 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:01 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:04:02.199 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:01 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:04:02.199 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:01 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:04:02.199 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:01 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:04:02.200 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:01 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:04:02.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:01 smithi204 ceph-mon[87071]: pgmap v1229: 119 pgs: 119 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T15:04:02.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:01 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:04:02.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:01 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:04:02.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:01 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:04:02.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:01 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:04:02.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:01 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:04:02.270 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:02.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.304 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:02.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:02.405 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:02.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.447 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:02.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:02.545 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T15:04:02.547 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:02.576 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:04:02.587 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:02.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:02.690 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:02.729 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:02.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:02.830 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:02.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.881 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:02.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:02.982 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:03.013 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:03.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:03.115 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:03.157 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:03.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:03.162 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:03.171 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 73 16 116838 116822 99.9944 71.25 0.0317572 0.00999638 2024-09-13T15:04:03.259 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:03.289 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:02 smithi096 ceph-mon[98901]: pgmap v1230: 119 pgs: 119 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T15:04:03.289 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:02 smithi096 ceph-mon[102928]: pgmap v1230: 119 pgs: 119 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T15:04:03.298 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:03.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:03.400 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:03.439 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:03.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:03.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:02 smithi204 ceph-mon[87071]: pgmap v1230: 119 pgs: 119 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T15:04:03.540 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:03.574 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:03.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:03.676 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:03.706 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:03.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:03.808 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:03.846 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:03.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:03.948 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:03.984 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:03.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:04.085 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:04.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.123 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:04.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:04.171 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 74 16 117973 117957 99.6017 70.9375 0.0150533 0.0100352 2024-09-13T15:04:04.225 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:04.254 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:04.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:04.356 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:04.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.391 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:04.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:04.492 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:04.510 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:04.525 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:04.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:04.626 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:04.655 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:04.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:04.757 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:04.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.793 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:04.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:04.894 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:04.926 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:04.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:05.028 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:05.066 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:05.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:05.168 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:05.171 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 75 16 119145 119129 99.2503 73.25 0.00451146 0.0100714 2024-09-13T15:04:05.204 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:05.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:05.306 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:05.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.348 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:05.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:05.449 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:05.489 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:05 smithi096 ceph-mon[102928]: pgmap v1231: 119 pgs: 119 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T15:04:05.490 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:05 smithi096 ceph-mon[98901]: pgmap v1231: 119 pgs: 119 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T15:04:05.491 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:05.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:05.593 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:05.636 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:05.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:05 smithi204 ceph-mon[87071]: pgmap v1231: 119 pgs: 119 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T15:04:05.737 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:05.769 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:05.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:05.871 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:05.907 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:05.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:06.009 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:06.043 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:06.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:06.148 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:06.174 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 76 16 120288 120272 98.8813 71.4375 0.0235285 0.0101082 2024-09-13T15:04:06.185 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:06.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:06.287 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:06.314 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:06.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:06.416 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:06.459 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:06.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:06.560 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:06.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:06.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:06.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.594 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:06.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:06.695 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:06.735 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:06.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:06.837 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:06.873 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:06.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:06.975 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:07.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.018 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:07.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:07.120 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:07.158 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:07.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:07.174 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 77 16 121453 121437 98.5427 72.8125 0.0124666 0.0101436 2024-09-13T15:04:07.204 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:04:07.204 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:04:07.240 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:04:07.241 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:04:07.260 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:07.281 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:04:07.282 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:04:07.307 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:07.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:07.317 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:04:07.318 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:04:07.355 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:04:07.355 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:04:07.390 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:04:07.390 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:04:07.409 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:07.429 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:04:07.429 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:04:07.447 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:07.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:07.459 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:07 smithi096 ceph-mon[98901]: pgmap v1232: 119 pgs: 119 active+clean; 7.1 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T15:04:07.459 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:07 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2930453069' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:07.459 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:07 smithi096 ceph-mon[102928]: pgmap v1232: 119 pgs: 119 active+clean; 7.1 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T15:04:07.459 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:07 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2930453069' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:07.459 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:04:07.460 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:04:07.549 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:07.591 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:07.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:07.693 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:07 smithi204 ceph-mon[87071]: pgmap v1232: 119 pgs: 119 active+clean; 7.1 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T15:04:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:07 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2930453069' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:07.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.723 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:07.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:07.824 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:07.865 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:07.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:07.967 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:08.006 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:08.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:08.055 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:04:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:04:07] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-09-13T15:04:08.108 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:08.144 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:08.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:08.174 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 78 16 122600 122584 98.1984 71.6875 0.00817306 0.0101794 2024-09-13T15:04:08.247 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:08.282 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T15:04:08.292 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:08.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:08.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:08 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3652959616' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:04:08.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:08 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:04:08.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:08 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3652959616' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:04:08.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:08 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:04:08.394 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:08.430 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:08.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:08.532 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:08.571 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:08.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:08.673 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:08.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.705 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:08.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:08.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:08 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3652959616' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:04:08.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:08 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:04:08.806 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:08.838 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:08.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:08.940 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:08.979 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:08.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:09.081 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:09.110 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:09.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:09.174 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 79 15 123753 123738 97.8683 72.125 0.00880073 0.0102134 2024-09-13T15:04:09.213 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:09.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.244 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:09.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:09.345 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:09.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.386 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:09.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:09.487 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:09.518 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:09.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:09 smithi096 ceph-mon[98901]: pgmap v1233: 119 pgs: 119 active+clean; 7.3 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-13T15:04:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:09 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:04:09.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:09 smithi096 ceph-mon[98901]: osdmap e367: 8 total, 8 up, 7 in 2024-09-13T15:04:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:09 smithi096 ceph-mon[102928]: pgmap v1233: 119 pgs: 119 active+clean; 7.3 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-13T15:04:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:09 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:04:09.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:09 smithi096 ceph-mon[102928]: osdmap e367: 8 total, 8 up, 7 in 2024-09-13T15:04:09.621 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:09.649 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:09.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:09 smithi204 ceph-mon[87071]: pgmap v1233: 119 pgs: 119 active+clean; 7.3 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-13T15:04:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:09 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:04:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:09 smithi204 ceph-mon[87071]: osdmap e367: 8 total, 8 up, 7 in 2024-09-13T15:04:09.751 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:09.791 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:09.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:09.893 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:09.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.930 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:09.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:10.031 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:10.063 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:10.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:10.102 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T15:04:10.165 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:10.174 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:2024-09-13T15:04:10.173835+0000 min lat: 0.00132706 max lat: 0.187508 avg lat: 0.0102481 2024-09-13T15:04:10.174 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:04:10.174 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 80 16 124892 124876 97.534 71.125 0.00483732 0.0102481 2024-09-13T15:04:10.211 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:10.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:10.313 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:10.347 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:10.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:10.450 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:04:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:04:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:04:10.486 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:10.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:10.558 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T15:04:10.559 INFO:teuthology.orchestra.run.smithi096.stdout: "size": 20, 2024-09-13T15:04:10.559 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 600, 2024-09-13T15:04:10.559 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T15:04:10.559 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.559 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:88714 9.0 e313/302 9:01cfaf0b:::benchmark_data_smithi096_256909_object88713:head v 313'5644, mlcod=313'5642)", 2024-09-13T15:04:10.559 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:58:42.928948+0000", 2024-09-13T15:04:10.559 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 327.47257709199999, 2024-09-13T15:04:10.559 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15184929999999999, 2024-09-13T15:04:10.559 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.559 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.559 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.559 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.560 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.560 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.560 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928948+0000", 2024-09-13T15:04:10.560 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.560 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.560 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.560 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.560 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928948+0000", 2024-09-13T15:04:10.560 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.560 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.560 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.560 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.560 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928949+0000", 2024-09-13T15:04:10.561 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.356e-06 2024-09-13T15:04:10.561 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.561 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.561 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.561 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.928994+0000", 2024-09-13T15:04:10.561 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4551000000000001e-05 2024-09-13T15:04:10.561 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.561 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.561 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.561 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.929009+0000", 2024-09-13T15:04:10.561 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5537e-05 2024-09-13T15:04:10.561 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.561 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.562 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.562 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.929018+0000", 2024-09-13T15:04:10.562 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.3699999999999995e-06 2024-09-13T15:04:10.562 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.562 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.562 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.562 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.929075+0000", 2024-09-13T15:04:10.562 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7500999999999997e-05 2024-09-13T15:04:10.563 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.563 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.563 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.563 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.929094+0000", 2024-09-13T15:04:10.563 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8757999999999999e-05 2024-09-13T15:04:10.563 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.563 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.563 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.563 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080745+0000", 2024-09-13T15:04:10.563 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15165108699999999 2024-09-13T15:04:10.563 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.564 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.564 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.564 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080797+0000", 2024-09-13T15:04:10.564 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2139999999999999e-05 2024-09-13T15:04:10.564 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.564 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.564 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.564 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.564 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.564 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:88718 9.2 e313/302 9:4ec1e6d3:::benchmark_data_smithi096_256909_object88717:head v 313'5533, mlcod=313'5531)", 2024-09-13T15:04:10.564 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:58:42.934191+0000", 2024-09-13T15:04:10.564 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 327.46733389000002, 2024-09-13T15:04:10.565 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.15776009699999999, 2024-09-13T15:04:10.565 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.565 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.565 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.565 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.565 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.565 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.565 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.934191+0000", 2024-09-13T15:04:10.565 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.565 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.565 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.565 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.565 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.934191+0000", 2024-09-13T15:04:10.565 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.566 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.566 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.566 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.566 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.934192+0000", 2024-09-13T15:04:10.566 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7199999999999997e-07 2024-09-13T15:04:10.566 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.566 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.566 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.566 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.934264+0000", 2024-09-13T15:04:10.566 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1389000000000004e-05 2024-09-13T15:04:10.566 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.566 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.566 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.566 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080739+0000", 2024-09-13T15:04:10.567 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.14647585900000001 2024-09-13T15:04:10.567 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.567 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.567 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.567 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080754+0000", 2024-09-13T15:04:10.567 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4899000000000001e-05 2024-09-13T15:04:10.567 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.567 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.567 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.567 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080821+0000", 2024-09-13T15:04:10.567 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7006000000000002e-05 2024-09-13T15:04:10.567 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.567 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.568 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.568 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080844+0000", 2024-09-13T15:04:10.568 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.2353000000000001e-05 2024-09-13T15:04:10.568 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.568 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.568 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.568 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.091909+0000", 2024-09-13T15:04:10.568 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011065643999999999 2024-09-13T15:04:10.568 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.568 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.568 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.568 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.091951+0000", 2024-09-13T15:04:10.568 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1974999999999997e-05 2024-09-13T15:04:10.569 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.569 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.569 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.569 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.569 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.569 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:88731 9.0 e313/302 9:088adfef:::benchmark_data_smithi096_256909_object88730:head v 313'5645, mlcod=313'5642)", 2024-09-13T15:04:10.569 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:58:42.941408+0000", 2024-09-13T15:04:10.569 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 327.460117212, 2024-09-13T15:04:10.569 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.14806446100000001, 2024-09-13T15:04:10.569 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.569 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.569 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.570 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.570 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.570 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.570 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.941408+0000", 2024-09-13T15:04:10.570 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.570 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.570 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.570 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.570 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.941408+0000", 2024-09-13T15:04:10.570 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.570 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.570 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.570 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.571 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.941409+0000", 2024-09-13T15:04:10.571 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2759999999999999e-06 2024-09-13T15:04:10.571 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.571 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.571 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.571 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:42.941459+0000", 2024-09-13T15:04:10.571 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9480000000000001e-05 2024-09-13T15:04:10.571 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.571 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.571 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.571 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080738+0000", 2024-09-13T15:04:10.571 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.139279245 2024-09-13T15:04:10.571 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.571 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.572 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.572 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080754+0000", 2024-09-13T15:04:10.572 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.6464e-05 2024-09-13T15:04:10.572 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.572 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.572 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.572 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080823+0000", 2024-09-13T15:04:10.572 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8757000000000006e-05 2024-09-13T15:04:10.572 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.572 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.572 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.572 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.080839+0000", 2024-09-13T15:04:10.572 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5909999999999998e-05 2024-09-13T15:04:10.573 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.573 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.573 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.573 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.089437+0000", 2024-09-13T15:04:10.573 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0085977299999999996 2024-09-13T15:04:10.573 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.573 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.573 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.573 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:58:43.089472+0000", 2024-09-13T15:04:10.573 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.5599000000000003e-05 2024-09-13T15:04:10.573 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.573 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.574 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.574 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.574 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.574 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:120877 9.f e320/316 9:f0ff22f2:::benchmark_data_smithi096_256909_object1469:head v 320'7500, mlcod=0'0)", 2024-09-13T15:04:10.574 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T14:59:23.437942+0000", 2024-09-13T15:04:10.574 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 286.963582589, 2024-09-13T15:04:10.574 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.088773064999999998, 2024-09-13T15:04:10.574 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.574 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.574 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.574 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.574 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.574 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.574 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.437942+0000", 2024-09-13T15:04:10.574 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.575 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.575 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.575 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.575 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.437942+0000", 2024-09-13T15:04:10.575 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.575 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.575 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.575 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.575 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.437943+0000", 2024-09-13T15:04:10.575 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.8100000000000002e-07 2024-09-13T15:04:10.575 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.576 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.576 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.576 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.437949+0000", 2024-09-13T15:04:10.576 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8150000000000002e-06 2024-09-13T15:04:10.576 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.576 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.576 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.576 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.437962+0000", 2024-09-13T15:04:10.576 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2673e-05 2024-09-13T15:04:10.576 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.576 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.576 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.576 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.437969+0000", 2024-09-13T15:04:10.576 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.627e-06 2024-09-13T15:04:10.577 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.577 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.577 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.577 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.489624+0000", 2024-09-13T15:04:10.577 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.051655049000000001 2024-09-13T15:04:10.577 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.577 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.577 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.577 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.489648+0000", 2024-09-13T15:04:10.577 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3193999999999998e-05 2024-09-13T15:04:10.577 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.577 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.577 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.577 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.526655+0000", 2024-09-13T15:04:10.578 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.037007557000000003 2024-09-13T15:04:10.578 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.578 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.578 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.578 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T14:59:23.526716+0000", 2024-09-13T15:04:10.578 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.0368999999999997e-05 2024-09-13T15:04:10.578 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.578 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.578 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.578 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.578 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.578 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:172468 9.f e328/316 9:fc4a529e:::benchmark_data_smithi096_256909_object53060:head v 328'10686, mlcod=0'0)", 2024-09-13T15:04:10.578 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:00:25.772205+0000", 2024-09-13T15:04:10.578 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 224.629320084, 2024-09-13T15:04:10.578 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.097298014000000002, 2024-09-13T15:04:10.579 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.579 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.579 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.579 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.579 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.579 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.579 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.772205+0000", 2024-09-13T15:04:10.579 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.579 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.579 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.579 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.579 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.772205+0000", 2024-09-13T15:04:10.579 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.579 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.579 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.580 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.580 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.772205+0000", 2024-09-13T15:04:10.580 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.67e-07 2024-09-13T15:04:10.580 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.580 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.580 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.580 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.772209+0000", 2024-09-13T15:04:10.580 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.3689999999999998e-06 2024-09-13T15:04:10.580 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.580 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.580 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.580 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.772215+0000", 2024-09-13T15:04:10.580 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2539999999999998e-06 2024-09-13T15:04:10.581 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.581 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.581 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.581 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.772219+0000", 2024-09-13T15:04:10.581 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1629999999999998e-06 2024-09-13T15:04:10.581 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.581 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.581 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.581 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.794100+0000", 2024-09-13T15:04:10.581 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.021881452999999999 2024-09-13T15:04:10.581 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.581 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.582 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.582 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.794124+0000", 2024-09-13T15:04:10.582 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3272e-05 2024-09-13T15:04:10.582 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.582 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.582 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.582 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.869433+0000", 2024-09-13T15:04:10.582 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.075309247999999995 2024-09-13T15:04:10.582 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.582 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.582 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.582 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.869503+0000", 2024-09-13T15:04:10.582 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.9987999999999995e-05 2024-09-13T15:04:10.582 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.583 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.583 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.583 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.583 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.583 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:172480 9.7 e328/316 9:ee5af5e1:::benchmark_data_smithi096_256909_object53072:head v 328'10755, mlcod=0'0)", 2024-09-13T15:04:10.583 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:00:25.776845+0000", 2024-09-13T15:04:10.583 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 224.62467986300001, 2024-09-13T15:04:10.583 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.102078725, 2024-09-13T15:04:10.583 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.583 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.583 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.583 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.583 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.584 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.584 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.776845+0000", 2024-09-13T15:04:10.584 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.584 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.584 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.584 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.584 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.776845+0000", 2024-09-13T15:04:10.584 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.584 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.584 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.584 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.584 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.776846+0000", 2024-09-13T15:04:10.584 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0589999999999999e-06 2024-09-13T15:04:10.585 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.585 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.585 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.585 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.776856+0000", 2024-09-13T15:04:10.585 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.4350000000000003e-06 2024-09-13T15:04:10.585 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.585 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.585 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.585 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.776870+0000", 2024-09-13T15:04:10.585 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.4715000000000001e-05 2024-09-13T15:04:10.585 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.585 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.585 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.586 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.776878+0000", 2024-09-13T15:04:10.586 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.7219999999999999e-06 2024-09-13T15:04:10.586 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.586 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.586 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.586 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.803244+0000", 2024-09-13T15:04:10.586 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.026365949999999999 2024-09-13T15:04:10.586 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.586 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.586 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.586 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.803270+0000", 2024-09-13T15:04:10.586 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.5522000000000001e-05 2024-09-13T15:04:10.586 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.587 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.587 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.587 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.878809+0000", 2024-09-13T15:04:10.587 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.075539236999999995 2024-09-13T15:04:10.587 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.587 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.587 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.587 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.878924+0000", 2024-09-13T15:04:10.587 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000115085 2024-09-13T15:04:10.588 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.588 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.588 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.588 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.588 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.588 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:172482 9.7 e328/316 9:e35ed075:::benchmark_data_smithi096_256909_object53074:head v 328'10756, mlcod=0'0)", 2024-09-13T15:04:10.588 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:00:25.779048+0000", 2024-09-13T15:04:10.588 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 224.622477355, 2024-09-13T15:04:10.588 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.099938185999999998, 2024-09-13T15:04:10.588 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.588 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.588 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.588 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.589 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.589 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.589 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.779048+0000", 2024-09-13T15:04:10.589 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.589 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.589 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.589 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.589 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.779048+0000", 2024-09-13T15:04:10.589 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.589 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.589 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.589 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.589 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.779049+0000", 2024-09-13T15:04:10.590 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.762e-06 2024-09-13T15:04:10.590 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.590 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.590 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.590 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.779062+0000", 2024-09-13T15:04:10.590 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.216e-05 2024-09-13T15:04:10.590 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.590 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.590 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.590 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.779080+0000", 2024-09-13T15:04:10.590 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8028000000000001e-05 2024-09-13T15:04:10.590 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.590 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.591 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.591 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.779088+0000", 2024-09-13T15:04:10.591 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6910000000000003e-06 2024-09-13T15:04:10.591 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.591 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.591 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.591 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.812008+0000", 2024-09-13T15:04:10.591 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.032919655999999999 2024-09-13T15:04:10.591 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.591 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.591 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.591 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.812030+0000", 2024-09-13T15:04:10.591 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.2189999999999999e-05 2024-09-13T15:04:10.591 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.592 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.592 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.592 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.878929+0000", 2024-09-13T15:04:10.592 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.066898843 2024-09-13T15:04:10.592 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.592 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.592 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.592 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:00:25.878986+0000", 2024-09-13T15:04:10.592 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6855999999999999e-05 2024-09-13T15:04:10.593 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.593 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.593 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.593 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.593 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.593 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:208635 9.7 e333/316 9:ea165d7c:::benchmark_data_smithi096_256909_object89227:head v 333'12963, mlcod=0'0)", 2024-09-13T15:04:10.593 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:10.872785+0000", 2024-09-13T15:04:10.593 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 179.52873990099999, 2024-09-13T15:04:10.593 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.095886740999999998, 2024-09-13T15:04:10.593 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.593 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.593 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.593 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.593 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.594 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.594 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.872785+0000", 2024-09-13T15:04:10.594 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.594 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.594 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.594 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.594 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.872785+0000", 2024-09-13T15:04:10.594 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.594 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.594 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.594 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.594 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.872786+0000", 2024-09-13T15:04:10.594 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8800000000000002e-07 2024-09-13T15:04:10.594 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.595 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.595 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.595 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.872791+0000", 2024-09-13T15:04:10.595 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9540000000000003e-06 2024-09-13T15:04:10.595 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.595 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.595 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.595 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.872801+0000", 2024-09-13T15:04:10.595 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0462000000000001e-05 2024-09-13T15:04:10.595 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.595 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.595 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.595 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.872808+0000", 2024-09-13T15:04:10.595 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1670000000000002e-06 2024-09-13T15:04:10.596 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.596 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.596 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.596 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.905006+0000", 2024-09-13T15:04:10.596 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.032197279000000002 2024-09-13T15:04:10.596 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.596 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.596 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.596 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.905033+0000", 2024-09-13T15:04:10.596 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7475999999999999e-05 2024-09-13T15:04:10.596 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.596 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.596 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.597 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.968622+0000", 2024-09-13T15:04:10.597 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.063589151999999996 2024-09-13T15:04:10.597 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.597 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.597 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.597 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.968672+0000", 2024-09-13T15:04:10.597 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.9563e-05 2024-09-13T15:04:10.597 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.597 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.597 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.597 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.597 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.597 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:208636 9.7 e333/316 9:eca12262:::benchmark_data_smithi096_256909_object89228:head v 333'12964, mlcod=0'0)", 2024-09-13T15:04:10.597 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:10.876514+0000", 2024-09-13T15:04:10.598 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 179.52501083600001, 2024-09-13T15:04:10.598 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.108805451, 2024-09-13T15:04:10.598 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.598 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.598 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.598 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.598 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.598 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.598 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.876514+0000", 2024-09-13T15:04:10.598 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.598 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.598 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.598 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.598 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.876514+0000", 2024-09-13T15:04:10.599 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.599 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.599 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.599 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.599 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.876515+0000", 2024-09-13T15:04:10.599 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.1699999999999997e-07 2024-09-13T15:04:10.599 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.599 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.599 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.599 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.876522+0000", 2024-09-13T15:04:10.599 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5220000000000004e-06 2024-09-13T15:04:10.599 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.599 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.599 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.600 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.876533+0000", 2024-09-13T15:04:10.600 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1195e-05 2024-09-13T15:04:10.600 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.600 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.600 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.600 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.876541+0000", 2024-09-13T15:04:10.600 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1110000000000008e-06 2024-09-13T15:04:10.600 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.600 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.600 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.600 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.912916+0000", 2024-09-13T15:04:10.600 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.036374778000000003 2024-09-13T15:04:10.600 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.601 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.601 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.601 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.912940+0000", 2024-09-13T15:04:10.601 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3856999999999999e-05 2024-09-13T15:04:10.601 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.601 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.601 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.601 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.985194+0000", 2024-09-13T15:04:10.601 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.072253917000000001 2024-09-13T15:04:10.601 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.601 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.601 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.601 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.985320+0000", 2024-09-13T15:04:10.601 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012615400000000001 2024-09-13T15:04:10.602 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.602 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.602 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.602 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.602 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.602 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:208642 9.7 e333/316 9:ea4e6dda:::benchmark_data_smithi096_256909_object89234:head v 333'12965, mlcod=0'0)", 2024-09-13T15:04:10.602 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:10.886728+0000", 2024-09-13T15:04:10.602 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 179.514796881, 2024-09-13T15:04:10.602 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.098854715999999995, 2024-09-13T15:04:10.602 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.602 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.602 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.602 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.603 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.603 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.603 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.886728+0000", 2024-09-13T15:04:10.603 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.603 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.603 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.603 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.603 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.886728+0000", 2024-09-13T15:04:10.603 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.603 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.603 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.603 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.603 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.886730+0000", 2024-09-13T15:04:10.604 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.364e-06 2024-09-13T15:04:10.604 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.604 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.604 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.604 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.886742+0000", 2024-09-13T15:04:10.604 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2275e-05 2024-09-13T15:04:10.604 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.604 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.604 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.604 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.886763+0000", 2024-09-13T15:04:10.604 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1438999999999999e-05 2024-09-13T15:04:10.604 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.604 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.605 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.605 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.886774+0000", 2024-09-13T15:04:10.605 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1066e-05 2024-09-13T15:04:10.605 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.605 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.605 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.605 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.925122+0000", 2024-09-13T15:04:10.605 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.038348078000000001 2024-09-13T15:04:10.605 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.605 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.605 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.605 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.925175+0000", 2024-09-13T15:04:10.605 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2797999999999998e-05 2024-09-13T15:04:10.605 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.606 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.606 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.606 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.985445+0000", 2024-09-13T15:04:10.606 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.060270160000000003 2024-09-13T15:04:10.606 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.606 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.606 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.606 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.985583+0000", 2024-09-13T15:04:10.606 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013753600000000001 2024-09-13T15:04:10.606 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.606 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.606 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.606 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.607 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.607 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:208637 9.f e333/316 9:f4dbcd12:::benchmark_data_smithi096_256909_object89229:head v 333'12953, mlcod=0'0)", 2024-09-13T15:04:10.607 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:10.887373+0000", 2024-09-13T15:04:10.607 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 179.51415245999999, 2024-09-13T15:04:10.607 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.098070253999999996, 2024-09-13T15:04:10.607 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.607 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.607 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.607 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.607 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.607 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.607 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.887373+0000", 2024-09-13T15:04:10.607 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.607 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.607 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.608 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.608 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.887373+0000", 2024-09-13T15:04:10.608 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.608 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.608 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.608 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.608 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.887373+0000", 2024-09-13T15:04:10.608 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.23e-07 2024-09-13T15:04:10.608 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.608 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.608 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.608 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.887380+0000", 2024-09-13T15:04:10.608 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2210000000000002e-06 2024-09-13T15:04:10.609 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.609 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.609 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.609 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.887396+0000", 2024-09-13T15:04:10.609 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5956000000000001e-05 2024-09-13T15:04:10.609 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.609 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.609 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.609 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.887404+0000", 2024-09-13T15:04:10.609 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0639999999999994e-06 2024-09-13T15:04:10.609 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.609 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.609 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.610 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.925038+0000", 2024-09-13T15:04:10.610 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.037633948 2024-09-13T15:04:10.610 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.610 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.610 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.610 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.925071+0000", 2024-09-13T15:04:10.610 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.3563999999999999e-05 2024-09-13T15:04:10.610 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.610 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.610 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.610 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.985323+0000", 2024-09-13T15:04:10.610 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.060251954000000003 2024-09-13T15:04:10.610 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.611 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.611 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.611 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:10.985443+0000", 2024-09-13T15:04:10.611 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011972399999999999 2024-09-13T15:04:10.611 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.611 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.611 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.611 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.611 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.611 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:212985 9.7 e339/316 9:e507835e:::benchmark_data_smithi096_256909_object93577:head v 339'13260, mlcod=0'0)", 2024-09-13T15:04:10.611 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:17.378972+0000", 2024-09-13T15:04:10.611 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 173.02255257300001, 2024-09-13T15:04:10.611 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.097035513000000004, 2024-09-13T15:04:10.611 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.612 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.612 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.612 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.612 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.612 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.612 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:17.378972+0000", 2024-09-13T15:04:10.612 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.612 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.612 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.612 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.612 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:17.378972+0000", 2024-09-13T15:04:10.612 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.612 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.613 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.613 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.613 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:17.378974+0000", 2024-09-13T15:04:10.613 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1710000000000001e-06 2024-09-13T15:04:10.613 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.613 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.613 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.613 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:17.378981+0000", 2024-09-13T15:04:10.613 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.8770000000000004e-06 2024-09-13T15:04:10.613 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.613 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.613 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.613 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:17.378992+0000", 2024-09-13T15:04:10.614 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1800000000000001e-05 2024-09-13T15:04:10.614 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.614 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.614 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.614 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:17.379000+0000", 2024-09-13T15:04:10.614 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.0649999999999999e-06 2024-09-13T15:04:10.614 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.614 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.614 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.614 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:17.405478+0000", 2024-09-13T15:04:10.614 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.026477125000000001 2024-09-13T15:04:10.614 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.614 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.614 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.614 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:17.405501+0000", 2024-09-13T15:04:10.615 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3247e-05 2024-09-13T15:04:10.615 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.615 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.615 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.615 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:17.475955+0000", 2024-09-13T15:04:10.615 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.070454483999999998 2024-09-13T15:04:10.615 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.615 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.615 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.615 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:17.476008+0000", 2024-09-13T15:04:10.615 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2744000000000002e-05 2024-09-13T15:04:10.615 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.616 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.616 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.616 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.616 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.616 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:226250 9.f e343/316 9:f34758ea:::benchmark_data_smithi096_256909_object106842:head v 343'14055, mlcod=0'0)", 2024-09-13T15:04:10.616 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:34.415570+0000", 2024-09-13T15:04:10.616 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 155.985955217, 2024-09-13T15:04:10.616 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.091296199999999994, 2024-09-13T15:04:10.616 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.616 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.616 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.616 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.616 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.616 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.617 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.415570+0000", 2024-09-13T15:04:10.617 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.617 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.617 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.617 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.617 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.415570+0000", 2024-09-13T15:04:10.617 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.617 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.617 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.617 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.617 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.415570+0000", 2024-09-13T15:04:10.617 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.89e-07 2024-09-13T15:04:10.617 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.617 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.618 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.618 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.415574+0000", 2024-09-13T15:04:10.618 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.0339999999999998e-06 2024-09-13T15:04:10.618 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.618 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.618 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.618 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.415585+0000", 2024-09-13T15:04:10.618 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0601999999999999e-05 2024-09-13T15:04:10.618 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.618 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.618 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.618 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.415590+0000", 2024-09-13T15:04:10.618 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0769999999999998e-06 2024-09-13T15:04:10.618 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.619 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.619 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.619 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.447541+0000", 2024-09-13T15:04:10.619 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.031950781999999997 2024-09-13T15:04:10.619 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.619 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.619 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.619 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.447608+0000", 2024-09-13T15:04:10.619 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7031000000000002e-05 2024-09-13T15:04:10.619 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.619 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.619 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.619 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.506772+0000", 2024-09-13T15:04:10.620 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.059164339000000003 2024-09-13T15:04:10.620 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.620 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.620 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.620 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.506866+0000", 2024-09-13T15:04:10.620 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3845999999999996e-05 2024-09-13T15:04:10.620 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.620 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.620 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.620 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.620 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.620 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:226391 9.7 e343/316 9:ec1c5f15:::benchmark_data_smithi096_256909_object106983:head v 343'14088, mlcod=0'0)", 2024-09-13T15:04:10.620 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:34.591033+0000", 2024-09-13T15:04:10.620 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 155.810492246, 2024-09-13T15:04:10.621 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.089313060999999999, 2024-09-13T15:04:10.621 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.621 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.621 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.621 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.621 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.621 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.621 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.591033+0000", 2024-09-13T15:04:10.621 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.621 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.621 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.621 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.621 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.591033+0000", 2024-09-13T15:04:10.622 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.622 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.622 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.622 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.622 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.591033+0000", 2024-09-13T15:04:10.622 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.41e-07 2024-09-13T15:04:10.622 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.622 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.622 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.622 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.591035+0000", 2024-09-13T15:04:10.622 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.3039999999999999e-06 2024-09-13T15:04:10.622 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.623 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.623 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.623 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.591039+0000", 2024-09-13T15:04:10.623 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.9770000000000002e-06 2024-09-13T15:04:10.623 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.623 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.623 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.623 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.591042+0000", 2024-09-13T15:04:10.623 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.8399999999999999e-06 2024-09-13T15:04:10.623 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.623 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.623 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.624 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.614952+0000", 2024-09-13T15:04:10.624 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.023909492000000001 2024-09-13T15:04:10.624 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.624 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.624 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.624 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.614987+0000", 2024-09-13T15:04:10.624 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4934999999999997e-05 2024-09-13T15:04:10.624 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.624 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.624 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.624 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.680290+0000", 2024-09-13T15:04:10.624 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.065303204000000004 2024-09-13T15:04:10.624 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.625 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.625 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.625 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:34.680346+0000", 2024-09-13T15:04:10.625 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6067999999999999e-05 2024-09-13T15:04:10.625 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.625 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.625 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.625 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.625 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.625 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:228152 9.f e343/316 9:fd3c4718:::benchmark_data_smithi096_256909_object108744:head v 343'14178, mlcod=0'0)", 2024-09-13T15:04:10.625 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:36.476290+0000", 2024-09-13T15:04:10.625 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 153.92523540299999, 2024-09-13T15:04:10.625 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.102153258, 2024-09-13T15:04:10.626 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.626 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.626 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.626 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.626 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.626 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.626 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.476290+0000", 2024-09-13T15:04:10.626 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.626 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.626 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.626 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.626 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.476290+0000", 2024-09-13T15:04:10.626 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.626 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.627 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.627 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.627 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.476290+0000", 2024-09-13T15:04:10.627 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3e-07 2024-09-13T15:04:10.627 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.627 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.627 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.627 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.476300+0000", 2024-09-13T15:04:10.627 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.1640000000000007e-06 2024-09-13T15:04:10.627 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.627 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.627 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.627 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.476313+0000", 2024-09-13T15:04:10.627 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3696000000000001e-05 2024-09-13T15:04:10.628 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.628 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.628 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.628 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.476320+0000", 2024-09-13T15:04:10.628 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.0829999999999998e-06 2024-09-13T15:04:10.628 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.628 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.628 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.628 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.505344+0000", 2024-09-13T15:04:10.628 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.029023203000000001 2024-09-13T15:04:10.628 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.628 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.628 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.629 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.505371+0000", 2024-09-13T15:04:10.629 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7253000000000001e-05 2024-09-13T15:04:10.629 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.629 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.629 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.629 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.578379+0000", 2024-09-13T15:04:10.629 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.073008726999999995 2024-09-13T15:04:10.629 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.629 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.629 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.629 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.578443+0000", 2024-09-13T15:04:10.629 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.3402e-05 2024-09-13T15:04:10.629 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.629 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.630 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.630 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.630 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.630 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:228171 9.f e343/316 9:fc418b8b:::benchmark_data_smithi096_256909_object108763:head v 343'14179, mlcod=0'0)", 2024-09-13T15:04:10.630 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:36.490289+0000", 2024-09-13T15:04:10.630 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 153.91123628400001, 2024-09-13T15:04:10.630 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.106449418, 2024-09-13T15:04:10.630 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.630 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.630 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.630 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.630 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.630 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.631 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.490289+0000", 2024-09-13T15:04:10.631 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.631 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.631 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.631 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.631 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.490289+0000", 2024-09-13T15:04:10.631 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.631 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.631 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.631 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.631 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.490290+0000", 2024-09-13T15:04:10.631 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.33e-07 2024-09-13T15:04:10.631 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.631 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.631 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.632 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.490299+0000", 2024-09-13T15:04:10.632 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.5210000000000008e-06 2024-09-13T15:04:10.632 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.632 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.632 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.632 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.490315+0000", 2024-09-13T15:04:10.632 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.5617999999999999e-05 2024-09-13T15:04:10.632 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.632 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.632 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.632 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.490322+0000", 2024-09-13T15:04:10.633 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.7970000000000003e-06 2024-09-13T15:04:10.633 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.633 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.633 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.633 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.513830+0000", 2024-09-13T15:04:10.633 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.023508775999999999 2024-09-13T15:04:10.633 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.633 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.633 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.633 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.513856+0000", 2024-09-13T15:04:10.633 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.5599999999999999e-05 2024-09-13T15:04:10.633 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.634 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.634 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.634 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.596659+0000", 2024-09-13T15:04:10.634 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.082802641999999996 2024-09-13T15:04:10.634 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.634 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.634 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.634 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.596738+0000", 2024-09-13T15:04:10.634 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.9530999999999995e-05 2024-09-13T15:04:10.634 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.634 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.634 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.634 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.635 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.635 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:228175 9.7 e343/316 9:ebe51f28:::benchmark_data_smithi096_256909_object108767:head v 343'14203, mlcod=0'0)", 2024-09-13T15:04:10.635 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:36.493703+0000", 2024-09-13T15:04:10.635 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 153.90782222000001, 2024-09-13T15:04:10.635 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.094346804000000006, 2024-09-13T15:04:10.635 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.635 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.635 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.635 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.635 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.635 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.635 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.493703+0000", 2024-09-13T15:04:10.635 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.636 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.636 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.636 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.636 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.493703+0000", 2024-09-13T15:04:10.636 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.636 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.636 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.636 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.636 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.493704+0000", 2024-09-13T15:04:10.636 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.6799999999999999e-07 2024-09-13T15:04:10.636 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.636 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.636 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.637 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.493712+0000", 2024-09-13T15:04:10.637 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6770000000000001e-06 2024-09-13T15:04:10.637 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.637 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.637 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.637 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.493725+0000", 2024-09-13T15:04:10.637 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2904e-05 2024-09-13T15:04:10.637 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.637 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.637 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.637 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.493733+0000", 2024-09-13T15:04:10.637 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.605e-06 2024-09-13T15:04:10.637 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.637 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.638 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.638 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.513801+0000", 2024-09-13T15:04:10.638 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.020068671999999999 2024-09-13T15:04:10.638 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.638 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.638 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.638 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.513826+0000", 2024-09-13T15:04:10.638 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.4369999999999999e-05 2024-09-13T15:04:10.638 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.638 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.638 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.638 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.587958+0000", 2024-09-13T15:04:10.638 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.074132036999999998 2024-09-13T15:04:10.639 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.639 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.639 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.639 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.588050+0000", 2024-09-13T15:04:10.639 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.1770999999999999e-05 2024-09-13T15:04:10.639 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.639 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.639 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.639 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.639 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.639 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:228172 9.f e343/316 9:f3f3ef04:::benchmark_data_smithi096_256909_object108764:head v 343'14180, mlcod=0'0)", 2024-09-13T15:04:10.639 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:36.493942+0000", 2024-09-13T15:04:10.639 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 153.907583219, 2024-09-13T15:04:10.640 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.102850238, 2024-09-13T15:04:10.640 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.640 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.640 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.640 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.640 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.640 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.640 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.493942+0000", 2024-09-13T15:04:10.640 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.640 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.640 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.640 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.640 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.493942+0000", 2024-09-13T15:04:10.641 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.641 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.641 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.641 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.641 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.493942+0000", 2024-09-13T15:04:10.641 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3300000000000003e-07 2024-09-13T15:04:10.641 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.641 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.641 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.641 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.493946+0000", 2024-09-13T15:04:10.641 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.3390000000000001e-06 2024-09-13T15:04:10.641 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.641 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.641 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.642 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.493956+0000", 2024-09-13T15:04:10.642 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0386e-05 2024-09-13T15:04:10.642 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.642 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.642 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.642 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.493961+0000", 2024-09-13T15:04:10.642 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4029999999999998e-06 2024-09-13T15:04:10.642 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.642 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.642 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.642 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.523913+0000", 2024-09-13T15:04:10.642 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.029951825000000001 2024-09-13T15:04:10.642 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.642 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.643 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.643 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.523943+0000", 2024-09-13T15:04:10.643 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.9896999999999999e-05 2024-09-13T15:04:10.643 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.643 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.643 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.643 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.596742+0000", 2024-09-13T15:04:10.643 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.072798935999999995 2024-09-13T15:04:10.643 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.643 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.643 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.643 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.596792+0000", 2024-09-13T15:04:10.643 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0018999999999997e-05 2024-09-13T15:04:10.643 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.644 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.644 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.644 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.644 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.644 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:228177 9.f e343/316 9:f050b6b1:::benchmark_data_smithi096_256909_object108769:head v 343'14181, mlcod=0'0)", 2024-09-13T15:04:10.644 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:36.497870+0000", 2024-09-13T15:04:10.644 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 153.90365490600001, 2024-09-13T15:04:10.644 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.098952666999999994, 2024-09-13T15:04:10.644 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.644 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.644 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.644 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.644 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.645 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.645 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.497870+0000", 2024-09-13T15:04:10.645 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.645 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.645 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.645 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.645 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.497870+0000", 2024-09-13T15:04:10.645 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.645 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.645 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.645 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.645 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.497871+0000", 2024-09-13T15:04:10.645 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.048e-06 2024-09-13T15:04:10.646 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.646 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.646 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.646 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.497876+0000", 2024-09-13T15:04:10.646 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.3989999999999997e-06 2024-09-13T15:04:10.646 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.646 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.646 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.646 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.497887+0000", 2024-09-13T15:04:10.646 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1191e-05 2024-09-13T15:04:10.646 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.646 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.646 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.647 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.497893+0000", 2024-09-13T15:04:10.647 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5250000000000002e-06 2024-09-13T15:04:10.647 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.647 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.647 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.647 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.533275+0000", 2024-09-13T15:04:10.647 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.035381675000000001 2024-09-13T15:04:10.647 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.647 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.647 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.647 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.533296+0000", 2024-09-13T15:04:10.647 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1047999999999999e-05 2024-09-13T15:04:10.648 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.648 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.648 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.648 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.596795+0000", 2024-09-13T15:04:10.648 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.063498924999999998 2024-09-13T15:04:10.648 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.648 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.648 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.648 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.596823+0000", 2024-09-13T15:04:10.648 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.7855999999999999e-05 2024-09-13T15:04:10.648 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.648 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.648 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.649 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.649 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.649 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.46040.0:228183 9.7 e343/316 9:efd78936:::benchmark_data_smithi096_256909_object108775:head v 343'14204, mlcod=0'0)", 2024-09-13T15:04:10.649 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:01:36.505627+0000", 2024-09-13T15:04:10.649 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 153.89589785499999, 2024-09-13T15:04:10.649 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.091225547000000004, 2024-09-13T15:04:10.649 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:04:10.649 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:04:10.649 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:10.649 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:04:10.649 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.649 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:04:10.649 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.505627+0000", 2024-09-13T15:04:10.650 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.650 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.650 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.650 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:04:10.650 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.505627+0000", 2024-09-13T15:04:10.650 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:04:10.650 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.650 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.650 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:04:10.650 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.505628+0000", 2024-09-13T15:04:10.650 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.2600000000000001e-07 2024-09-13T15:04:10.650 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.650 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.651 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:04:10.651 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.505634+0000", 2024-09-13T15:04:10.651 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.2979999999999997e-06 2024-09-13T15:04:10.651 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.651 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.651 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:04:10.651 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.505644+0000", 2024-09-13T15:04:10.651 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.6979999999999993e-06 2024-09-13T15:04:10.651 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.651 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.651 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:04:10.651 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.505652+0000", 2024-09-13T15:04:10.651 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.4000000000000003e-06 2024-09-13T15:04:10.652 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.652 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.652 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:04:10.652 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.541502+0000", 2024-09-13T15:04:10.652 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.035850013 2024-09-13T15:04:10.652 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.652 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.652 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:04:10.652 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.541523+0000", 2024-09-13T15:04:10.652 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.1505000000000002e-05 2024-09-13T15:04:10.652 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.652 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.652 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "commit_sent", 2024-09-13T15:04:10.653 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.596827+0000", 2024-09-13T15:04:10.653 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.055303883999999998 2024-09-13T15:04:10.653 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:04:10.653 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:04:10.653 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "done", 2024-09-13T15:04:10.653 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:01:36.596853+0000", 2024-09-13T15:04:10.653 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.5823e-05 2024-09-13T15:04:10.653 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.653 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.653 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.653 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:04:10.653 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:04:10.653 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T15:04:10.654 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:10.684 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:10.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:10.759 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:10.787 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:10.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.836 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:10.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:10.937 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:10.982 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:10.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:11.004 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:04:11.083 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:11.123 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:11.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:11.174 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 81 16 126026 126010 97.2048 70.875 0.0159544 0.0102837 2024-09-13T15:04:11.225 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:11.260 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:11.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:11.362 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:11.393 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:11.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:11.496 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:11.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.533 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:11.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:11.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:11 smithi096 ceph-mon[98901]: pgmap v1235: 119 pgs: 119 active+clean; 7.4 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T15:04:11.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:11 smithi096 ceph-mon[102928]: pgmap v1235: 119 pgs: 119 active+clean; 7.4 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T15:04:11.634 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:11.665 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:11.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:11.696 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T15:04:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:11 smithi204 ceph-mon[87071]: pgmap v1235: 119 pgs: 119 active+clean; 7.4 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T15:04:11.767 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:11.793 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:11.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:11.895 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:11.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.941 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:11.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:12.042 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:12.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.091 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:12.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:12.130 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:12.174 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 82 16 127087 127071 96.8281 66.3125 0.0176388 0.0103232 2024-09-13T15:04:12.191 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:12.236 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:12.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:12.338 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:12.363 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:12.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:12.465 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:12.511 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:12.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:12.613 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:12.657 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:12.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:12.759 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:12.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.790 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:12.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:12.891 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:12.928 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:12.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:13.029 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:13.062 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:13.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:13.104 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:13.164 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:13.174 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 83 16 128194 128178 96.495 69.1875 0.0162318 0.0103587 2024-09-13T15:04:13.192 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:13.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:13.294 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:13.329 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:13.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:13.431 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:13.468 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:13.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:13.471 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:13 smithi096 ceph-mon[98901]: pgmap v1236: 119 pgs: 119 active+clean; 7.7 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T15:04:13.472 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:13 smithi096 ceph-mon[102928]: pgmap v1236: 119 pgs: 119 active+clean; 7.7 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T15:04:13.541 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:13 smithi204 ceph-mon[87071]: pgmap v1236: 119 pgs: 119 active+clean; 7.7 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T15:04:13.570 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:13.614 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:13.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:13.622 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:13.716 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:13.759 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:13.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:13.861 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:13.900 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:13.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:14.002 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:14.042 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:14.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:14.144 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:14.174 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 84 16 129287 129271 96.1595 68.3125 0.0168306 0.0103953 2024-09-13T15:04:14.184 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:14.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:14.286 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:14.312 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T15:04:14.329 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:14.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:14.339 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:04:14.339 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T15:04:14.339 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.339 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.36552.0:129434 10.2 e368/355 10:40733862:::benchmark_data_smithi096_288650_object129433:head v 368'7976, mlcod=367'7974)", 2024-09-13T15:04:14.339 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:14.329103+0000", 2024-09-13T15:04:14.340 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0079631240000000002, 2024-09-13T15:04:14.340 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0079899940000000003, 2024-09-13T15:04:14.340 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:14.340 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:04:14.340 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:14.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.340 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:14.340 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.329103+0000", 2024-09-13T15:04:14.340 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:14.340 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.340 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.340 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:14.340 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.329103+0000", 2024-09-13T15:04:14.341 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:14.341 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.341 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.341 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:14.341 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.329103+0000", 2024-09-13T15:04:14.341 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:14.341 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.341 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.341 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:14.341 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.329135+0000", 2024-09-13T15:04:14.341 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2378000000000001e-05 2024-09-13T15:04:14.341 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.341 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.341 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:14.341 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.329145+0000", 2024-09-13T15:04:14.342 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.022e-05 2024-09-13T15:04:14.342 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.342 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.342 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:14.342 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.329151+0000", 2024-09-13T15:04:14.342 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8699999999999997e-06 2024-09-13T15:04:14.342 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.342 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.342 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:14.342 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.329283+0000", 2024-09-13T15:04:14.342 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013244600000000001 2024-09-13T15:04:14.342 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.342 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.343 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:14.343 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.329303+0000", 2024-09-13T15:04:14.343 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.9743999999999999e-05 2024-09-13T15:04:14.343 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:14.343 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:14.343 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:14.343 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.343 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.343 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.36552.0:129436 10.c e368/355 10:3f63a931:::benchmark_data_smithi096_288650_object129435:head v 368'8075, mlcod=367'8073)", 2024-09-13T15:04:14.343 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:14.330164+0000", 2024-09-13T15:04:14.343 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0069015860000000004, 2024-09-13T15:04:14.343 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0069962039999999998, 2024-09-13T15:04:14.343 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:14.343 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:04:14.344 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:14.344 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.344 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:14.344 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.330164+0000", 2024-09-13T15:04:14.344 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:14.344 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.344 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.344 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:14.344 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.330164+0000", 2024-09-13T15:04:14.344 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:14.344 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.344 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.344 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:14.344 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.330164+0000", 2024-09-13T15:04:14.345 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:04:14.345 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.345 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.345 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:14.345 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.330187+0000", 2024-09-13T15:04:14.345 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.3606999999999999e-05 2024-09-13T15:04:14.345 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.345 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.345 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:14.345 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.330201+0000", 2024-09-13T15:04:14.346 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4294000000000001e-05 2024-09-13T15:04:14.346 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.346 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.346 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:14.346 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.330207+0000", 2024-09-13T15:04:14.346 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5140000000000001e-06 2024-09-13T15:04:14.346 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.346 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.346 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:14.346 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.330262+0000", 2024-09-13T15:04:14.346 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4922e-05 2024-09-13T15:04:14.346 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.346 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.346 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:14.346 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.330280+0000", 2024-09-13T15:04:14.347 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8043e-05 2024-09-13T15:04:14.347 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:14.347 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:14.347 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:14.347 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.347 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.347 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.36552.0:129439 10.4 e368/355 10:2e876ac5:::benchmark_data_smithi096_288650_object129438:head v 368'7975, mlcod=367'7973)", 2024-09-13T15:04:14.347 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:14.331943+0000", 2024-09-13T15:04:14.347 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0051225680000000001, 2024-09-13T15:04:14.347 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0052772610000000001, 2024-09-13T15:04:14.347 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:14.347 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:04:14.347 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:14.348 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.348 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:14.348 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.331943+0000", 2024-09-13T15:04:14.348 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:14.348 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.348 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.348 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:14.348 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.331943+0000", 2024-09-13T15:04:14.348 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:14.348 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.348 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.348 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:14.348 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.331943+0000", 2024-09-13T15:04:14.349 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:04:14.349 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.349 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.349 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:14.349 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.332007+0000", 2024-09-13T15:04:14.349 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4220999999999994e-05 2024-09-13T15:04:14.349 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.349 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.349 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:14.349 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.332017+0000", 2024-09-13T15:04:14.349 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0445000000000001e-05 2024-09-13T15:04:14.349 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.349 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.350 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:14.350 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.332023+0000", 2024-09-13T15:04:14.350 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4790000000000004e-06 2024-09-13T15:04:14.350 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.350 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.350 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:14.350 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.332069+0000", 2024-09-13T15:04:14.350 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6363999999999999e-05 2024-09-13T15:04:14.350 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.350 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.350 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:14.350 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.332087+0000", 2024-09-13T15:04:14.350 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7798000000000001e-05 2024-09-13T15:04:14.350 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:14.351 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:14.351 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:14.351 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.351 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.351 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.36552.0:129440 10.3 e368/355 10:cfa5a590:::benchmark_data_smithi096_288650_object129439:head v 368'8223, mlcod=367'8221)", 2024-09-13T15:04:14.351 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:14.333012+0000", 2024-09-13T15:04:14.351 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0040542080000000001, 2024-09-13T15:04:14.351 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0042703250000000002, 2024-09-13T15:04:14.351 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:14.351 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:04:14.351 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:14.351 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.351 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:14.352 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.333012+0000", 2024-09-13T15:04:14.352 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:14.352 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.352 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.352 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:14.352 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.333012+0000", 2024-09-13T15:04:14.352 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:14.352 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.352 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.352 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:14.352 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.333011+0000", 2024-09-13T15:04:14.352 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:14.352 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.353 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.353 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:14.353 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.333034+0000", 2024-09-13T15:04:14.353 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2796999999999999e-05 2024-09-13T15:04:14.353 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.353 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.353 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:14.353 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.333042+0000", 2024-09-13T15:04:14.353 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8529999999999992e-06 2024-09-13T15:04:14.353 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.353 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.353 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:14.353 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.333047+0000", 2024-09-13T15:04:14.353 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6179999999999997e-06 2024-09-13T15:04:14.354 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.354 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.354 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:14.354 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.334284+0000", 2024-09-13T15:04:14.354 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0012371249999999999 2024-09-13T15:04:14.354 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.354 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.354 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:14.354 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.334306+0000", 2024-09-13T15:04:14.354 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2789000000000002e-05 2024-09-13T15:04:14.354 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:14.354 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:14.354 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:14.355 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.355 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.355 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.36552.0:129437 10.2 e368/355 10:4cbe6b88:::benchmark_data_smithi096_288650_object129436:head v 368'7977, mlcod=367'7974)", 2024-09-13T15:04:14.355 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:14.334095+0000", 2024-09-13T15:04:14.355 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0029706709999999998, 2024-09-13T15:04:14.355 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0032406790000000002, 2024-09-13T15:04:14.355 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:14.355 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:04:14.355 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:14.355 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.355 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:14.355 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.334095+0000", 2024-09-13T15:04:14.355 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:14.355 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.356 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.356 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:14.356 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.334095+0000", 2024-09-13T15:04:14.356 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:14.356 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.356 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.356 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:14.356 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.334095+0000", 2024-09-13T15:04:14.356 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:14.356 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.356 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.356 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:14.356 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.334141+0000", 2024-09-13T15:04:14.357 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6032999999999998e-05 2024-09-13T15:04:14.357 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.357 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.357 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:14.357 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.334154+0000", 2024-09-13T15:04:14.357 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3869000000000001e-05 2024-09-13T15:04:14.357 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.357 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.357 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:14.357 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.334160+0000", 2024-09-13T15:04:14.357 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0170000000000004e-06 2024-09-13T15:04:14.357 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.357 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.358 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:14.358 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.335072+0000", 2024-09-13T15:04:14.358 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00091244899999999996 2024-09-13T15:04:14.358 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.358 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.358 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:14.358 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.335089+0000", 2024-09-13T15:04:14.358 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7005999999999999e-05 2024-09-13T15:04:14.358 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:14.358 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:14.358 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:14.358 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.358 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.359 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop(client.36552.0:129435 10.d e368/355 10:bd561565:::benchmark_data_smithi096_288650_object129434:head v 368'8164, mlcod=367'8161)", 2024-09-13T15:04:14.359 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:14.335074+0000", 2024-09-13T15:04:14.359 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0019918050000000001, 2024-09-13T15:04:14.359 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0023132539999999998, 2024-09-13T15:04:14.359 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:14.359 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:04:14.359 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:14.359 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.359 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:14.359 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.335074+0000", 2024-09-13T15:04:14.359 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:14.359 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.359 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.360 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:14.360 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.335074+0000", 2024-09-13T15:04:14.360 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:14.360 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.360 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.360 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:14.360 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.335074+0000", 2024-09-13T15:04:14.360 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:14.360 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.360 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.360 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:14.360 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.335095+0000", 2024-09-13T15:04:14.360 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1052000000000001e-05 2024-09-13T15:04:14.361 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.361 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.361 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:14.361 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.335105+0000", 2024-09-13T15:04:14.361 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0665999999999999e-05 2024-09-13T15:04:14.361 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.361 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.361 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:14.361 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.335112+0000", 2024-09-13T15:04:14.361 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0570000000000004e-06 2024-09-13T15:04:14.361 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.361 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.362 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:14.362 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.335891+0000", 2024-09-13T15:04:14.362 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00077840299999999999 2024-09-13T15:04:14.362 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:14.362 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:14.362 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:14.362 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:14.335908+0000", 2024-09-13T15:04:14.362 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7798000000000001e-05 2024-09-13T15:04:14.362 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:14.363 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:14.363 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:14.363 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:14.363 INFO:teuthology.orchestra.run.smithi204.stdout: ], 2024-09-13T15:04:14.363 INFO:teuthology.orchestra.run.smithi204.stdout: "num_ops": 6 2024-09-13T15:04:14.363 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:04:14.431 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:14.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.460 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:14.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:14.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:14 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2850674431' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:14 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.561 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:14.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:14 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2850674431' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:14 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:14 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2850674431' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:14 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.602 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:14.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:14.704 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:14.743 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:14.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:14.844 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:14.876 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:14.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:14.978 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:15.006 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:15.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:15.108 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:15.110 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T15:04:15.135 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:15.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:15.174 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 85 16 130373 130357 95.8267 67.875 0.0134695 0.0104311 2024-09-13T15:04:15.237 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:15.289 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:15.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:15.392 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:15.421 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:15.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:15.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:15 smithi204 ceph-mon[87071]: pgmap v1237: 119 pgs: 119 active+clean; 7.7 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T15:04:15.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:15 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:04:15.470 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:15 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T15:04:15.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:15 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:15.471 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:15 smithi204 ceph-mon[87071]: osdmap e368: 8 total, 8 up, 7 in 2024-09-13T15:04:15.520 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:15 smithi096 ceph-mon[102928]: pgmap v1237: 119 pgs: 119 active+clean; 7.7 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T15:04:15.520 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:15 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:04:15.521 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:15 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T15:04:15.521 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:15 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:15.521 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:15 smithi096 ceph-mon[102928]: osdmap e368: 8 total, 8 up, 7 in 2024-09-13T15:04:15.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:15 smithi096 ceph-mon[98901]: pgmap v1237: 119 pgs: 119 active+clean; 7.7 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T15:04:15.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:15 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:04:15.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:15 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T15:04:15.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:15 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:15.521 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:15 smithi096 ceph-mon[98901]: osdmap e368: 8 total, 8 up, 7 in 2024-09-13T15:04:15.523 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:15.552 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:15.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:15.580 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:15.654 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:15.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.683 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:15.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:15.784 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:15.815 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:15.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:15.917 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:15.960 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:16.062 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:16.107 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:16.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:16.175 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 86 16 131443 131427 95.4901 66.875 0.0139569 0.0104679 2024-09-13T15:04:16.209 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:16.241 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:16.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:16.343 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:16.380 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:16.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:16.485 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:16.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.516 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:16.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:16.617 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:16.649 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:16.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:16.751 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:16.784 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:16.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:16.886 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:16.913 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:16.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:17.015 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:17.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.065 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:17.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:17.167 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:17.175 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 87 16 132560 132544 95.1949 69.8125 0.0146955 0.0105001 2024-09-13T15:04:17.208 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:17.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:17.310 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:17.340 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:17.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:17.442 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:17.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:17 smithi204 ceph-mon[87071]: pgmap v1239: 119 pgs: 119 active+clean; 7.8 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 56 MiB/s wr, 889 op/s 2024-09-13T15:04:17.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.491 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:17.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:17.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:17 smithi096 ceph-mon[98901]: pgmap v1239: 119 pgs: 119 active+clean; 7.8 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 56 MiB/s wr, 889 op/s 2024-09-13T15:04:17.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:17 smithi096 ceph-mon[102928]: pgmap v1239: 119 pgs: 119 active+clean; 7.8 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 56 MiB/s wr, 889 op/s 2024-09-13T15:04:17.592 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:17.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.628 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:17.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:17.729 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:17.765 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:17.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:17.867 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:17.898 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:17.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:17.910 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:04:17.910 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:04:17.937 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:04:17.937 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:04:17.963 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:04:17.963 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:04:17.990 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:04:17.991 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:04:18.000 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:18.002 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:04:18.032 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:04:18.032 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:04:18.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.034 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:18.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:18.069 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:04:18.070 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:04:18.104 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:04:18.104 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:04:18.135 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:18.156 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:04:18.156 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:04:18.157 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:04:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:04:17] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-09-13T15:04:18.163 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:04:18.163 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:04:18.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.176 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:18.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:18.177 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 88 16 133619 133603 94.8641 66.1875 0.0140477 0.010537 2024-09-13T15:04:18.277 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:18.317 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:18.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:18.419 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:18.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.465 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:18.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:18.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:18 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3603719619' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:18.498 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:18 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3603719619' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:18.499 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:18 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3603719619' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:18.566 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:18.612 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:18.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:18.714 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:18.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.757 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:18.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:18.858 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:18.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.889 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:18.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:18.990 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:19.031 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:19.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:19.133 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:19.166 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:19.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:19.171 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:19.176 INFO:tasks.radosbench.radosbench.0.smithi096.stdout: 89 16 134650 134634 94.5222 64.4375 0.0282061 0.0105752 2024-09-13T15:04:19.268 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:19.306 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:19.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:19.350 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T15:04:19.408 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:19.442 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:19.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:19.544 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:19.565 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:19.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:19.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:19 smithi096 ceph-mon[98901]: pgmap v1240: 119 pgs: 119 active+clean; 8.0 GiB data, 29 GiB used, 596 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T15:04:19.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:19 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1034523979' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:04:19.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:19 smithi096 ceph-mon[102928]: pgmap v1240: 119 pgs: 119 active+clean; 8.0 GiB data, 29 GiB used, 596 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T15:04:19.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:19 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1034523979' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:04:19.666 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:19.700 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:19.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:19.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:19 smithi204 ceph-mon[87071]: pgmap v1240: 119 pgs: 119 active+clean; 8.0 GiB data, 29 GiB used, 596 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T15:04:19.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:19 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1034523979' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:04:19.802 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:19.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.841 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:19.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:19.943 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:19.978 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:19.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:20.080 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:20.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.115 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:20.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:20.176 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Total time run: 90.0093 2024-09-13T15:04:20.176 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Total writes made: 135689 2024-09-13T15:04:20.176 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Write size: 65536 2024-09-13T15:04:20.176 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Object size: 65536 2024-09-13T15:04:20.176 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Bandwidth (MB/sec): 94.2188 2024-09-13T15:04:20.176 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev Bandwidth: 39.5784 2024-09-13T15:04:20.177 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max bandwidth (MB/sec): 254.125 2024-09-13T15:04:20.177 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min bandwidth (MB/sec): 63.875 2024-09-13T15:04:20.177 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Average IOPS: 1507 2024-09-13T15:04:20.177 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev IOPS: 633.255 2024-09-13T15:04:20.177 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max IOPS: 4066 2024-09-13T15:04:20.177 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min IOPS: 1022 2024-09-13T15:04:20.177 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Average Latency(s): 0.0106097 2024-09-13T15:04:20.177 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Stddev Latency(s): 0.00681485 2024-09-13T15:04:20.177 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Max latency(s): 0.187508 2024-09-13T15:04:20.177 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Min latency(s): 0.00132706 2024-09-13T15:04:20.182 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T15:04:20.216 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:20.234 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T15:04:20.248 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:20.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:20.350 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:20.396 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:20.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:20.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:20 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1034523979' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:04:20.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:20 smithi204 ceph-mon[87071]: osdmap e369: 8 total, 8 up, 7 in 2024-09-13T15:04:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:04:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:04:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:04:20.499 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:20.532 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:20.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:20.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:20 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1034523979' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:04:20.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:20 smithi096 ceph-mon[98901]: osdmap e369: 8 total, 8 up, 7 in 2024-09-13T15:04:20.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:20 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1034523979' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:04:20.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:20 smithi096 ceph-mon[102928]: osdmap e369: 8 total, 8 up, 7 in 2024-09-13T15:04:20.634 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:20.664 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:20.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:20.766 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:20.796 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:20.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:20.897 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:20.914 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T15:04:20.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.930 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:20.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:21.030 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:21.057 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:21.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:21.064 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:04:21.158 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:21.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.189 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:21.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:21.290 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:21.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.342 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:21.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:21.385 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:21.443 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:21.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.480 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:21.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:21.561 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:21.581 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:21.607 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:21.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:21.709 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:21.740 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:21.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:21.842 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:21.863 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:21.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:21.965 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:21.986 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:21.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:22.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:21 smithi096 ceph-mon[98901]: pgmap v1242: 119 pgs: 119 active+clean; 8.1 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 53 MiB/s wr, 843 op/s 2024-09-13T15:04:22.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:04:22.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:21 smithi096 ceph-mon[102928]: pgmap v1242: 119 pgs: 119 active+clean; 8.1 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 53 MiB/s wr, 843 op/s 2024-09-13T15:04:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:04:22.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:22.088 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:22.113 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:22.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:22.215 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:21 smithi204 ceph-mon[87071]: pgmap v1242: 119 pgs: 119 active+clean; 8.1 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 53 MiB/s wr, 843 op/s 2024-09-13T15:04:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:04:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:22.251 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:22.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:22.353 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:22.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.395 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:22.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:22.495 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:22.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.530 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:22.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:22.631 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:22.660 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:22.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:22.761 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:22.811 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:22.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:22.870 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T15:04:22.913 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:22.943 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:22.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:23.045 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:23.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.084 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:23.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:23.185 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:23.213 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:23.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:23.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:22 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2497598799' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:23.315 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:23.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:22 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2497598799' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:23.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:22 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2497598799' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:23.353 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:23.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:23.455 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:23.490 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:23.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:23.592 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:23.619 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:23.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:23.663 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:23.721 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:23.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.748 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:23.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:23.849 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:23.875 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:23.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:23.977 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:24.012 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:24.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:24.021 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:23 smithi096 ceph-mon[98901]: pgmap v1243: 119 pgs: 119 active+clean; 8.2 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 127 B/s rd, 72 MiB/s wr, 1.30k op/s 2024-09-13T15:04:24.021 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:23 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:24.021 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:23 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2497598799' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:04:24.021 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:23 smithi096 ceph-mon[98901]: osdmap e370: 8 total, 8 up, 7 in 2024-09-13T15:04:24.022 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:23 smithi096 ceph-mon[102928]: pgmap v1243: 119 pgs: 119 active+clean; 8.2 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 127 B/s rd, 72 MiB/s wr, 1.30k op/s 2024-09-13T15:04:24.022 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:23 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:24.022 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:23 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2497598799' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:04:24.022 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:23 smithi096 ceph-mon[102928]: osdmap e370: 8 total, 8 up, 7 in 2024-09-13T15:04:24.113 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:24.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.146 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:24.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:24.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:23 smithi204 ceph-mon[87071]: pgmap v1243: 119 pgs: 119 active+clean; 8.2 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 127 B/s rd, 72 MiB/s wr, 1.30k op/s 2024-09-13T15:04:24.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:23 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:24.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:23 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2497598799' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:04:24.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:23 smithi204 ceph-mon[87071]: osdmap e370: 8 total, 8 up, 7 in 2024-09-13T15:04:24.220 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:04:24.220 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:04:24.247 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:04:24.247 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:04:24.247 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:24.274 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:04:24.274 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:04:24.285 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:24.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:24.302 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:04:24.302 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:04:24.332 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:04:24.332 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:04:24.375 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:04:24.375 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:04:24.387 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:24.417 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:04:24.418 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:04:24.421 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:24.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:24.448 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:04:24.448 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:04:24.522 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:24.556 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:24.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:24.658 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:24.690 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:24.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:24.792 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:24.833 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:24.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:24.935 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:24.962 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:24.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:25.065 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:25.095 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:25.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:25.197 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:25.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:24 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3415609703' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:25.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.236 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:25.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:25.295 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:24 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3415609703' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:25.296 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:24 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3415609703' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:25.337 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:25.364 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:25.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:25.466 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:25.510 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:25.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:25.612 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:25.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.658 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:25.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:25.759 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:25.801 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:25.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:25.902 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:25.927 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:25 smithi204 ceph-mon[87071]: pgmap v1245: 119 pgs: 119 active+clean; 8.2 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 127 B/s rd, 65 MiB/s wr, 1.21k op/s 2024-09-13T15:04:25.954 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:25.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:25.985 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:04:25.985 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T15:04:25.985 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T15:04:25.985 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T15:04:25.985 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.985 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45749.0:214052 8.2 8:4759b8cd:::benchmark_data_smithi096_227542_object88657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-09-13T15:04:25.985 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:56:30.955593+0000", 2024-09-13T15:04:25.985 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 475.02459445699998, 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.142896519, 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45749", 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 214052 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955593+0000", 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.986 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:25.987 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955593+0000", 2024-09-13T15:04:25.987 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:25.987 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.987 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.987 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:25.987 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955592+0000", 2024-09-13T15:04:25.987 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:25.987 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.987 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.987 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:25.987 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955599+0000", 2024-09-13T15:04:25.987 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5920000000000003e-06 2024-09-13T15:04:25.987 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.987 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.988 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:25.988 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955614+0000", 2024-09-13T15:04:25.988 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4346e-05 2024-09-13T15:04:25.988 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.988 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.988 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:25.988 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955620+0000", 2024-09-13T15:04:25.988 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2339999999999999e-06 2024-09-13T15:04:25.988 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.988 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.988 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:25.988 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955663+0000", 2024-09-13T15:04:25.988 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3448999999999997e-05 2024-09-13T15:04:25.989 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.989 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.989 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:25.989 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955791+0000", 2024-09-13T15:04:25.989 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012775500000000001 2024-09-13T15:04:25.989 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.989 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.989 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:04:25.989 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.955936+0000", 2024-09-13T15:04:25.989 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014486100000000001 2024-09-13T15:04:25.989 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.990 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.990 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:25.990 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.960011+0000", 2024-09-13T15:04:25.990 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.004075431 2024-09-13T15:04:25.990 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.990 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.990 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:25.990 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:30.964835+0000", 2024-09-13T15:04:25.990 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0048238700000000001 2024-09-13T15:04:25.990 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.990 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.990 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:25.990 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:31.098384+0000", 2024-09-13T15:04:25.991 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.13354844499999999 2024-09-13T15:04:25.991 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.991 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.991 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:25.991 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:31.098445+0000", 2024-09-13T15:04:25.991 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1470000000000006e-05 2024-09-13T15:04:25.991 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.992 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.992 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:25.992 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:31.098490+0000", 2024-09-13T15:04:25.992 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4700000000000002e-05 2024-09-13T15:04:25.992 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:25.993 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:25.993 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:25.993 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.993 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.993 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45749.0:238705 8.2 8:41667315:::benchmark_data_smithi096_227542_object113310:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T15:04:25.993 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:56:56.922681+0000", 2024-09-13T15:04:25.994 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 449.05750667500001, 2024-09-13T15:04:25.994 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.23348930900000001, 2024-09-13T15:04:25.994 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:25.994 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:25.994 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:25.994 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45749", 2024-09-13T15:04:25.994 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T15:04:25.994 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 238705 2024-09-13T15:04:25.994 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.994 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:25.994 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.994 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:25.994 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922681+0000", 2024-09-13T15:04:25.994 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:25.995 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.995 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.995 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:25.995 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922681+0000", 2024-09-13T15:04:25.995 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:25.995 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.995 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.995 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:25.995 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922680+0000", 2024-09-13T15:04:25.995 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:04:25.995 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.995 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.995 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:25.995 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922686+0000", 2024-09-13T15:04:25.995 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6210000000000001e-06 2024-09-13T15:04:25.996 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.996 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.996 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:25.996 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922696+0000", 2024-09-13T15:04:25.996 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0036e-05 2024-09-13T15:04:25.996 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.996 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.996 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:25.996 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922701+0000", 2024-09-13T15:04:25.996 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7219999999999996e-06 2024-09-13T15:04:25.996 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.996 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.997 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:25.997 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922792+0000", 2024-09-13T15:04:25.997 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1013000000000004e-05 2024-09-13T15:04:25.997 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.997 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.997 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:25.997 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.922916+0000", 2024-09-13T15:04:25.997 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012372800000000001 2024-09-13T15:04:25.997 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.997 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.997 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:04:25.997 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.923042+0000", 2024-09-13T15:04:25.997 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000125974 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.926845+0000", 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0038031290000000001 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.927035+0000", 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00019018600000000001 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.156091+0000", 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.22905610600000001 2024-09-13T15:04:25.998 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.999 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.999 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:25.999 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.156129+0000", 2024-09-13T15:04:25.999 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7125000000000001e-05 2024-09-13T15:04:25.999 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.999 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.999 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:25.999 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.156170+0000", 2024-09-13T15:04:25.999 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1739999999999997e-05 2024-09-13T15:04:25.999 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:25.999 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:25.999 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:25.999 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:25.999 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:25.999 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45749.0:238719 8.2 8:45fb8e4f:::benchmark_data_smithi096_227542_object113324:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T15:04:26.000 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:56:56.937206+0000", 2024-09-13T15:04:26.000 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 449.04298200300002, 2024-09-13T15:04:26.000 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.23103955300000001, 2024-09-13T15:04:26.000 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.000 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.000 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.000 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45749", 2024-09-13T15:04:26.000 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T15:04:26.000 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 238719 2024-09-13T15:04:26.000 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.000 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.000 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.000 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.001 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937206+0000", 2024-09-13T15:04:26.001 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.001 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.001 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.001 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.001 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937206+0000", 2024-09-13T15:04:26.001 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.001 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.001 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.001 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.001 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937204+0000", 2024-09-13T15:04:26.001 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:26.001 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.001 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.001 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.002 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937212+0000", 2024-09-13T15:04:26.002 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7729999999999999e-06 2024-09-13T15:04:26.002 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.002 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.002 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.002 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937226+0000", 2024-09-13T15:04:26.002 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.434e-05 2024-09-13T15:04:26.002 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.002 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.002 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.002 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937231+0000", 2024-09-13T15:04:26.002 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7489999999999998e-06 2024-09-13T15:04:26.002 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.002 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.003 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.003 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937287+0000", 2024-09-13T15:04:26.003 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5995999999999997e-05 2024-09-13T15:04:26.003 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.003 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.003 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.003 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937384+0000", 2024-09-13T15:04:26.003 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6912999999999999e-05 2024-09-13T15:04:26.003 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.003 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.004 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:04:26.004 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.937508+0000", 2024-09-13T15:04:26.004 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000124578 2024-09-13T15:04:26.004 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.004 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.004 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.004 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.940699+0000", 2024-09-13T15:04:26.004 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00319024 2024-09-13T15:04:26.005 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.005 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.005 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.005 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.941399+0000", 2024-09-13T15:04:26.005 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00070025800000000004 2024-09-13T15:04:26.005 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.005 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.005 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.005 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.168173+0000", 2024-09-13T15:04:26.005 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.22677360699999999 2024-09-13T15:04:26.005 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.005 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.006 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.006 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.168206+0000", 2024-09-13T15:04:26.006 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3843000000000001e-05 2024-09-13T15:04:26.006 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.006 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.006 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.006 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.168245+0000", 2024-09-13T15:04:26.006 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8862999999999997e-05 2024-09-13T15:04:26.006 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.006 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.006 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.006 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.006 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.006 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.45749.0:238724 8.2 8:4164a1e4:::benchmark_data_smithi096_227542_object113329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-13T15:04:26.006 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:56:56.942313+0000", 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 449.03787458900001, 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.226006811, 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.45749", 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/595268574", 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 238724 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942313+0000", 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.007 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.008 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.008 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942313+0000", 2024-09-13T15:04:26.008 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.008 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.008 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.008 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.008 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942312+0000", 2024-09-13T15:04:26.008 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:26.008 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.008 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.008 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.008 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942318+0000", 2024-09-13T15:04:26.008 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4649999999999999e-06 2024-09-13T15:04:26.008 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.008 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.009 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.009 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942332+0000", 2024-09-13T15:04:26.009 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3596e-05 2024-09-13T15:04:26.009 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.009 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.009 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.009 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942337+0000", 2024-09-13T15:04:26.009 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4530000000000001e-06 2024-09-13T15:04:26.009 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.009 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.009 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.009 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942374+0000", 2024-09-13T15:04:26.010 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6749999999999999e-05 2024-09-13T15:04:26.010 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.010 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.010 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.010 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942461+0000", 2024-09-13T15:04:26.010 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.6698000000000002e-05 2024-09-13T15:04:26.010 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.010 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.010 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:04:26.010 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.942534+0000", 2024-09-13T15:04:26.010 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.3245999999999996e-05 2024-09-13T15:04:26.010 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.010 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.010 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.011 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.945803+0000", 2024-09-13T15:04:26.011 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003268703 2024-09-13T15:04:26.011 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.011 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.011 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.011 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:56.946325+0000", 2024-09-13T15:04:26.011 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00052236999999999997 2024-09-13T15:04:26.011 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.011 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.011 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.011 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.168274+0000", 2024-09-13T15:04:26.011 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.221949167 2024-09-13T15:04:26.011 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.011 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.011 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.012 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.168301+0000", 2024-09-13T15:04:26.012 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.6923999999999998e-05 2024-09-13T15:04:26.012 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.012 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.012 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.012 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:56:57.168320+0000", 2024-09-13T15:04:26.012 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8893e-05 2024-09-13T15:04:26.012 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.012 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.012 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.012 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.012 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.012 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:113285 9.c 9:3711587a:::benchmark_data_smithi096_256909_object113284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-09-13T15:04:26.012 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:11.892740+0000", 2024-09-13T15:04:26.012 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.08744779800003, 2024-09-13T15:04:26.013 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.025987193, 2024-09-13T15:04:26.013 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.013 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.013 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.013 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.013 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.013 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 113285 2024-09-13T15:04:26.013 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.013 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.013 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.013 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.013 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.892740+0000", 2024-09-13T15:04:26.013 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.014 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.014 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.014 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.014 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.892740+0000", 2024-09-13T15:04:26.014 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.014 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.014 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.014 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.014 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.892738+0000", 2024-09-13T15:04:26.014 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:26.014 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.014 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.014 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.014 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.892807+0000", 2024-09-13T15:04:26.015 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9183000000000005e-05 2024-09-13T15:04:26.015 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.015 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.015 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.015 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.892821+0000", 2024-09-13T15:04:26.015 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3861e-05 2024-09-13T15:04:26.015 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.016 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.016 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.016 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.892827+0000", 2024-09-13T15:04:26.016 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2439999999999998e-06 2024-09-13T15:04:26.016 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.016 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.016 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.016 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.894455+0000", 2024-09-13T15:04:26.016 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001627257 2024-09-13T15:04:26.016 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.016 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.016 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:04:26.016 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.894464+0000", 2024-09-13T15:04:26.016 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.4809999999999995e-06 2024-09-13T15:04:26.016 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.017 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.017 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.017 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.890832+0000", 2024-09-13T15:04:26.017 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99636818599999999 2024-09-13T15:04:26.017 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.017 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.017 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.017 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.890979+0000", 2024-09-13T15:04:26.017 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014698099999999999 2024-09-13T15:04:26.017 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.017 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.017 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.017 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.891141+0000", 2024-09-13T15:04:26.017 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016161800000000001 2024-09-13T15:04:26.018 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.018 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.018 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.018 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.908915+0000", 2024-09-13T15:04:26.018 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.017774366999999999 2024-09-13T15:04:26.018 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.018 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.018 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.018 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.908932+0000", 2024-09-13T15:04:26.018 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6820000000000002e-05 2024-09-13T15:04:26.018 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.018 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.018 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.018 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918564+0000", 2024-09-13T15:04:26.019 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0096315329999999994 2024-09-13T15:04:26.019 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.019 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.019 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.019 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918624+0000", 2024-09-13T15:04:26.019 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0449999999999999e-05 2024-09-13T15:04:26.019 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.019 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.019 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.020 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918727+0000", 2024-09-13T15:04:26.020 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000103019 2024-09-13T15:04:26.020 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.020 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.020 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.020 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.020 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.020 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:113315 9.c 9:33905679:::benchmark_data_smithi096_256909_object113314:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-09-13T15:04:26.020 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:11.954719+0000", 2024-09-13T15:04:26.020 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.025468558, 2024-09-13T15:04:26.020 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.96417554699999997, 2024-09-13T15:04:26.020 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.020 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.020 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.020 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.021 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.021 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 113315 2024-09-13T15:04:26.021 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.021 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.021 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.021 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.021 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954719+0000", 2024-09-13T15:04:26.021 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.021 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.021 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.021 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.021 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954719+0000", 2024-09-13T15:04:26.021 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.021 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.021 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.022 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.022 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954717+0000", 2024-09-13T15:04:26.022 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:26.022 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.022 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.022 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.022 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954773+0000", 2024-09-13T15:04:26.022 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6189e-05 2024-09-13T15:04:26.022 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.022 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.022 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.022 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954792+0000", 2024-09-13T15:04:26.023 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8705000000000001e-05 2024-09-13T15:04:26.023 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.023 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.023 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.023 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954798+0000", 2024-09-13T15:04:26.023 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0990000000000004e-06 2024-09-13T15:04:26.023 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.023 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.023 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.023 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954849+0000", 2024-09-13T15:04:26.023 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0444e-05 2024-09-13T15:04:26.023 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.023 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.023 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:04:26.023 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.954858+0000", 2024-09-13T15:04:26.024 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.8410000000000006e-06 2024-09-13T15:04:26.024 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.024 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.024 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.024 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.895625+0000", 2024-09-13T15:04:26.024 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.94076707699999995 2024-09-13T15:04:26.024 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.024 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.024 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.024 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.895768+0000", 2024-09-13T15:04:26.024 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014290900000000001 2024-09-13T15:04:26.024 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.024 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.024 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.025 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.895943+0000", 2024-09-13T15:04:26.025 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00017526999999999999 2024-09-13T15:04:26.025 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.025 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.025 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.025 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.908972+0000", 2024-09-13T15:04:26.025 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013028814999999999 2024-09-13T15:04:26.025 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.025 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.025 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.025 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.908979+0000", 2024-09-13T15:04:26.025 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9140000000000002e-06 2024-09-13T15:04:26.025 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.025 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.025 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918764+0000", 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0097855540000000001 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918808+0000", 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3510999999999997e-05 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918895+0000", 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7065999999999996e-05 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.026 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.027 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.027 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:113320 9.c 9:3f46f27a:::benchmark_data_smithi096_256909_object113319:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-09-13T15:04:26.027 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:11.961842+0000", 2024-09-13T15:04:26.027 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.01834606599999, 2024-09-13T15:04:26.027 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.95725164600000001, 2024-09-13T15:04:26.027 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.027 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.027 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.027 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.027 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.027 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 113320 2024-09-13T15:04:26.027 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.027 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.027 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.027 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.028 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961842+0000", 2024-09-13T15:04:26.028 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.028 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.028 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.028 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.028 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961842+0000", 2024-09-13T15:04:26.028 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.028 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.028 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.028 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.028 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961840+0000", 2024-09-13T15:04:26.028 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:26.028 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.028 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.028 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.029 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961882+0000", 2024-09-13T15:04:26.029 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2212999999999997e-05 2024-09-13T15:04:26.029 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.029 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.029 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.029 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961900+0000", 2024-09-13T15:04:26.029 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8196999999999999e-05 2024-09-13T15:04:26.029 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.029 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.029 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.029 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961908+0000", 2024-09-13T15:04:26.029 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8949999999999998e-06 2024-09-13T15:04:26.029 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.029 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.030 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.030 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961977+0000", 2024-09-13T15:04:26.030 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8628999999999996e-05 2024-09-13T15:04:26.030 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.030 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.030 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:04:26.030 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.961987+0000", 2024-09-13T15:04:26.030 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.8079999999999999e-06 2024-09-13T15:04:26.030 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.030 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.030 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.030 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.899547+0000", 2024-09-13T15:04:26.030 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.93756020399999995 2024-09-13T15:04:26.030 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.030 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.031 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.031 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.899680+0000", 2024-09-13T15:04:26.031 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013346699999999999 2024-09-13T15:04:26.031 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.031 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.031 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.031 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.899841+0000", 2024-09-13T15:04:26.031 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016104300000000001 2024-09-13T15:04:26.031 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.031 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.031 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.031 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.908988+0000", 2024-09-13T15:04:26.031 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.009146955 2024-09-13T15:04:26.031 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.032 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.032 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.032 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.909031+0000", 2024-09-13T15:04:26.032 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2908000000000003e-05 2024-09-13T15:04:26.032 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.032 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.032 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.032 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918935+0000", 2024-09-13T15:04:26.032 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0099035110000000003 2024-09-13T15:04:26.032 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.032 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.032 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.032 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.918993+0000", 2024-09-13T15:04:26.032 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8156999999999999e-05 2024-09-13T15:04:26.032 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.033 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.033 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.033 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.919093+0000", 2024-09-13T15:04:26.033 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000100342 2024-09-13T15:04:26.033 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.033 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.033 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.033 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.033 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.033 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:113329 9.c 9:339064f9:::benchmark_data_smithi096_256909_object113328:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-09-13T15:04:26.033 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:11.989105+0000", 2024-09-13T15:04:26.033 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 313.99108317899999, 2024-09-13T15:04:26.033 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.94653837500000004, 2024-09-13T15:04:26.033 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.033 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 113329 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989105+0000", 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989105+0000", 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.034 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989103+0000", 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989158+0000", 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5356999999999999e-05 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989170+0000", 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1674999999999999e-05 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.035 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.036 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.036 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989175+0000", 2024-09-13T15:04:26.036 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1270000000000002e-06 2024-09-13T15:04:26.036 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.036 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.036 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.036 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989225+0000", 2024-09-13T15:04:26.036 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9323000000000002e-05 2024-09-13T15:04:26.036 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.036 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.036 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:04:26.036 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.989232+0000", 2024-09-13T15:04:26.036 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7409999999999992e-06 2024-09-13T15:04:26.036 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.036 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.037 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.037 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.904069+0000", 2024-09-13T15:04:26.037 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.914836172 2024-09-13T15:04:26.037 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.037 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.037 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.037 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.904202+0000", 2024-09-13T15:04:26.037 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013350999999999999 2024-09-13T15:04:26.037 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.037 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.037 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.037 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.904370+0000", 2024-09-13T15:04:26.037 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016814400000000001 2024-09-13T15:04:26.037 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.037 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.038 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.038 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.908994+0000", 2024-09-13T15:04:26.038 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0046242970000000003 2024-09-13T15:04:26.038 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.038 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.038 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.038 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.909068+0000", 2024-09-13T15:04:26.038 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.3784999999999999e-05 2024-09-13T15:04:26.038 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.038 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.038 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.038 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.935324+0000", 2024-09-13T15:04:26.038 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.026256152000000001 2024-09-13T15:04:26.039 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.039 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.039 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.039 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.935381+0000", 2024-09-13T15:04:26.039 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6595000000000003e-05 2024-09-13T15:04:26.039 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.039 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.039 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.039 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.935643+0000", 2024-09-13T15:04:26.039 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00026189200000000001 2024-09-13T15:04:26.039 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.039 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.039 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.039 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.040 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.040 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:211155 9.c 9:36b1346c:::benchmark_data_smithi096_256909_object91747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2024-09-13T15:04:26.040 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:13.948289+0000", 2024-09-13T15:04:26.040 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 192.031898813, 2024-09-13T15:04:26.040 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4472571279999999, 2024-09-13T15:04:26.040 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.040 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.040 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.040 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.040 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.041 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 211155 2024-09-13T15:04:26.041 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.041 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.041 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.041 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.041 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:13.948289+0000", 2024-09-13T15:04:26.041 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.041 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.041 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.041 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.041 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:13.948289+0000", 2024-09-13T15:04:26.041 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.041 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.042 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.042 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.042 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:13.948287+0000", 2024-09-13T15:04:26.042 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:26.042 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.042 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.042 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.042 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:13.948296+0000", 2024-09-13T15:04:26.042 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4509999999999992e-06 2024-09-13T15:04:26.042 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.042 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.042 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.042 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:13.948313+0000", 2024-09-13T15:04:26.042 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7105e-05 2024-09-13T15:04:26.042 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.043 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.043 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.043 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:13.948320+0000", 2024-09-13T15:04:26.043 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0600000000000002e-06 2024-09-13T15:04:26.043 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.043 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.043 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.043 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:13.948485+0000", 2024-09-13T15:04:26.043 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016511299999999999 2024-09-13T15:04:26.043 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.043 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.043 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:04:26.043 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:13.948507+0000", 2024-09-13T15:04:26.044 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1727999999999999e-05 2024-09-13T15:04:26.044 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.044 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.044 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.044 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.329166+0000", 2024-09-13T15:04:26.044 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.38065899600000003 2024-09-13T15:04:26.044 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.044 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.044 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:04:26.044 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.329175+0000", 2024-09-13T15:04:26.044 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.2890000000000003e-06 2024-09-13T15:04:26.044 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.044 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.045 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.045 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.365470+0000", 2024-09-13T15:04:26.045 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.036294848 2024-09-13T15:04:26.045 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.045 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.045 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.045 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.365613+0000", 2024-09-13T15:04:26.045 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000143306 2024-09-13T15:04:26.045 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.045 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.045 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.045 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.365734+0000", 2024-09-13T15:04:26.045 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012084899999999999 2024-09-13T15:04:26.045 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.046 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.046 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.046 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.375802+0000", 2024-09-13T15:04:26.046 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010068107 2024-09-13T15:04:26.046 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.046 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.046 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.046 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.395334+0000", 2024-09-13T15:04:26.046 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.019531915 2024-09-13T15:04:26.046 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.046 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.046 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.046 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.395399+0000", 2024-09-13T15:04:26.046 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5122000000000004e-05 2024-09-13T15:04:26.047 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.047 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.047 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.047 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.395449+0000", 2024-09-13T15:04:26.047 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9412000000000001e-05 2024-09-13T15:04:26.047 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.047 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.047 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.047 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.395546+0000", 2024-09-13T15:04:26.047 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7497999999999994e-05 2024-09-13T15:04:26.047 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.047 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.047 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.047 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.047 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.048 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:211202 9.c 9:331ea6e0:::benchmark_data_smithi096_256909_object91794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2024-09-13T15:04:26.048 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:14.012688+0000", 2024-09-13T15:04:26.048 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 191.967499745, 2024-09-13T15:04:26.048 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3841229259999999, 2024-09-13T15:04:26.048 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.048 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.048 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.048 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.048 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.048 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 211202 2024-09-13T15:04:26.048 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.048 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.048 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.048 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.048 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.012688+0000", 2024-09-13T15:04:26.049 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.049 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.049 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.049 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.049 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.012688+0000", 2024-09-13T15:04:26.049 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.049 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.049 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.049 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.049 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.012686+0000", 2024-09-13T15:04:26.049 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:26.049 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.049 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.050 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.050 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.012693+0000", 2024-09-13T15:04:26.050 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.669e-06 2024-09-13T15:04:26.050 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.050 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.050 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.050 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.012706+0000", 2024-09-13T15:04:26.050 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3370000000000001e-05 2024-09-13T15:04:26.050 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.050 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.050 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.050 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.012712+0000", 2024-09-13T15:04:26.050 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3319999999999999e-06 2024-09-13T15:04:26.050 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.051 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.051 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.051 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.012767+0000", 2024-09-13T15:04:26.051 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5568000000000001e-05 2024-09-13T15:04:26.051 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.051 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.051 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:04:26.051 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.012796+0000", 2024-09-13T15:04:26.051 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9002000000000001e-05 2024-09-13T15:04:26.051 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.051 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.051 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.051 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.329208+0000", 2024-09-13T15:04:26.052 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.31641154999999999 2024-09-13T15:04:26.052 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.052 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.052 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:04:26.052 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.329214+0000", 2024-09-13T15:04:26.052 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1e-06 2024-09-13T15:04:26.052 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.052 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.052 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.052 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.369020+0000", 2024-09-13T15:04:26.052 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.039805764 2024-09-13T15:04:26.052 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.052 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.052 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.053 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.369139+0000", 2024-09-13T15:04:26.053 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119444 2024-09-13T15:04:26.053 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.053 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.053 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.053 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.369275+0000", 2024-09-13T15:04:26.053 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000135969 2024-09-13T15:04:26.053 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.053 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.053 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.053 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.378787+0000", 2024-09-13T15:04:26.053 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0095121149999999998 2024-09-13T15:04:26.053 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.053 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.054 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.054 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.395371+0000", 2024-09-13T15:04:26.054 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.016583743000000001 2024-09-13T15:04:26.054 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.054 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.054 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.054 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.396628+0000", 2024-09-13T15:04:26.054 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0012573479999999999 2024-09-13T15:04:26.054 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.054 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.054 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.054 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.396708+0000", 2024-09-13T15:04:26.054 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0037000000000006e-05 2024-09-13T15:04:26.054 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.055 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.055 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.055 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.396811+0000", 2024-09-13T15:04:26.055 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010258400000000001 2024-09-13T15:04:26.055 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.055 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.055 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.055 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.055 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.056 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:211210 9.c 9:31e8f7ef:::benchmark_data_smithi096_256909_object91802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2024-09-13T15:04:26.056 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:14.023816+0000", 2024-09-13T15:04:26.056 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 191.95637211499999, 2024-09-13T15:04:26.056 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3732601360000001, 2024-09-13T15:04:26.056 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.056 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.056 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.056 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.056 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.057 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 211210 2024-09-13T15:04:26.057 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.057 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.057 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.057 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.057 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.023816+0000", 2024-09-13T15:04:26.057 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.057 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.057 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.057 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.057 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.023816+0000", 2024-09-13T15:04:26.057 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.057 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.057 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.058 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.058 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.023814+0000", 2024-09-13T15:04:26.058 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:26.058 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.058 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.058 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.058 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.023823+0000", 2024-09-13T15:04:26.058 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6649999999999998e-06 2024-09-13T15:04:26.058 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.058 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.058 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.058 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.023844+0000", 2024-09-13T15:04:26.058 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0593000000000001e-05 2024-09-13T15:04:26.059 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.059 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.059 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.059 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.023851+0000", 2024-09-13T15:04:26.059 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.272e-06 2024-09-13T15:04:26.059 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.059 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.059 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.059 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.023908+0000", 2024-09-13T15:04:26.059 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6938999999999998e-05 2024-09-13T15:04:26.059 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.059 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.059 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:04:26.059 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.023919+0000", 2024-09-13T15:04:26.060 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0672999999999999e-05 2024-09-13T15:04:26.060 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.060 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.060 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.060 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.329249+0000", 2024-09-13T15:04:26.060 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.305329929 2024-09-13T15:04:26.060 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.060 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.060 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:04:26.060 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.329255+0000", 2024-09-13T15:04:26.060 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9029999999999998e-06 2024-09-13T15:04:26.060 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.060 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.060 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.061 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.372552+0000", 2024-09-13T15:04:26.061 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.043296513 2024-09-13T15:04:26.061 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.061 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.061 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.061 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.372685+0000", 2024-09-13T15:04:26.061 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013253000000000001 2024-09-13T15:04:26.061 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.061 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.061 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.061 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.372830+0000", 2024-09-13T15:04:26.061 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014501100000000001 2024-09-13T15:04:26.061 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.061 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.062 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.062 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.381770+0000", 2024-09-13T15:04:26.062 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0089403420000000004 2024-09-13T15:04:26.062 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.062 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.062 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.062 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.396870+0000", 2024-09-13T15:04:26.062 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.015100583000000001 2024-09-13T15:04:26.062 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.062 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.063 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.063 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.396899+0000", 2024-09-13T15:04:26.063 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8464999999999999e-05 2024-09-13T15:04:26.063 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.063 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.063 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.064 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.396983+0000", 2024-09-13T15:04:26.064 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4189000000000003e-05 2024-09-13T15:04:26.064 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.064 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.064 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.064 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.397076+0000", 2024-09-13T15:04:26.065 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.2608999999999994e-05 2024-09-13T15:04:26.065 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.065 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.065 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.065 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.065 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.065 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:211228 9.c 9:3250b113:::benchmark_data_smithi096_256909_object91820:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-13T15:04:26.066 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:14.618117+0000", 2024-09-13T15:04:26.066 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 191.36207078800001, 2024-09-13T15:04:26.066 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.77920723599999997, 2024-09-13T15:04:26.066 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.066 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.066 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.066 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.066 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.067 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 211228 2024-09-13T15:04:26.067 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.067 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.067 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.067 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.067 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.618117+0000", 2024-09-13T15:04:26.067 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.068 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.068 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.068 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.068 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.618117+0000", 2024-09-13T15:04:26.068 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.068 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.069 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.069 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.069 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.618116+0000", 2024-09-13T15:04:26.069 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:26.069 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.069 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.070 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.070 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.618121+0000", 2024-09-13T15:04:26.070 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4720000000000003e-06 2024-09-13T15:04:26.070 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.070 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.070 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.070 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.618131+0000", 2024-09-13T15:04:26.070 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0006e-05 2024-09-13T15:04:26.070 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.071 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.071 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.071 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.618135+0000", 2024-09-13T15:04:26.071 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9299999999999996e-06 2024-09-13T15:04:26.071 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.071 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.071 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.071 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.618176+0000", 2024-09-13T15:04:26.071 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1383e-05 2024-09-13T15:04:26.071 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.071 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.071 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:04:26.071 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.618187+0000", 2024-09-13T15:04:26.072 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0412e-05 2024-09-13T15:04:26.072 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.072 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.072 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.072 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.375831+0000", 2024-09-13T15:04:26.072 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.75764410999999998 2024-09-13T15:04:26.072 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.072 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.072 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.072 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.375948+0000", 2024-09-13T15:04:26.072 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000117113 2024-09-13T15:04:26.072 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.072 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.073 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.073 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.376058+0000", 2024-09-13T15:04:26.073 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000109744 2024-09-13T15:04:26.073 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.073 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.073 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.073 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.394836+0000", 2024-09-13T15:04:26.073 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.018778242000000001 2024-09-13T15:04:26.073 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.073 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.073 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.074 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.397109+0000", 2024-09-13T15:04:26.074 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0022729299999999998 2024-09-13T15:04:26.074 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.074 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.074 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.074 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.397161+0000", 2024-09-13T15:04:26.074 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1610000000000002e-05 2024-09-13T15:04:26.074 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.074 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.074 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.074 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.397226+0000", 2024-09-13T15:04:26.074 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5798999999999998e-05 2024-09-13T15:04:26.074 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.075 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.075 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.075 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.397324+0000", 2024-09-13T15:04:26.075 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7847e-05 2024-09-13T15:04:26.075 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.075 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.075 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.075 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.075 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.075 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:211244 9.c 9:39d9008a:::benchmark_data_smithi096_256909_object91836:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-13T15:04:26.075 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:14.652802+0000", 2024-09-13T15:04:26.075 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 191.32738615, 2024-09-13T15:04:26.076 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.75183837499999995, 2024-09-13T15:04:26.076 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.076 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.076 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.076 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.076 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.076 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 211244 2024-09-13T15:04:26.076 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.076 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.076 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.076 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.076 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.652802+0000", 2024-09-13T15:04:26.077 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.077 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.077 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.077 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.077 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.652802+0000", 2024-09-13T15:04:26.077 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.077 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.077 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.077 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.077 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.652800+0000", 2024-09-13T15:04:26.077 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:26.077 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.078 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.078 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.078 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.652808+0000", 2024-09-13T15:04:26.078 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5609999999999999e-06 2024-09-13T15:04:26.078 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.078 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.078 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.078 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.652824+0000", 2024-09-13T15:04:26.078 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6008999999999999e-05 2024-09-13T15:04:26.078 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.078 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.078 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.079 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.652832+0000", 2024-09-13T15:04:26.079 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5809999999999999e-06 2024-09-13T15:04:26.079 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.079 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.079 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.079 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.652893+0000", 2024-09-13T15:04:26.079 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1519999999999994e-05 2024-09-13T15:04:26.079 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.079 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.079 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:04:26.080 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.652900+0000", 2024-09-13T15:04:26.080 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3799999999999999e-06 2024-09-13T15:04:26.080 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.080 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.080 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.080 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.378818+0000", 2024-09-13T15:04:26.080 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.72591796799999997 2024-09-13T15:04:26.080 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.080 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.080 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.080 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.378940+0000", 2024-09-13T15:04:26.081 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012196700000000001 2024-09-13T15:04:26.081 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.081 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.081 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.081 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.379026+0000", 2024-09-13T15:04:26.081 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5997999999999999e-05 2024-09-13T15:04:26.081 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.081 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.081 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.081 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.394988+0000", 2024-09-13T15:04:26.082 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.015962701999999999 2024-09-13T15:04:26.082 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.082 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.082 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.082 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.397135+0000", 2024-09-13T15:04:26.082 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0021466739999999999 2024-09-13T15:04:26.082 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.082 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.082 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.082 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.404495+0000", 2024-09-13T15:04:26.082 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0073596199999999999 2024-09-13T15:04:26.082 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.082 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.083 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.083 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.404542+0000", 2024-09-13T15:04:26.083 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7383999999999999e-05 2024-09-13T15:04:26.083 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.083 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.083 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.083 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.404640+0000", 2024-09-13T15:04:26.083 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.8029999999999995e-05 2024-09-13T15:04:26.083 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.083 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.083 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.083 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.083 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.083 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:211247 9.c 9:30a7c3ea:::benchmark_data_smithi096_256909_object91839:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-13T15:04:26.084 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:14.657300+0000", 2024-09-13T15:04:26.084 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 191.322887442, 2024-09-13T15:04:26.084 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.74749475300000001, 2024-09-13T15:04:26.084 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.084 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.084 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.084 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.084 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.084 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 211247 2024-09-13T15:04:26.085 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.085 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.085 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.085 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.085 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.657300+0000", 2024-09-13T15:04:26.085 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.085 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.085 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.085 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.085 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.657300+0000", 2024-09-13T15:04:26.085 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.085 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.086 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.086 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.086 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.657299+0000", 2024-09-13T15:04:26.086 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:26.086 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.086 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.086 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.086 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.657306+0000", 2024-09-13T15:04:26.086 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.024e-06 2024-09-13T15:04:26.086 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.086 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.086 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.086 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.657323+0000", 2024-09-13T15:04:26.086 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7516e-05 2024-09-13T15:04:26.087 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.087 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.087 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.087 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.657331+0000", 2024-09-13T15:04:26.087 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.43e-06 2024-09-13T15:04:26.087 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.087 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.087 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.087 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.657382+0000", 2024-09-13T15:04:26.087 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1555999999999999e-05 2024-09-13T15:04:26.087 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.087 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.087 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:04:26.087 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.657388+0000", 2024-09-13T15:04:26.088 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1770000000000003e-06 2024-09-13T15:04:26.088 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.088 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.088 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.088 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.381817+0000", 2024-09-13T15:04:26.088 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.72442888999999999 2024-09-13T15:04:26.088 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.088 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.088 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.088 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.381964+0000", 2024-09-13T15:04:26.088 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014648999999999999 2024-09-13T15:04:26.088 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.088 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.088 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.089 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.382069+0000", 2024-09-13T15:04:26.089 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010542700000000001 2024-09-13T15:04:26.089 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.089 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.089 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.089 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.394994+0000", 2024-09-13T15:04:26.089 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012925256 2024-09-13T15:04:26.089 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.089 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.089 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.089 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.397440+0000", 2024-09-13T15:04:26.089 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0024461090000000001 2024-09-13T15:04:26.089 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.090 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.090 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.090 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.404674+0000", 2024-09-13T15:04:26.090 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0072331260000000003 2024-09-13T15:04:26.090 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.090 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.090 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.090 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.404708+0000", 2024-09-13T15:04:26.090 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4802999999999999e-05 2024-09-13T15:04:26.090 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.090 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.090 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.090 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.404795+0000", 2024-09-13T15:04:26.091 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.6613999999999994e-05 2024-09-13T15:04:26.091 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.091 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.091 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.091 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.091 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.091 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:211258 9.c 9:341b5830:::benchmark_data_smithi096_256909_object91850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-13T15:04:26.091 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:14.672629+0000", 2024-09-13T15:04:26.091 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 191.307558433, 2024-09-13T15:04:26.091 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.732345625, 2024-09-13T15:04:26.092 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.092 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.092 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.092 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.092 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.092 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 211258 2024-09-13T15:04:26.092 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.092 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.092 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.092 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.092 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.672629+0000", 2024-09-13T15:04:26.092 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.092 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.093 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.093 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.093 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.672629+0000", 2024-09-13T15:04:26.093 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.093 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.093 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.093 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.093 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.672628+0000", 2024-09-13T15:04:26.093 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:26.093 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.093 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.093 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.093 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.672635+0000", 2024-09-13T15:04:26.093 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5050000000000001e-06 2024-09-13T15:04:26.094 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.094 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.094 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.094 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.672649+0000", 2024-09-13T15:04:26.094 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4379000000000001e-05 2024-09-13T15:04:26.094 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.094 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.094 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.094 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.672656+0000", 2024-09-13T15:04:26.094 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6239999999999996e-06 2024-09-13T15:04:26.094 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.094 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.094 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.094 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.672707+0000", 2024-09-13T15:04:26.095 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0451999999999997e-05 2024-09-13T15:04:26.095 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.095 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.095 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:04:26.095 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.672716+0000", 2024-09-13T15:04:26.095 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6169999999999996e-06 2024-09-13T15:04:26.095 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.095 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.095 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.095 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.385410+0000", 2024-09-13T15:04:26.095 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.71269392499999995 2024-09-13T15:04:26.095 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.095 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.096 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.096 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.385541+0000", 2024-09-13T15:04:26.096 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013125699999999999 2024-09-13T15:04:26.096 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.096 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.096 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.096 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.385682+0000", 2024-09-13T15:04:26.096 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000140469 2024-09-13T15:04:26.096 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.096 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.096 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.096 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.395000+0000", 2024-09-13T15:04:26.096 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0093177899999999994 2024-09-13T15:04:26.096 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.097 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.097 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.097 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.397468+0000", 2024-09-13T15:04:26.097 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0024681059999999999 2024-09-13T15:04:26.097 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.097 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.098 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.098 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.404826+0000", 2024-09-13T15:04:26.098 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0073586609999999998 2024-09-13T15:04:26.098 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.098 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.098 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.099 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.404867+0000", 2024-09-13T15:04:26.099 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0222999999999998e-05 2024-09-13T15:04:26.099 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.099 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.099 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.099 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.404975+0000", 2024-09-13T15:04:26.099 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000108312 2024-09-13T15:04:26.099 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.099 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.100 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.100 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.100 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.100 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:211265 9.c 9:344320ff:::benchmark_data_smithi096_256909_object91857:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-13T15:04:26.100 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:14.704095+0000", 2024-09-13T15:04:26.100 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 191.27609265300001, 2024-09-13T15:04:26.100 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.71696991200000004, 2024-09-13T15:04:26.100 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.100 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.100 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.100 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.100 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.101 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 211265 2024-09-13T15:04:26.101 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.101 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.101 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.101 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.101 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.704095+0000", 2024-09-13T15:04:26.101 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.101 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.101 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.101 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.101 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.704095+0000", 2024-09-13T15:04:26.101 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.102 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.102 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.102 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.102 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.704094+0000", 2024-09-13T15:04:26.102 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:26.102 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.102 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.102 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.102 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.704100+0000", 2024-09-13T15:04:26.102 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.5649999999999998e-06 2024-09-13T15:04:26.102 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.102 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.102 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.103 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.704112+0000", 2024-09-13T15:04:26.103 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1704e-05 2024-09-13T15:04:26.103 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.103 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.103 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.103 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.704117+0000", 2024-09-13T15:04:26.103 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3009999999999997e-06 2024-09-13T15:04:26.103 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.103 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.103 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.103 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.704156+0000", 2024-09-13T15:04:26.103 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8974000000000003e-05 2024-09-13T15:04:26.103 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.103 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.104 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:04:26.104 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:14.704164+0000", 2024-09-13T15:04:26.104 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9410000000000007e-06 2024-09-13T15:04:26.104 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.104 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.104 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.104 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.391383+0000", 2024-09-13T15:04:26.104 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.68721898999999997 2024-09-13T15:04:26.104 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.104 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.104 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.104 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.391521+0000", 2024-09-13T15:04:26.105 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000137923 2024-09-13T15:04:26.105 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.105 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.105 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.105 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.391647+0000", 2024-09-13T15:04:26.105 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012601600000000001 2024-09-13T15:04:26.105 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.105 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.105 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.105 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.397494+0000", 2024-09-13T15:04:26.105 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0058471729999999998 2024-09-13T15:04:26.105 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.106 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.106 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.106 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.405035+0000", 2024-09-13T15:04:26.106 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0075403110000000001 2024-09-13T15:04:26.106 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.106 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.106 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.106 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:26.108 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:25 smithi096 ceph-mon[102928]: pgmap v1245: 119 pgs: 119 active+clean; 8.2 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 127 B/s rd, 65 MiB/s wr, 1.21k op/s 2024-09-13T15:04:26.108 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:25 smithi096 ceph-mon[98901]: pgmap v1245: 119 pgs: 119 active+clean; 8.2 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 127 B/s rd, 65 MiB/s wr, 1.21k op/s 2024-09-13T15:04:26.109 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.420886+0000", 2024-09-13T15:04:26.109 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.015851046000000001 2024-09-13T15:04:26.109 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.109 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.109 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.109 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.420961+0000", 2024-09-13T15:04:26.109 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5298999999999999e-05 2024-09-13T15:04:26.109 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.109 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.109 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.109 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:15.421065+0000", 2024-09-13T15:04:26.110 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010404300000000001 2024-09-13T15:04:26.110 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.110 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.110 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.110 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.110 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.110 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:224588 9.c 9:36960f8a:::benchmark_data_smithi096_256909_object105180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-09-13T15:04:26.110 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:32.268673+0000", 2024-09-13T15:04:26.110 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 173.711514763, 2024-09-13T15:04:26.110 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.14422964799999999, 2024-09-13T15:04:26.110 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.110 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.110 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.111 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.111 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.111 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 224588 2024-09-13T15:04:26.111 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.111 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.111 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.111 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.111 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268673+0000", 2024-09-13T15:04:26.111 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.111 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.111 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.111 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.111 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268673+0000", 2024-09-13T15:04:26.111 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.112 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.112 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.112 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.112 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268673+0000", 2024-09-13T15:04:26.112 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:26.112 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.112 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.112 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.112 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268675+0000", 2024-09-13T15:04:26.112 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.339e-06 2024-09-13T15:04:26.112 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.112 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.113 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.113 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268679+0000", 2024-09-13T15:04:26.113 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.352e-06 2024-09-13T15:04:26.113 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.113 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.113 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.113 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268682+0000", 2024-09-13T15:04:26.113 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.841e-06 2024-09-13T15:04:26.113 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.113 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.113 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.113 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.281823+0000", 2024-09-13T15:04:26.113 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013140681 2024-09-13T15:04:26.114 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.114 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.114 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.114 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.281966+0000", 2024-09-13T15:04:26.114 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014265299999999999 2024-09-13T15:04:26.114 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.114 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.114 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.114 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.282078+0000", 2024-09-13T15:04:26.114 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011268 2024-09-13T15:04:26.114 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.114 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.114 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.114 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.292698+0000", 2024-09-13T15:04:26.115 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010620124 2024-09-13T15:04:26.115 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.115 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.115 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.115 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.412664+0000", 2024-09-13T15:04:26.115 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.11996593799999999 2024-09-13T15:04:26.115 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.115 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.115 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.115 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.412708+0000", 2024-09-13T15:04:26.115 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3912999999999998e-05 2024-09-13T15:04:26.116 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.116 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.116 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.116 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.412775+0000", 2024-09-13T15:04:26.116 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7175999999999995e-05 2024-09-13T15:04:26.117 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.117 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.117 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.117 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.412903+0000", 2024-09-13T15:04:26.117 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012724000000000001 2024-09-13T15:04:26.117 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.117 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.118 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.118 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.118 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.118 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:224598 9.c 9:35a0ac15:::benchmark_data_smithi096_256909_object105190:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-09-13T15:04:26.118 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:32.268685+0000", 2024-09-13T15:04:26.118 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 173.71150228900001, 2024-09-13T15:04:26.118 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.14449662899999999, 2024-09-13T15:04:26.119 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.119 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.119 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.119 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.119 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.119 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 224598 2024-09-13T15:04:26.120 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.120 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.120 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.120 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.120 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268685+0000", 2024-09-13T15:04:26.120 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.121 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.121 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.121 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.121 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268685+0000", 2024-09-13T15:04:26.121 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.121 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.121 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.122 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.122 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268685+0000", 2024-09-13T15:04:26.122 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:26.122 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.122 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.122 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.123 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268687+0000", 2024-09-13T15:04:26.123 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.81e-06 2024-09-13T15:04:26.123 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.123 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.123 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.123 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268691+0000", 2024-09-13T15:04:26.123 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4159999999999997e-06 2024-09-13T15:04:26.123 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.124 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.124 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.124 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268695+0000", 2024-09-13T15:04:26.124 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1640000000000001e-06 2024-09-13T15:04:26.124 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.124 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.124 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.124 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.288808+0000", 2024-09-13T15:04:26.124 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.020113848 2024-09-13T15:04:26.124 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.124 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.125 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.125 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.288941+0000", 2024-09-13T15:04:26.125 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013222099999999999 2024-09-13T15:04:26.125 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.125 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.125 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.125 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.289072+0000", 2024-09-13T15:04:26.125 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000131376 2024-09-13T15:04:26.125 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.125 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.125 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.125 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.412701+0000", 2024-09-13T15:04:26.125 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.12362851499999999 2024-09-13T15:04:26.125 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.412914+0000", 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00021308000000000001 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.413005+0000", 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1817999999999994e-05 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.413065+0000", 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9956e-05 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.126 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.127 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.127 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.413182+0000", 2024-09-13T15:04:26.127 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011668 2024-09-13T15:04:26.127 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.127 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.127 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.127 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.127 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.127 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:224602 9.c 9:3b951dca:::benchmark_data_smithi096_256909_object105194:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-09-13T15:04:26.127 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:32.268698+0000", 2024-09-13T15:04:26.127 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 173.71148997, 2024-09-13T15:04:26.127 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.14566997800000001, 2024-09-13T15:04:26.127 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.127 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.127 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.128 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.128 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.128 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 224602 2024-09-13T15:04:26.128 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.128 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.128 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.128 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.128 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268698+0000", 2024-09-13T15:04:26.128 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.128 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.128 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.128 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.128 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268698+0000", 2024-09-13T15:04:26.128 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.128 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.129 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.129 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.129 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268697+0000", 2024-09-13T15:04:26.129 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:26.129 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.129 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.129 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.129 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268699+0000", 2024-09-13T15:04:26.130 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.857e-06 2024-09-13T15:04:26.130 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.130 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.130 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.130 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268704+0000", 2024-09-13T15:04:26.130 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4530000000000004e-06 2024-09-13T15:04:26.130 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.130 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.130 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.130 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268707+0000", 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.258e-06 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.292730+0000", 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.024022768999999999 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.292846+0000", 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011649700000000001 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.292980+0000", 2024-09-13T15:04:26.131 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000133347 2024-09-13T15:04:26.132 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.132 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.132 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.132 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.412919+0000", 2024-09-13T15:04:26.132 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.11993981200000001 2024-09-13T15:04:26.132 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.132 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.132 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.132 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.412963+0000", 2024-09-13T15:04:26.132 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3387000000000003e-05 2024-09-13T15:04:26.132 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.132 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.132 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.132 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.413863+0000", 2024-09-13T15:04:26.132 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00089978200000000001 2024-09-13T15:04:26.133 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.133 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.133 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.133 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.414005+0000", 2024-09-13T15:04:26.133 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014225500000000001 2024-09-13T15:04:26.133 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.133 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.133 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.133 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.414368+0000", 2024-09-13T15:04:26.133 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00036284899999999999 2024-09-13T15:04:26.133 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.133 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.133 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:224604 9.c 9:3bd9d4b6:::benchmark_data_smithi096_256909_object105196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:32.268710+0000", 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 173.711477489, 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.15106735299999999, 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 224604 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.134 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268710+0000", 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268710+0000", 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268710+0000", 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:26.135 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268712+0000", 2024-09-13T15:04:26.136 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.761e-06 2024-09-13T15:04:26.136 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.136 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.136 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:26.136 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268716+0000", 2024-09-13T15:04:26.136 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4320000000000001e-06 2024-09-13T15:04:26.136 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.136 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.136 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:26.136 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.268719+0000", 2024-09-13T15:04:26.136 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8779999999999998e-06 2024-09-13T15:04:26.136 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.136 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.136 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:26.136 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.296379+0000", 2024-09-13T15:04:26.137 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.027659996999999999 2024-09-13T15:04:26.137 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.137 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.137 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:26.137 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.296513+0000", 2024-09-13T15:04:26.137 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013369 2024-09-13T15:04:26.137 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.137 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.137 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:04:26.137 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.409339+0000", 2024-09-13T15:04:26.137 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.112826675 2024-09-13T15:04:26.137 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.137 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.137 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:26.138 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.412970+0000", 2024-09-13T15:04:26.138 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0036302650000000001 2024-09-13T15:04:26.138 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.138 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.138 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.138 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.416509+0000", 2024-09-13T15:04:26.138 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035391960000000001 2024-09-13T15:04:26.138 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.138 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.138 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:26.138 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.419161+0000", 2024-09-13T15:04:26.138 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0026526290000000001 2024-09-13T15:04:26.138 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.138 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.138 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:26.139 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.419293+0000", 2024-09-13T15:04:26.139 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013157699999999999 2024-09-13T15:04:26.139 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:26.139 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:26.139 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:26.139 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.419778+0000", 2024-09-13T15:04:26.139 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00048451800000000003 2024-09-13T15:04:26.139 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.139 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.139 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.139 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:26.139 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:26.139 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:04:26.179 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:26.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:26.281 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:26.311 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:26.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:26.412 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:26.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.436 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:26.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:26.537 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:26.563 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:26.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:26.665 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:26.696 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:26.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:26.798 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:26.843 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:26.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:26.945 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:26.967 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:26.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:27.068 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:27.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.098 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:27.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:27.199 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:27.237 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:27.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:27.334 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:04:27.339 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:27.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.422 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:27.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:27.523 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:27.544 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:27.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:27.646 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:27.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.681 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:27.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:27.783 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:27.824 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:27.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:27.874 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:04:27.912 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:27.926 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:27.941 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T15:04:27.963 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:27.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:28.066 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:28.119 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:28.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:28.203 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:27 smithi096 ceph-mon[102928]: pgmap v1246: 119 pgs: 119 active+clean; 8.2 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 127 B/s rd, 34 MiB/s wr, 810 op/s 2024-09-13T15:04:28.204 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:04:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:04:27] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.51.0" 2024-09-13T15:04:28.205 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:27 smithi096 ceph-mon[98901]: pgmap v1246: 119 pgs: 119 active+clean; 8.2 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 127 B/s rd, 34 MiB/s wr, 810 op/s 2024-09-13T15:04:28.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:27 smithi204 ceph-mon[87071]: pgmap v1246: 119 pgs: 119 active+clean; 8.2 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 127 B/s rd, 34 MiB/s wr, 810 op/s 2024-09-13T15:04:28.221 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:28.252 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:28.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:28.353 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:28.405 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:28.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.419 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:28.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:28.520 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:28.547 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:28.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:28.649 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:28.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.689 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:28.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:28.790 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:28.820 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:28.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:28.903 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T15:04:28.921 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:28.957 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:28.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:29.058 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:29.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.098 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:29.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:29.199 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:28 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/762023480' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:04:29.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:28 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:04:29.228 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:29.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:29.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:28 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/762023480' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:04:29.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:28 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:04:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:28 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/762023480' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:04:29.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:28 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:04:29.330 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:29.367 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:29.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:29.469 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:29.498 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:29.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:29.600 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:29.628 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:29.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:29.729 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:29.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.766 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:29.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:29.867 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:29.900 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:29.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:30.002 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:30.031 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:30.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:30.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:29 smithi096 ceph-mon[98901]: pgmap v1247: 119 pgs: 119 active+clean; 7.9 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 115 B/s rd, 29 MiB/s wr, 1.11k op/s 2024-09-13T15:04:30.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:29 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:04:30.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:29 smithi096 ceph-mon[98901]: osdmap e371: 8 total, 8 up, 7 in 2024-09-13T15:04:30.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:29 smithi096 ceph-mon[102928]: pgmap v1247: 119 pgs: 119 active+clean; 7.9 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 115 B/s rd, 29 MiB/s wr, 1.11k op/s 2024-09-13T15:04:30.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:29 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:04:30.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:29 smithi096 ceph-mon[102928]: osdmap e371: 8 total, 8 up, 7 in 2024-09-13T15:04:30.133 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:30.172 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:30.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:30.216 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:29 smithi204 ceph-mon[87071]: pgmap v1247: 119 pgs: 119 active+clean; 7.9 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 115 B/s rd, 29 MiB/s wr, 1.11k op/s 2024-09-13T15:04:30.216 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:29 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:04:30.216 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:29 smithi204 ceph-mon[87071]: osdmap e371: 8 total, 8 up, 7 in 2024-09-13T15:04:30.275 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:30.301 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:30.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:30.396 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:04:30.396 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T15:04:30.397 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.397 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:146081 10.9 10:9a457b95:::benchmark_data_smithi096_288650_object10389:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2024-09-13T15:04:30.397 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:30.376380+0000", 2024-09-13T15:04:30.397 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.017210380000000001, 2024-09-13T15:04:30.397 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.017240833000000001, 2024-09-13T15:04:30.397 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:30.397 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:04:30.397 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:30.397 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:30.397 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:30.397 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 146081 2024-09-13T15:04:30.397 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.397 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:30.398 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.398 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:30.398 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376380+0000", 2024-09-13T15:04:30.398 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.398 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.398 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.398 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:30.398 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376380+0000", 2024-09-13T15:04:30.398 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.398 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.398 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.398 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:30.398 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376378+0000", 2024-09-13T15:04:30.399 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:30.399 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.399 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.399 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:30.399 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376385+0000", 2024-09-13T15:04:30.399 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6749999999999996e-06 2024-09-13T15:04:30.399 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.399 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.399 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:30.399 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376398+0000", 2024-09-13T15:04:30.400 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3533000000000001e-05 2024-09-13T15:04:30.400 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.400 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.400 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:30.400 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376404+0000", 2024-09-13T15:04:30.400 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.891e-06 2024-09-13T15:04:30.400 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.400 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.400 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:30.400 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376848+0000", 2024-09-13T15:04:30.400 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00044337400000000001 2024-09-13T15:04:30.400 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.400 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.401 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:30.401 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.377467+0000", 2024-09-13T15:04:30.401 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00061907300000000004 2024-09-13T15:04:30.401 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.401 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.401 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:04:30.401 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.377569+0000", 2024-09-13T15:04:30.401 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010242399999999999 2024-09-13T15:04:30.401 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.401 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.402 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:30.402 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387506+0000", 2024-09-13T15:04:30.402 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0099364929999999994 2024-09-13T15:04:30.402 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.402 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.402 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:30.402 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387540+0000", 2024-09-13T15:04:30.402 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4337e-05 2024-09-13T15:04:30.402 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.402 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:30.402 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.403 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.403 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.403 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:146085 10.9 10:96b8ebbc:::benchmark_data_smithi096_288650_object10393:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2024-09-13T15:04:30.403 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:30.376729+0000", 2024-09-13T15:04:30.403 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.016860825999999999, 2024-09-13T15:04:30.403 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.016976694000000001, 2024-09-13T15:04:30.403 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:30.403 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:04:30.403 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:30.403 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:30.403 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:30.403 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 146085 2024-09-13T15:04:30.403 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.404 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:30.404 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.404 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:30.404 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376729+0000", 2024-09-13T15:04:30.404 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.404 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.404 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.404 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:30.404 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376729+0000", 2024-09-13T15:04:30.404 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.404 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.404 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.404 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:30.404 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376729+0000", 2024-09-13T15:04:30.405 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:04:30.405 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.405 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.405 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:30.405 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376734+0000", 2024-09-13T15:04:30.405 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9080000000000003e-06 2024-09-13T15:04:30.405 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.405 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.405 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:30.405 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376742+0000", 2024-09-13T15:04:30.405 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3699999999999995e-06 2024-09-13T15:04:30.405 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.405 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.405 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:30.406 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376747+0000", 2024-09-13T15:04:30.406 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2650000000000004e-06 2024-09-13T15:04:30.406 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.406 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.406 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:30.406 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.381626+0000", 2024-09-13T15:04:30.406 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0048784190000000002 2024-09-13T15:04:30.406 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.406 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.406 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:30.406 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.381935+0000", 2024-09-13T15:04:30.406 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00030947400000000001 2024-09-13T15:04:30.406 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.406 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.407 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:04:30.407 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.382276+0000", 2024-09-13T15:04:30.407 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00034140899999999998 2024-09-13T15:04:30.407 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.407 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.407 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:30.407 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387568+0000", 2024-09-13T15:04:30.407 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0052919530000000003 2024-09-13T15:04:30.407 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.407 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:30.407 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.407 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.407 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.407 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:146086 10.6 10:6e1c3a31:::benchmark_data_smithi096_288650_object10394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2024-09-13T15:04:30.408 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:30.376879+0000", 2024-09-13T15:04:30.408 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.016710815, 2024-09-13T15:04:30.408 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.016898301000000001, 2024-09-13T15:04:30.408 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:30.408 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:04:30.408 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:30.408 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:30.409 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:30.409 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 146086 2024-09-13T15:04:30.409 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.409 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:30.409 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.409 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:30.409 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376879+0000", 2024-09-13T15:04:30.409 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.409 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.409 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.409 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:30.409 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376879+0000", 2024-09-13T15:04:30.410 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.410 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.410 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.410 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:30.410 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376879+0000", 2024-09-13T15:04:30.410 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:30.410 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.410 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.410 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:30.411 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376883+0000", 2024-09-13T15:04:30.411 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.2989999999999998e-06 2024-09-13T15:04:30.411 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.411 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.411 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:30.411 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376890+0000", 2024-09-13T15:04:30.411 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2269999999999996e-06 2024-09-13T15:04:30.411 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.411 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.411 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:30.411 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.376895+0000", 2024-09-13T15:04:30.411 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6450000000000004e-06 2024-09-13T15:04:30.411 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.411 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.412 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:30.412 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.377044+0000", 2024-09-13T15:04:30.412 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014939900000000001 2024-09-13T15:04:30.412 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.412 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.412 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:30.412 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:30.414 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.377295+0000", 2024-09-13T15:04:30.414 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00025112499999999999 2024-09-13T15:04:30.414 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.414 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.414 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,4", 2024-09-13T15:04:30.414 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.377434+0000", 2024-09-13T15:04:30.414 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013815799999999999 2024-09-13T15:04:30.414 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.414 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.414 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:30.414 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.383888+0000", 2024-09-13T15:04:30.414 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0064541360000000001 2024-09-13T15:04:30.415 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.415 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.415 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:30.415 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.388160+0000", 2024-09-13T15:04:30.415 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0042726980000000001 2024-09-13T15:04:30.415 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.415 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:30.415 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.415 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.415 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.415 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:146091 10.9 10:972bc361:::benchmark_data_smithi096_288650_object10399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2024-09-13T15:04:30.415 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:30.381153+0000", 2024-09-13T15:04:30.415 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.012437099, 2024-09-13T15:04:30.416 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012700372999999999, 2024-09-13T15:04:30.416 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:30.416 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:04:30.416 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:30.416 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:30.416 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:30.416 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 146091 2024-09-13T15:04:30.416 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.416 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:30.416 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.416 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:30.416 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.381153+0000", 2024-09-13T15:04:30.416 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.416 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.417 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.417 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:30.417 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.381153+0000", 2024-09-13T15:04:30.417 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.417 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.417 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.417 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:30.417 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.381152+0000", 2024-09-13T15:04:30.417 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:30.418 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.418 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.418 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:30.418 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.381158+0000", 2024-09-13T15:04:30.418 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4819999999999999e-06 2024-09-13T15:04:30.418 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.418 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.418 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:30.418 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.381172+0000", 2024-09-13T15:04:30.418 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4411e-05 2024-09-13T15:04:30.418 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.418 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.419 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:30.419 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.381178+0000", 2024-09-13T15:04:30.419 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.0530000000000003e-06 2024-09-13T15:04:30.419 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.419 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.419 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:30.419 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387649+0000", 2024-09-13T15:04:30.419 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0064706039999999996 2024-09-13T15:04:30.419 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.419 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.419 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:30.419 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.388047+0000", 2024-09-13T15:04:30.419 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00039765500000000001 2024-09-13T15:04:30.420 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.420 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.420 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:04:30.420 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.388164+0000", 2024-09-13T15:04:30.420 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011686199999999999 2024-09-13T15:04:30.420 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.420 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:30.420 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.420 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.420 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.420 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:146096 10.3 10:ca1e3c70:::benchmark_data_smithi096_288650_object10404:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2024-09-13T15:04:30.420 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:30.387752+0000", 2024-09-13T15:04:30.420 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0058380799999999998, 2024-09-13T15:04:30.420 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.006165556, 2024-09-13T15:04:30.421 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:30.421 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:04:30.421 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:30.421 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:30.421 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:30.421 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 146096 2024-09-13T15:04:30.421 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.421 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:30.421 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.421 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:30.421 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387752+0000", 2024-09-13T15:04:30.421 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.421 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.421 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.422 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:30.422 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387752+0000", 2024-09-13T15:04:30.422 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.422 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.422 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.422 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:30.422 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387751+0000", 2024-09-13T15:04:30.422 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:30.422 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.422 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.423 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:30.423 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387757+0000", 2024-09-13T15:04:30.423 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6449999999999999e-06 2024-09-13T15:04:30.423 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.423 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.423 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:30.423 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387768+0000", 2024-09-13T15:04:30.423 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1107000000000001e-05 2024-09-13T15:04:30.423 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.423 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.423 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:30.423 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387774+0000", 2024-09-13T15:04:30.423 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8799999999999996e-06 2024-09-13T15:04:30.424 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.424 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.424 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:30.424 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387875+0000", 2024-09-13T15:04:30.424 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000100771 2024-09-13T15:04:30.424 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.424 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.424 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:30.424 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.391509+0000", 2024-09-13T15:04:30.424 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003633524 2024-09-13T15:04:30.424 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.424 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.424 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,4", 2024-09-13T15:04:30.425 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.391630+0000", 2024-09-13T15:04:30.425 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012170799999999999 2024-09-13T15:04:30.425 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.425 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:30.425 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.425 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.425 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.425 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:146098 10.9 10:9c062a6a:::benchmark_data_smithi096_288650_object10406:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2024-09-13T15:04:30.425 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:30.387789+0000", 2024-09-13T15:04:30.425 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0058011119999999998, 2024-09-13T15:04:30.425 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0061997270000000004, 2024-09-13T15:04:30.425 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:30.425 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:04:30.426 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:30.426 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:30.426 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:30.426 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 146098 2024-09-13T15:04:30.426 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.426 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:30.426 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.426 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:30.426 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387789+0000", 2024-09-13T15:04:30.426 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.426 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.426 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.426 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:30.426 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387789+0000", 2024-09-13T15:04:30.427 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.427 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.427 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.427 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:30.427 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387789+0000", 2024-09-13T15:04:30.427 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:30.427 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.427 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.427 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:30.427 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387792+0000", 2024-09-13T15:04:30.427 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7440000000000001e-06 2024-09-13T15:04:30.427 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.427 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.427 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:30.428 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387801+0000", 2024-09-13T15:04:30.428 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5499999999999995e-06 2024-09-13T15:04:30.428 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.428 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.428 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:30.428 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387805+0000", 2024-09-13T15:04:30.428 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3390000000000003e-06 2024-09-13T15:04:30.428 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.428 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.428 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:30.428 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.391941+0000", 2024-09-13T15:04:30.428 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0041353120000000004 2024-09-13T15:04:30.428 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.429 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.429 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:30.429 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.392199+0000", 2024-09-13T15:04:30.429 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000258328 2024-09-13T15:04:30.429 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.429 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.429 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:04:30.429 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.392308+0000", 2024-09-13T15:04:30.429 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000109182 2024-09-13T15:04:30.429 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.429 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:30.429 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.429 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.430 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.430 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.36552.0:146085 10.9 e371/355)", 2024-09-13T15:04:30.430 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:30.387823+0000", 2024-09-13T15:04:30.430 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0057674909999999996, 2024-09-13T15:04:30.430 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0062280199999999999, 2024-09-13T15:04:30.430 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:30.430 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "queued for pg", 2024-09-13T15:04:30.430 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:30.430 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.430 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:30.430 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387823+0000", 2024-09-13T15:04:30.431 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.431 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.431 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.431 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:30.431 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387823+0000", 2024-09-13T15:04:30.431 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.431 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.431 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.431 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:30.431 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387822+0000", 2024-09-13T15:04:30.431 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:30.431 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.431 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.431 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:30.432 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387826+0000", 2024-09-13T15:04:30.432 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5329999999999999e-06 2024-09-13T15:04:30.432 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.432 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.432 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:30.432 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387833+0000", 2024-09-13T15:04:30.432 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8190000000000003e-06 2024-09-13T15:04:30.432 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.432 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.432 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:30.432 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.387838+0000", 2024-09-13T15:04:30.432 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0130000000000001e-06 2024-09-13T15:04:30.432 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.433 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:30.433 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.433 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.433 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.433 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.36552.0:146081 10.9 e371/355)", 2024-09-13T15:04:30.433 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:30.388094+0000", 2024-09-13T15:04:30.433 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 0.0054960210000000002, 2024-09-13T15:04:30.433 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0059962940000000001, 2024-09-13T15:04:30.433 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:30.434 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "queued for pg", 2024-09-13T15:04:30.434 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:30.434 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.434 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:30.434 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.388094+0000", 2024-09-13T15:04:30.434 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.434 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.434 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.434 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:30.434 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.388094+0000", 2024-09-13T15:04:30.434 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:30.434 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.434 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.434 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:30.435 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.388094+0000", 2024-09-13T15:04:30.435 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:30.435 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.435 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.435 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:30.435 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.388098+0000", 2024-09-13T15:04:30.435 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4190000000000004e-06 2024-09-13T15:04:30.435 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.435 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.435 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:30.435 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.388110+0000", 2024-09-13T15:04:30.435 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1535000000000001e-05 2024-09-13T15:04:30.435 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:30.436 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:30.436 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:30.436 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:30.388116+0000", 2024-09-13T15:04:30.436 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9939999999999997e-06 2024-09-13T15:04:30.436 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.436 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:30.436 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.436 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:30.436 INFO:teuthology.orchestra.run.smithi204.stdout: ], 2024-09-13T15:04:30.436 INFO:teuthology.orchestra.run.smithi204.stdout: "num_ops": 8 2024-09-13T15:04:30.436 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:04:30.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.438 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:30.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:04:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:04:30.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:04:30.540 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:30.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.578 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:30.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:30.679 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:30.727 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:30.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:30.829 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:30.866 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:30.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:30.968 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:31.000 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:31.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:31.102 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:31.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.137 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:31.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:31.163 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T15:04:31.237 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:31.282 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:31.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:31.384 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:31.431 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:31.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:31.533 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:31.580 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:31.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:31.618 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:31.642 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T15:04:31.682 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:31.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.731 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:31.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:31.832 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:31.913 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:31 smithi096 ceph-mon[98901]: pgmap v1249: 119 pgs: 119 active+clean; 7.9 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 1.8 MiB/s wr, 773 op/s 2024-09-13T15:04:31.946 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:31.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:32.048 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:32.062 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:31 smithi204 ceph-mon[87071]: pgmap v1249: 119 pgs: 119 active+clean; 7.9 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 1.8 MiB/s wr, 773 op/s 2024-09-13T15:04:32.084 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:32.099 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:32.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:32.201 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:32.222 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:32.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:32.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:31 smithi096 ceph-mon[102928]: pgmap v1249: 119 pgs: 119 active+clean; 7.9 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 1.8 MiB/s wr, 773 op/s 2024-09-13T15:04:32.323 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:32.360 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:32.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:32.462 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:32.492 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:32.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:32.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:32.626 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:32.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:32.656 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:04:32.657 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:04:32.693 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:04:32.693 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:04:32.715 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:04:32.715 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:04:32.728 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:32.746 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:04:32.746 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:04:32.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.765 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:32.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:32.777 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:04:32.777 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:04:32.809 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:04:32.809 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:04:32.845 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:04:32.845 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:04:32.866 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:32.873 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:04:32.874 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:04:32.898 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:32.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:33.000 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:33.036 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:33.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:33.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:32 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1714179455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:33.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:32 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1714179455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:33.097 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:32 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1714179455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:33.138 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:33.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.174 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:33.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:33.276 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:33.307 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:33.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:33.409 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:33.445 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:33.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:33.547 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:33.572 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:33.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:33.602 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:04:33.603 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:04:33.674 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:33.710 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:33.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:33.813 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:33.847 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:33.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:33.949 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:33.987 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:33.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:34.089 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:34.133 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:34.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:34.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:33 smithi204 ceph-mon[87071]: pgmap v1250: 119 pgs: 119 active+clean; 7.6 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 1.5 MiB/s wr, 1.10k op/s 2024-09-13T15:04:34.235 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:34.264 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:34.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:34.305 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:33 smithi096 ceph-mon[98901]: pgmap v1250: 119 pgs: 119 active+clean; 7.6 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 1.5 MiB/s wr, 1.10k op/s 2024-09-13T15:04:34.306 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:33 smithi096 ceph-mon[102928]: pgmap v1250: 119 pgs: 119 active+clean; 7.6 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 1.5 MiB/s wr, 1.10k op/s 2024-09-13T15:04:34.330 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T15:04:34.365 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:34.404 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:34.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.411 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:34.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:34.512 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:34.541 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:34.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:34.642 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:34.669 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:34.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:34.770 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:34.773 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:34.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.835 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:34.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:34.936 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:34.974 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:34.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:35.076 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:35.105 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:35.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:35.206 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:35.246 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:35.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:35.347 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:35.383 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:35.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:35.485 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:35.520 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:35.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:35.622 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:35.656 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:35.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:35.758 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:35.792 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:35.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:35.894 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:35.930 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:35.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:36.032 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:36.062 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:36.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:36.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:35 smithi096 ceph-mon[98901]: pgmap v1251: 119 pgs: 119 active+clean; 7.6 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 877 KiB/s wr, 1.03k op/s 2024-09-13T15:04:36.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:04:36.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:35 smithi096 ceph-mon[102928]: pgmap v1251: 119 pgs: 119 active+clean; 7.6 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 877 KiB/s wr, 1.03k op/s 2024-09-13T15:04:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:04:36.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:36.164 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:36.209 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:36.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:36.216 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:35 smithi204 ceph-mon[87071]: pgmap v1251: 119 pgs: 119 active+clean; 7.6 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 877 KiB/s wr, 1.03k op/s 2024-09-13T15:04:36.216 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:04:36.216 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:36.311 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:36.339 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:36.340 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:04:36.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:36.444 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:36.474 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:36.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:36.576 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:36.612 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:36.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:36.692 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:04:36.692 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:04:36.715 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:36.750 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:36.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:36.851 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:36.887 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:36.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:36.955 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T15:04:36.989 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:37.019 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:37.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:37.121 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:37.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.155 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:37.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:37.177 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:36 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1979553662' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:37.177 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:36 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:37.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:36 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1979553662' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:37.177 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:36 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:36 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1979553662' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:36 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:37.257 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:37.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.294 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:37.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:37.395 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:37.425 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:37.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:37.488 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T15:04:37.527 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:37.565 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:37.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:37.667 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:37.712 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:37.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:37.814 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:37.842 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:37.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:37.934 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:37.943 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:37.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.999 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:37.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:38.100 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:38.129 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:38.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:37 smithi204 ceph-mon[87071]: pgmap v1252: 119 pgs: 119 active+clean; 7.5 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T15:04:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:37 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:37 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:37 smithi204 ceph-mon[87071]: osdmap e372: 8 total, 8 up, 7 in 2024-09-13T15:04:38.230 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:38.264 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:37 smithi096 ceph-mon[98901]: pgmap v1252: 119 pgs: 119 active+clean; 7.5 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T15:04:38.264 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:37 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:38.264 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:37 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:38.264 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:37 smithi096 ceph-mon[98901]: osdmap e372: 8 total, 8 up, 7 in 2024-09-13T15:04:38.264 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:04:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:04:37] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.51.0" 2024-09-13T15:04:38.265 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:37 smithi096 ceph-mon[102928]: pgmap v1252: 119 pgs: 119 active+clean; 7.5 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T15:04:38.265 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:37 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:38.265 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:37 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:38.265 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:37 smithi096 ceph-mon[102928]: osdmap e372: 8 total, 8 up, 7 in 2024-09-13T15:04:38.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.266 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:38.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:38.273 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:38.367 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:38.412 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:38.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:38.514 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:38.547 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:38.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:38.649 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:38.677 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:38.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:38.778 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:38.807 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:38.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:38.909 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:38.949 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:38.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:38.959 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T15:04:39.051 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:39.086 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:39.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:39.188 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:39.212 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:38 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3558672767' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:04:39.212 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:38 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:04:39.220 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:39.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:39.322 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:39.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:38 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3558672767' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:04:39.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:38 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:04:39.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:38 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3558672767' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:04:39.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:38 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:04:39.352 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:39.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:39.454 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:39.489 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:39.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:39.591 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:39.619 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:39.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:39.721 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:39.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.760 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:39.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:39.861 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:39.905 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:39.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:39.933 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:04:39.933 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:04:40.007 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:40.010 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:04:40.041 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:40.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:40.143 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:40.179 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:39 smithi096 ceph-mon[98901]: pgmap v1254: 119 pgs: 119 active+clean; 7.3 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-13T15:04:40.179 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:39 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:04:40.179 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:39 smithi096 ceph-mon[98901]: osdmap e373: 8 total, 8 up, 7 in 2024-09-13T15:04:40.180 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:39 smithi096 ceph-mon[102928]: pgmap v1254: 119 pgs: 119 active+clean; 7.3 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-13T15:04:40.180 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:39 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:04:40.180 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:39 smithi096 ceph-mon[102928]: osdmap e373: 8 total, 8 up, 7 in 2024-09-13T15:04:40.183 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:40.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:40.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:39 smithi204 ceph-mon[87071]: pgmap v1254: 119 pgs: 119 active+clean; 7.3 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-13T15:04:40.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:39 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:04:40.215 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:39 smithi204 ceph-mon[87071]: osdmap e373: 8 total, 8 up, 7 in 2024-09-13T15:04:40.219 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T15:04:40.286 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:40.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.325 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:40.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:40.430 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:04:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:04:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:04:40.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.476 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:40.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:40.577 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:40.607 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:40.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:40.642 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T15:04:40.709 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:40.743 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:40.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:40.799 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:04:40.799 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:04:40.813 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:40.833 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:04:40.833 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:04:40.846 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:40.866 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:04:40.866 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:04:40.884 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:40.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:40.887 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:04:40.887 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:04:40.916 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:04:40.916 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:04:40.941 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:04:40.941 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:04:40.957 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:40 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2684536282' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:40.973 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:04:40.974 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:04:40.986 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:41.000 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:40 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2684536282' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:41.001 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:40 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2684536282' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:41.005 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:04:41.005 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:04:41.019 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:41.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:41.095 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:41.121 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:41.173 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:41.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:41.275 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:41.313 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:41.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:41.415 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:41.441 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:41.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:41.543 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:41.570 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:41.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:41.672 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:41.705 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:41.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:41.806 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:41.851 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:41.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:41.953 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:41.984 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:41.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:42.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:41 smithi096 ceph-mon[98901]: pgmap v1256: 119 pgs: 119 active+clean; 7.2 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 886 op/s 2024-09-13T15:04:42.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:41 smithi096 ceph-mon[102928]: pgmap v1256: 119 pgs: 119 active+clean; 7.2 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 886 op/s 2024-09-13T15:04:42.086 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:42.122 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:42.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:42.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:41 smithi204 ceph-mon[87071]: pgmap v1256: 119 pgs: 119 active+clean; 7.2 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 886 op/s 2024-09-13T15:04:42.223 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:42.264 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:42.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:42.366 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:42.405 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:42.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:42.507 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:42.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.543 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:42.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:42.644 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:42.675 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:42.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:42.778 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:42.814 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:42.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:42.916 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:42.950 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:42.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:43.052 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:43.087 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:43.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:43.182 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:42 smithi096 ceph-mon[98901]: pgmap v1257: 119 pgs: 119 active+clean; 6.9 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 1.38k op/s 2024-09-13T15:04:43.182 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:42 smithi096 ceph-mon[102928]: pgmap v1257: 119 pgs: 119 active+clean; 6.9 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 1.38k op/s 2024-09-13T15:04:43.189 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:43.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:42 smithi204 ceph-mon[87071]: pgmap v1257: 119 pgs: 119 active+clean; 6.9 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 1.38k op/s 2024-09-13T15:04:43.234 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:43.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:43.336 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:43.361 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:43.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:43.463 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:43.492 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:43.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:43.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:43.597 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:43.646 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:43.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:43.748 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:43.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.791 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:43.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:43.892 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:43.918 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:43.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:44.013 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:04:44.013 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [], 2024-09-13T15:04:44.013 INFO:teuthology.orchestra.run.smithi204.stdout: "complaint_time": 30, 2024-09-13T15:04:44.013 INFO:teuthology.orchestra.run.smithi204.stdout: "num_blocked_ops": 0 2024-09-13T15:04:44.014 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:04:44.019 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:44.057 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:44.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:44.159 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:44.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.189 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:44.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:44.290 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:44.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.336 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:44.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:44.437 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:44.474 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:44.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:44.576 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:44.608 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:44.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:44.709 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:44.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.745 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:44.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:44.783 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T15:04:44.846 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:44.921 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:44.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:45.023 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:45.056 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:45.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:45.158 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:45.219 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:45.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:45.265 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:45.323 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:45.351 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:45.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:45.453 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:45.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.487 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:45.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:45.568 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:45 smithi096 ceph-mon[98901]: pgmap v1258: 119 pgs: 119 active+clean; 6.9 GiB data, 26 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 1.29k op/s 2024-09-13T15:04:45.569 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:45 smithi096 ceph-mon[102928]: pgmap v1258: 119 pgs: 119 active+clean; 6.9 GiB data, 26 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 1.29k op/s 2024-09-13T15:04:45.588 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:45.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.619 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:45.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:45.632 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:04:45.668 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:45 smithi204 ceph-mon[87071]: pgmap v1258: 119 pgs: 119 active+clean; 6.9 GiB data, 26 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 1.29k op/s 2024-09-13T15:04:45.721 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:45.777 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:45.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:45.879 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:45.928 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:45.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:46.030 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:46.065 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:46.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:46.167 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:46.205 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:46.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:46.307 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:46.344 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:46.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:46.388 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T15:04:46.446 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:46.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:46 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3760780998' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:04:46.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:46 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:04:46.480 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:46.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:46.535 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:46 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3760780998' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:04:46.535 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:46 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:04:46.535 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:46 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3760780998' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:04:46.535 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:46 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:04:46.583 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:46.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.632 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:46.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:46.733 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:46.785 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:46.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:46.842 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:46.887 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:46.933 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:46.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:47.034 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:47.085 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:47.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:47.158 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:04:47.158 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T15:04:47.158 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T15:04:47.158 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T15:04:47.158 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.158 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:224571 9.1 9:8f1c1396:::benchmark_data_smithi096_256909_object105163:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-09-13T15:04:47.158 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:32.156909+0000", 2024-09-13T15:04:47.158 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 194.994204844, 2024-09-13T15:04:47.158 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.14063650799999999, 2024-09-13T15:04:47.158 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.158 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.159 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.159 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:47.159 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:47.159 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 224571 2024-09-13T15:04:47.159 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.159 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.159 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.159 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.159 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.156909+0000", 2024-09-13T15:04:47.159 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.159 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.159 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.159 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.160 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.156909+0000", 2024-09-13T15:04:47.160 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.160 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.160 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.160 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.160 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.156907+0000", 2024-09-13T15:04:47.160 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:47.160 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.160 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.160 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.160 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.156914+0000", 2024-09-13T15:04:47.160 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8950000000000001e-06 2024-09-13T15:04:47.160 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.161 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.161 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.161 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.156937+0000", 2024-09-13T15:04:47.161 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.3108999999999998e-05 2024-09-13T15:04:47.161 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.161 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.161 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.161 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.156944+0000", 2024-09-13T15:04:47.161 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2019999999999998e-06 2024-09-13T15:04:47.161 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.161 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.161 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.161 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.156991+0000", 2024-09-13T15:04:47.161 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6414e-05 2024-09-13T15:04:47.161 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.162 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.162 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.162 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.157154+0000", 2024-09-13T15:04:47.162 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016353900000000001 2024-09-13T15:04:47.162 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.162 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.162 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:04:47.162 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.157298+0000", 2024-09-13T15:04:47.162 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014372199999999999 2024-09-13T15:04:47.162 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.162 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.162 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.162 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.160136+0000", 2024-09-13T15:04:47.163 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0028384009999999999 2024-09-13T15:04:47.163 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.163 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.163 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.163 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.160474+0000", 2024-09-13T15:04:47.163 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00033741100000000003 2024-09-13T15:04:47.163 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.163 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.163 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.163 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.296868+0000", 2024-09-13T15:04:47.163 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.13639464600000001 2024-09-13T15:04:47.163 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.163 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.164 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.164 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.296956+0000", 2024-09-13T15:04:47.164 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7497000000000006e-05 2024-09-13T15:04:47.164 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.164 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.164 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.164 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.297545+0000", 2024-09-13T15:04:47.164 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00058905700000000004 2024-09-13T15:04:47.164 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.164 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.164 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.164 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.164 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.164 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:224574 9.1 9:8908acb2:::benchmark_data_smithi096_256909_object105166:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-09-13T15:04:47.164 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:32.161708+0000", 2024-09-13T15:04:47.165 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 194.989405229, 2024-09-13T15:04:47.165 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.13641946099999999, 2024-09-13T15:04:47.165 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.165 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.165 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.165 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:47.165 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:47.165 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 224574 2024-09-13T15:04:47.165 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.165 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.165 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.165 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.165 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.161708+0000", 2024-09-13T15:04:47.165 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.165 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.161708+0000", 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.161706+0000", 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999976 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.161718+0000", 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.219e-05 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.166 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.167 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.167 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.161742+0000", 2024-09-13T15:04:47.167 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4329e-05 2024-09-13T15:04:47.167 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.167 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.167 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.167 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.161755+0000", 2024-09-13T15:04:47.167 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2796000000000001e-05 2024-09-13T15:04:47.167 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.167 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.167 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.167 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.161868+0000", 2024-09-13T15:04:47.167 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000113066 2024-09-13T15:04:47.167 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.167 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.168 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.168 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.161994+0000", 2024-09-13T15:04:47.168 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012584799999999999 2024-09-13T15:04:47.168 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.168 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.168 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:04:47.168 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.162077+0000", 2024-09-13T15:04:47.168 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2917000000000006e-05 2024-09-13T15:04:47.168 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.168 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.168 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.168 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.165236+0000", 2024-09-13T15:04:47.168 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0031586930000000002 2024-09-13T15:04:47.168 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.168 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.169 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.169 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.165694+0000", 2024-09-13T15:04:47.169 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00045864499999999998 2024-09-13T15:04:47.169 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.169 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.169 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.169 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.297583+0000", 2024-09-13T15:04:47.169 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.13188907899999999 2024-09-13T15:04:47.169 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.169 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.169 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.169 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.297729+0000", 2024-09-13T15:04:47.169 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014528300000000001 2024-09-13T15:04:47.169 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.169 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.170 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.170 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.298128+0000", 2024-09-13T15:04:47.170 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00039898100000000002 2024-09-13T15:04:47.170 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.170 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.170 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.170 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.170 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.170 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:224587 9.1 9:82b65f6c:::benchmark_data_smithi096_256909_object105179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-09-13T15:04:47.170 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:32.178300+0000", 2024-09-13T15:04:47.170 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 194.97281313100001, 2024-09-13T15:04:47.170 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.120287001, 2024-09-13T15:04:47.170 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.170 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.170 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.171 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:47.171 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:47.171 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 224587 2024-09-13T15:04:47.171 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.171 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.171 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.171 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.171 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.178300+0000", 2024-09-13T15:04:47.171 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.171 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.171 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.171 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.171 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.178300+0000", 2024-09-13T15:04:47.172 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.172 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.172 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.172 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.172 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.178299+0000", 2024-09-13T15:04:47.172 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:47.172 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.172 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.172 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.172 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.178305+0000", 2024-09-13T15:04:47.172 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3080000000000004e-06 2024-09-13T15:04:47.172 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.172 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.172 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.172 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.178319+0000", 2024-09-13T15:04:47.173 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3518e-05 2024-09-13T15:04:47.173 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.173 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.173 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.173 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.178324+0000", 2024-09-13T15:04:47.173 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9420000000000003e-06 2024-09-13T15:04:47.173 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.173 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.173 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.173 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.178365+0000", 2024-09-13T15:04:47.173 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1050000000000002e-05 2024-09-13T15:04:47.173 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.173 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.173 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.174 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.178466+0000", 2024-09-13T15:04:47.174 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000101479 2024-09-13T15:04:47.174 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.174 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.174 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:04:47.174 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.178600+0000", 2024-09-13T15:04:47.174 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000134299 2024-09-13T15:04:47.174 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.174 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.174 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.174 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.181034+0000", 2024-09-13T15:04:47.174 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002433504 2024-09-13T15:04:47.174 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.174 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.174 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.175 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.181237+0000", 2024-09-13T15:04:47.175 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000203597 2024-09-13T15:04:47.175 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.175 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.175 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.175 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.298164+0000", 2024-09-13T15:04:47.175 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.11692651599999999 2024-09-13T15:04:47.175 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.175 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.175 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.175 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.298306+0000", 2024-09-13T15:04:47.175 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014188699999999999 2024-09-13T15:04:47.175 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.175 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.176 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.176 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.298587+0000", 2024-09-13T15:04:47.176 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00028136600000000002 2024-09-13T15:04:47.176 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.176 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.176 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.176 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.176 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.176 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:224632 9.1 9:81fa3417:::benchmark_data_smithi096_256909_object105224:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-09-13T15:04:47.176 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:01:32.299007+0000", 2024-09-13T15:04:47.176 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 194.85210639499999, 2024-09-13T15:04:47.176 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.118543944, 2024-09-13T15:04:47.176 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.176 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.176 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.177 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:04:47.177 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:04:47.177 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 224632 2024-09-13T15:04:47.177 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.177 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.177 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.177 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.177 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.299007+0000", 2024-09-13T15:04:47.177 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.177 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.177 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.177 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.177 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.299007+0000", 2024-09-13T15:04:47.177 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.177 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.178 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.178 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.178 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.299006+0000", 2024-09-13T15:04:47.178 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:47.178 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.178 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.178 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.178 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.299012+0000", 2024-09-13T15:04:47.178 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.421e-06 2024-09-13T15:04:47.178 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.178 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.178 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.178 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.299033+0000", 2024-09-13T15:04:47.178 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1356999999999999e-05 2024-09-13T15:04:47.179 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.179 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.179 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.179 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.299039+0000", 2024-09-13T15:04:47.179 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5899999999999998e-06 2024-09-13T15:04:47.179 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.179 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.179 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.179 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.300533+0000", 2024-09-13T15:04:47.179 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00149449 2024-09-13T15:04:47.179 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.179 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.179 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.179 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.300676+0000", 2024-09-13T15:04:47.179 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014245100000000001 2024-09-13T15:04:47.180 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.180 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.180 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:04:47.180 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.300817+0000", 2024-09-13T15:04:47.180 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014151100000000001 2024-09-13T15:04:47.180 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.180 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.180 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.180 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.306174+0000", 2024-09-13T15:04:47.180 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0053561449999999997 2024-09-13T15:04:47.180 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.180 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.180 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.181 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.308271+0000", 2024-09-13T15:04:47.181 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0020976229999999998 2024-09-13T15:04:47.181 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.181 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.181 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.181 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.416900+0000", 2024-09-13T15:04:47.181 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.108628556 2024-09-13T15:04:47.181 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.181 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.181 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.181 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.416966+0000", 2024-09-13T15:04:47.181 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6680000000000005e-05 2024-09-13T15:04:47.181 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.181 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.182 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.182 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:01:32.417551+0000", 2024-09-13T15:04:47.182 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00058445000000000003 2024-09-13T15:04:47.182 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.182 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.182 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.182 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.182 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.182 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:68762 10.a 10:5c1502e1:::benchmark_data_smithi096_288650_object68761:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2024-09-13T15:04:47.182 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:03:22.360176+0000", 2024-09-13T15:04:47.182 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 84.790937842999995, 2024-09-13T15:04:47.182 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.115694464, 2024-09-13T15:04:47.182 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.182 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.183 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.183 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:47.183 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:47.183 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 68762 2024-09-13T15:04:47.183 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.183 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.183 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.183 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.183 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.360176+0000", 2024-09-13T15:04:47.183 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.183 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.183 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.183 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.183 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.360176+0000", 2024-09-13T15:04:47.184 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.184 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.184 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.184 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.184 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.360174+0000", 2024-09-13T15:04:47.184 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:47.184 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.184 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.184 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.184 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.360203+0000", 2024-09-13T15:04:47.184 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8445e-05 2024-09-13T15:04:47.184 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.184 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.185 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.185 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.360212+0000", 2024-09-13T15:04:47.185 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.764e-06 2024-09-13T15:04:47.185 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.185 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.185 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.185 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.360218+0000", 2024-09-13T15:04:47.185 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4550000000000003e-06 2024-09-13T15:04:47.185 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.185 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.185 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.185 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.366146+0000", 2024-09-13T15:04:47.185 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.005928465 2024-09-13T15:04:47.185 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.186 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.186 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.186 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.366260+0000", 2024-09-13T15:04:47.186 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011349999999999999 2024-09-13T15:04:47.186 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.186 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.186 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:04:47.186 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.366387+0000", 2024-09-13T15:04:47.186 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012729700000000001 2024-09-13T15:04:47.186 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.186 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.186 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.186 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.371522+0000", 2024-09-13T15:04:47.186 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0051345849999999997 2024-09-13T15:04:47.187 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.187 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.187 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.187 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.371533+0000", 2024-09-13T15:04:47.187 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0997e-05 2024-09-13T15:04:47.187 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.187 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.187 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.187 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.475760+0000", 2024-09-13T15:04:47.187 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.104227284 2024-09-13T15:04:47.187 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.187 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.187 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.187 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.475830+0000", 2024-09-13T15:04:47.187 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9898000000000001e-05 2024-09-13T15:04:47.188 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.188 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.188 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.188 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.475870+0000", 2024-09-13T15:04:47.188 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0159e-05 2024-09-13T15:04:47.188 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.188 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.188 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.188 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.188 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.188 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:68771 10.a 10:55ee1d2e:::benchmark_data_smithi096_288650_object68770:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2024-09-13T15:04:47.188 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:03:22.365546+0000", 2024-09-13T15:04:47.188 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 84.785567037000007, 2024-09-13T15:04:47.188 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.110565976, 2024-09-13T15:04:47.188 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.189 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.189 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.189 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:47.189 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:47.189 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 68771 2024-09-13T15:04:47.189 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.189 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.189 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.189 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.189 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.365546+0000", 2024-09-13T15:04:47.189 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.189 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.189 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.190 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.190 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.365546+0000", 2024-09-13T15:04:47.190 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.190 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.190 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.190 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.190 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.365545+0000", 2024-09-13T15:04:47.190 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:04:47.190 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.190 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.190 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.190 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.365583+0000", 2024-09-13T15:04:47.190 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7897e-05 2024-09-13T15:04:47.190 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.191 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.191 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.191 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.365599+0000", 2024-09-13T15:04:47.191 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6121999999999999e-05 2024-09-13T15:04:47.191 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.191 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.191 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.191 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.365605+0000", 2024-09-13T15:04:47.191 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6860000000000003e-06 2024-09-13T15:04:47.191 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.191 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.191 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.191 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.368990+0000", 2024-09-13T15:04:47.191 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003384883 2024-09-13T15:04:47.192 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.192 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.192 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.192 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.369092+0000", 2024-09-13T15:04:47.192 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010180500000000001 2024-09-13T15:04:47.192 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.192 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.192 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:04:47.192 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.369207+0000", 2024-09-13T15:04:47.192 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011547799999999999 2024-09-13T15:04:47.192 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.192 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.192 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.192 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.373011+0000", 2024-09-13T15:04:47.192 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0038038 2024-09-13T15:04:47.193 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.193 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.193 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.193 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.373290+0000", 2024-09-13T15:04:47.193 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00027917099999999999 2024-09-13T15:04:47.193 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.193 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.193 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.193 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.476026+0000", 2024-09-13T15:04:47.193 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.102735936 2024-09-13T15:04:47.193 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.193 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.193 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.193 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.476078+0000", 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2469999999999997e-05 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.476112+0000", 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3837000000000001e-05 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:68775 10.d 10:b2784b21:::benchmark_data_smithi096_288650_object68774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:03:22.367835+0000", 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 84.783277999000006, 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.11732701, 2024-09-13T15:04:47.194 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 68775 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.367835+0000", 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.367835+0000", 2024-09-13T15:04:47.195 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.196 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.196 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.196 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.196 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.367834+0000", 2024-09-13T15:04:47.196 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:47.196 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.196 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.196 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.196 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.367870+0000", 2024-09-13T15:04:47.196 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6508999999999999e-05 2024-09-13T15:04:47.196 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.196 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.196 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.196 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.367884+0000", 2024-09-13T15:04:47.197 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3241000000000001e-05 2024-09-13T15:04:47.197 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.197 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.197 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.197 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.367890+0000", 2024-09-13T15:04:47.197 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8510000000000001e-06 2024-09-13T15:04:47.197 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.197 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.197 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.197 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.367947+0000", 2024-09-13T15:04:47.197 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6687000000000001e-05 2024-09-13T15:04:47.197 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.197 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.198 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:47.199 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.199 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.368063+0000", 2024-09-13T15:04:47.199 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000115685 2024-09-13T15:04:47.199 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.199 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.199 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,4", 2024-09-13T15:04:47.199 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.368466+0000", 2024-09-13T15:04:47.199 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00040299599999999997 2024-09-13T15:04:47.199 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.200 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.200 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.200 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.371445+0000", 2024-09-13T15:04:47.200 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029788179999999998 2024-09-13T15:04:47.200 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.200 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.200 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.200 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.373484+0000", 2024-09-13T15:04:47.200 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002039782 2024-09-13T15:04:47.200 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.200 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.200 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.200 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.485075+0000", 2024-09-13T15:04:47.201 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.111590629 2024-09-13T15:04:47.201 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.201 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.201 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.201 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.485124+0000", 2024-09-13T15:04:47.201 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9345999999999998e-05 2024-09-13T15:04:47.201 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.201 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.201 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.201 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.485162+0000", 2024-09-13T15:04:47.201 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7972000000000002e-05 2024-09-13T15:04:47.201 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.201 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.201 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.202 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.202 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.202 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:68782 10.d 10:bc7f16d0:::benchmark_data_smithi096_288650_object68781:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2024-09-13T15:04:47.202 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:03:22.374758+0000", 2024-09-13T15:04:47.202 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 84.776355218000006, 2024-09-13T15:04:47.202 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.11098612400000001, 2024-09-13T15:04:47.202 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.202 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.202 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.202 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:47.202 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:47.202 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 68782 2024-09-13T15:04:47.202 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.202 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.203 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.203 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.203 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.374758+0000", 2024-09-13T15:04:47.203 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.203 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.203 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.203 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.203 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.374758+0000", 2024-09-13T15:04:47.203 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.203 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.203 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.203 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.203 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.374757+0000", 2024-09-13T15:04:47.203 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:04:47.203 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.204 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.204 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.204 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.374807+0000", 2024-09-13T15:04:47.204 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9798e-05 2024-09-13T15:04:47.204 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.204 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.204 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.204 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.374814+0000", 2024-09-13T15:04:47.204 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0740000000000004e-06 2024-09-13T15:04:47.204 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.204 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.204 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.204 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.374818+0000", 2024-09-13T15:04:47.204 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7400000000000002e-06 2024-09-13T15:04:47.205 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.205 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.205 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.205 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.374849+0000", 2024-09-13T15:04:47.205 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.0834999999999999e-05 2024-09-13T15:04:47.205 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.205 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.205 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.205 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.374952+0000", 2024-09-13T15:04:47.205 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000103708 2024-09-13T15:04:47.205 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.205 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.205 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,4", 2024-09-13T15:04:47.206 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.375054+0000", 2024-09-13T15:04:47.206 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000102001 2024-09-13T15:04:47.206 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.206 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.206 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.206 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.376889+0000", 2024-09-13T15:04:47.206 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0018346860000000001 2024-09-13T15:04:47.206 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.206 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.206 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.206 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.377118+0000", 2024-09-13T15:04:47.206 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000228667 2024-09-13T15:04:47.206 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.207 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.207 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.207 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.485608+0000", 2024-09-13T15:04:47.207 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.10849081200000001 2024-09-13T15:04:47.207 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.207 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.207 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.207 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.485653+0000", 2024-09-13T15:04:47.207 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4301000000000002e-05 2024-09-13T15:04:47.207 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.207 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.207 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.207 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:22.485744+0000", 2024-09-13T15:04:47.208 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1520999999999993e-05 2024-09-13T15:04:47.208 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.208 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.208 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.208 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.208 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.208 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:69685 10.a 10:5e1e00a5:::benchmark_data_smithi096_288650_object69684:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2024-09-13T15:04:47.208 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:03:23.118831+0000", 2024-09-13T15:04:47.208 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 84.032282269999996, 2024-09-13T15:04:47.208 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.12993777200000001, 2024-09-13T15:04:47.208 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.208 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.208 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.208 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:47.209 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:47.209 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 69685 2024-09-13T15:04:47.209 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.209 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.209 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.209 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.209 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.118831+0000", 2024-09-13T15:04:47.209 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.209 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.209 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.209 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.209 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.118831+0000", 2024-09-13T15:04:47.209 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.209 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.209 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.210 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.210 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.118830+0000", 2024-09-13T15:04:47.210 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:47.210 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.210 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.210 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.210 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.118886+0000", 2024-09-13T15:04:47.210 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5924999999999997e-05 2024-09-13T15:04:47.210 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.210 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.210 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.210 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.118898+0000", 2024-09-13T15:04:47.210 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2523e-05 2024-09-13T15:04:47.210 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.210 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.211 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.211 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.118904+0000", 2024-09-13T15:04:47.211 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9490000000000001e-06 2024-09-13T15:04:47.211 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.211 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.211 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.211 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.126117+0000", 2024-09-13T15:04:47.211 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0072129569999999999 2024-09-13T15:04:47.211 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.211 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.211 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.211 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.126237+0000", 2024-09-13T15:04:47.211 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000120044 2024-09-13T15:04:47.211 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.211 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.212 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:04:47.212 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.126505+0000", 2024-09-13T15:04:47.212 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00026828000000000003 2024-09-13T15:04:47.212 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.212 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.212 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.212 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.134748+0000", 2024-09-13T15:04:47.212 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0082427319999999991 2024-09-13T15:04:47.212 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.212 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.212 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.212 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135172+0000", 2024-09-13T15:04:47.212 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000423962 2024-09-13T15:04:47.212 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.213 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.213 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.213 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135200+0000", 2024-09-13T15:04:47.213 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7489000000000001e-05 2024-09-13T15:04:47.213 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.213 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.213 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.213 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135249+0000", 2024-09-13T15:04:47.213 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9030000000000003e-05 2024-09-13T15:04:47.213 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.213 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.213 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.213 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248769+0000", 2024-09-13T15:04:47.213 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.113520158 2024-09-13T15:04:47.214 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.214 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.214 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.214 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.214 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.214 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:69689 10.a 10:56bc2790:::benchmark_data_smithi096_288650_object69688:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2024-09-13T15:04:47.214 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:03:23.123688+0000", 2024-09-13T15:04:47.214 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 84.027425162, 2024-09-13T15:04:47.214 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.12531210600000001, 2024-09-13T15:04:47.214 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.214 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.214 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.214 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:47.214 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:47.215 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 69689 2024-09-13T15:04:47.215 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.215 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.215 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.215 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.215 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.123688+0000", 2024-09-13T15:04:47.215 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.215 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.215 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.215 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.215 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.123688+0000", 2024-09-13T15:04:47.215 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.215 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.215 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.123687+0000", 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.123728+0000", 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1097000000000003e-05 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.123742+0000", 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3619e-05 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.216 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.217 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.123747+0000", 2024-09-13T15:04:47.217 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5890000000000002e-06 2024-09-13T15:04:47.217 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.217 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.217 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.217 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.129830+0000", 2024-09-13T15:04:47.217 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0060825080000000004 2024-09-13T15:04:47.217 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.217 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.217 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.217 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.129952+0000", 2024-09-13T15:04:47.217 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012264 2024-09-13T15:04:47.217 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.217 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.218 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:04:47.218 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.130078+0000", 2024-09-13T15:04:47.218 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012534199999999999 2024-09-13T15:04:47.218 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.218 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.218 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.218 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248849+0000", 2024-09-13T15:04:47.218 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.118771064 2024-09-13T15:04:47.218 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.218 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.218 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.218 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248860+0000", 2024-09-13T15:04:47.218 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0983e-05 2024-09-13T15:04:47.218 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.218 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.219 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.219 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248912+0000", 2024-09-13T15:04:47.219 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1867999999999999e-05 2024-09-13T15:04:47.219 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.219 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.219 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.219 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248965+0000", 2024-09-13T15:04:47.219 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3822000000000001e-05 2024-09-13T15:04:47.219 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.219 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.219 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.219 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.249000+0000", 2024-09-13T15:04:47.219 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4916000000000002e-05 2024-09-13T15:04:47.220 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.220 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.220 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.220 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.220 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.220 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:69702 10.f 10:f104961d:::benchmark_data_smithi096_288650_object69701:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2024-09-13T15:04:47.220 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:03:23.131888+0000", 2024-09-13T15:04:47.220 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 84.019224894000004, 2024-09-13T15:04:47.220 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.119430865, 2024-09-13T15:04:47.220 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.220 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.220 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.220 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:47.220 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:47.221 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 69702 2024-09-13T15:04:47.221 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.221 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.221 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.221 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.221 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.131888+0000", 2024-09-13T15:04:47.221 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.221 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.221 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.221 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.221 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.131888+0000", 2024-09-13T15:04:47.221 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.221 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.221 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.221 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.222 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.131887+0000", 2024-09-13T15:04:47.222 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:47.222 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.222 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.222 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.222 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.131933+0000", 2024-09-13T15:04:47.222 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6181999999999999e-05 2024-09-13T15:04:47.222 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.222 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.222 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.222 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.131946+0000", 2024-09-13T15:04:47.222 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3191e-05 2024-09-13T15:04:47.222 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.222 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.223 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.223 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.131953+0000", 2024-09-13T15:04:47.223 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3860000000000004e-06 2024-09-13T15:04:47.223 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.223 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.223 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.223 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132102+0000", 2024-09-13T15:04:47.223 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000149102 2024-09-13T15:04:47.223 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.223 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.223 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.223 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132202+0000", 2024-09-13T15:04:47.223 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9735000000000006e-05 2024-09-13T15:04:47.223 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.223 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.224 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,4", 2024-09-13T15:04:47.224 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132333+0000", 2024-09-13T15:04:47.224 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013190899999999999 2024-09-13T15:04:47.224 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.224 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.224 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.224 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.251139+0000", 2024-09-13T15:04:47.224 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.118805613 2024-09-13T15:04:47.224 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.224 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.224 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.224 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.251214+0000", 2024-09-13T15:04:47.224 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5400000000000003e-05 2024-09-13T15:04:47.224 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.224 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.225 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.225 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.251244+0000", 2024-09-13T15:04:47.225 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.9028000000000001e-05 2024-09-13T15:04:47.225 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.225 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.225 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.225 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.251284+0000", 2024-09-13T15:04:47.225 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0601e-05 2024-09-13T15:04:47.225 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.225 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.225 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.225 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.251319+0000", 2024-09-13T15:04:47.225 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5207999999999999e-05 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:69703 10.0 10:01b1f82f:::benchmark_data_smithi096_288650_object69702:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:03:23.131968+0000", 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 84.019145511999994, 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.12125338099999999, 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 69703 2024-09-13T15:04:47.226 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.227 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.227 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.227 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.227 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.131968+0000", 2024-09-13T15:04:47.227 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.227 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.227 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.227 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.227 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.131968+0000", 2024-09-13T15:04:47.227 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.227 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.227 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.227 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.227 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.131967+0000", 2024-09-13T15:04:47.227 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:47.228 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.228 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.228 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.228 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.131999+0000", 2024-09-13T15:04:47.228 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1832999999999998e-05 2024-09-13T15:04:47.228 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.228 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.228 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.228 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132007+0000", 2024-09-13T15:04:47.228 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8010000000000004e-06 2024-09-13T15:04:47.228 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.228 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.228 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.228 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132011+0000", 2024-09-13T15:04:47.228 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3089999999999997e-06 2024-09-13T15:04:47.229 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.229 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.229 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.229 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132076+0000", 2024-09-13T15:04:47.229 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4794000000000003e-05 2024-09-13T15:04:47.229 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.229 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.229 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.229 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132502+0000", 2024-09-13T15:04:47.229 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00042601599999999998 2024-09-13T15:04:47.229 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.229 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.229 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:04:47.229 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132847+0000", 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000345163 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.251004+0000", 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.11815701100000001 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.253128+0000", 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0021235099999999999 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.253138+0000", 2024-09-13T15:04:47.230 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0180000000000001e-05 2024-09-13T15:04:47.231 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.231 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.231 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.231 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.253186+0000", 2024-09-13T15:04:47.231 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8031999999999997e-05 2024-09-13T15:04:47.231 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.231 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.231 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.231 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.253221+0000", 2024-09-13T15:04:47.231 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5311000000000001e-05 2024-09-13T15:04:47.231 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.231 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.231 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.231 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:69706 10.f 10:f3360246:::benchmark_data_smithi096_288650_object69705:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:03:23.132169+0000", 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 84.018943863999993, 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.12994407499999999, 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 69706 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.232 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132169+0000", 2024-09-13T15:04:47.233 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.233 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.233 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.233 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.233 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132169+0000", 2024-09-13T15:04:47.233 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.233 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.233 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.233 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.233 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132169+0000", 2024-09-13T15:04:47.233 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:04:47.233 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.233 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.233 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.233 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132309+0000", 2024-09-13T15:04:47.234 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014011000000000001 2024-09-13T15:04:47.234 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.234 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.234 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.234 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132330+0000", 2024-09-13T15:04:47.234 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1043000000000002e-05 2024-09-13T15:04:47.234 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.234 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.234 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.234 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132335+0000", 2024-09-13T15:04:47.234 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.3959999999999997e-06 2024-09-13T15:04:47.234 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.234 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.234 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.235 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135215+0000", 2024-09-13T15:04:47.235 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0028803069999999999 2024-09-13T15:04:47.235 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.235 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.235 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.235 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248441+0000", 2024-09-13T15:04:47.235 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.113225478 2024-09-13T15:04:47.235 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.235 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.235 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,4", 2024-09-13T15:04:47.235 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248575+0000", 2024-09-13T15:04:47.235 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013424500000000001 2024-09-13T15:04:47.235 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.235 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.235 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.236 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.251328+0000", 2024-09-13T15:04:47.236 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0027530520000000002 2024-09-13T15:04:47.236 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.236 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.236 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.236 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.251357+0000", 2024-09-13T15:04:47.236 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8430000000000001e-05 2024-09-13T15:04:47.236 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.236 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.236 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.236 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.262033+0000", 2024-09-13T15:04:47.236 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010676066999999999 2024-09-13T15:04:47.236 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.236 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.236 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.237 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.262080+0000", 2024-09-13T15:04:47.237 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7416000000000001e-05 2024-09-13T15:04:47.237 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.237 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.237 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.237 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.262114+0000", 2024-09-13T15:04:47.237 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3376e-05 2024-09-13T15:04:47.237 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.237 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.237 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.237 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.237 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.237 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:69708 10.0 10:0d296375:::benchmark_data_smithi096_288650_object69707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2024-09-13T15:04:47.237 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:03:23.132341+0000", 2024-09-13T15:04:47.237 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 84.018772026999997, 2024-09-13T15:04:47.238 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.128205349, 2024-09-13T15:04:47.238 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.238 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.238 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.238 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:47.239 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:47.239 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 69708 2024-09-13T15:04:47.239 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.239 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.239 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.239 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.239 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132341+0000", 2024-09-13T15:04:47.239 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.239 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.239 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.239 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.239 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132341+0000", 2024-09-13T15:04:47.239 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.239 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.240 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.240 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.240 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132341+0000", 2024-09-13T15:04:47.240 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:47.240 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.240 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.240 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.240 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132373+0000", 2024-09-13T15:04:47.240 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2642999999999999e-05 2024-09-13T15:04:47.240 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.240 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.240 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.240 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132380+0000", 2024-09-13T15:04:47.240 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4479999999999999e-06 2024-09-13T15:04:47.241 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.241 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.241 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.241 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.132384+0000", 2024-09-13T15:04:47.241 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5889999999999996e-06 2024-09-13T15:04:47.241 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.241 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.241 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.241 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248052+0000", 2024-09-13T15:04:47.241 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.115668018 2024-09-13T15:04:47.241 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.241 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.241 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.242 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248185+0000", 2024-09-13T15:04:47.242 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013266299999999999 2024-09-13T15:04:47.242 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.242 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.242 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:04:47.242 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248317+0000", 2024-09-13T15:04:47.242 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013196599999999999 2024-09-13T15:04:47.242 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.242 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.242 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.242 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.253229+0000", 2024-09-13T15:04:47.242 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0049116180000000004 2024-09-13T15:04:47.242 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.242 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.242 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.243 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.254621+0000", 2024-09-13T15:04:47.243 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001392412 2024-09-13T15:04:47.243 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.243 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.243 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.243 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.260436+0000", 2024-09-13T15:04:47.243 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0058145000000000002 2024-09-13T15:04:47.243 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.243 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.243 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.243 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.260510+0000", 2024-09-13T15:04:47.243 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.4472000000000001e-05 2024-09-13T15:04:47.243 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.243 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.243 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.244 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.260547+0000", 2024-09-13T15:04:47.244 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6640999999999997e-05 2024-09-13T15:04:47.244 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.244 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.244 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.244 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.244 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.244 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.36552.0:69685 10.a e362/355 ondisk, result = 0)", 2024-09-13T15:04:47.244 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:03:23.135014+0000", 2024-09-13T15:04:47.244 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 84.016099198999996, 2024-09-13T15:04:47.244 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.113766797, 2024-09-13T15:04:47.244 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.244 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:04:47.244 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.244 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.245 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.245 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135014+0000", 2024-09-13T15:04:47.245 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.245 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.245 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.245 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.245 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135014+0000", 2024-09-13T15:04:47.245 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.245 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.245 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.245 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.245 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135014+0000", 2024-09-13T15:04:47.245 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:47.245 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.245 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.246 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.246 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135017+0000", 2024-09-13T15:04:47.246 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.309e-06 2024-09-13T15:04:47.246 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.246 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.246 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.246 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135025+0000", 2024-09-13T15:04:47.246 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0579999999999998e-06 2024-09-13T15:04:47.246 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.246 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.246 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.246 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135029+0000", 2024-09-13T15:04:47.246 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1080000000000003e-06 2024-09-13T15:04:47.246 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135190+0000", 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00016124800000000001 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135195+0000", 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.8810000000000004e-06 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248781+0000", 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.11358563300000001 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.248 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.248 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.248 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.248 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.36552.0:69702 10.f e362/355 ondisk, result = 0)", 2024-09-13T15:04:47.248 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:03:23.135028+0000", 2024-09-13T15:04:47.248 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 84.016085692000004, 2024-09-13T15:04:47.248 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.116113987, 2024-09-13T15:04:47.248 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.248 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:04:47.248 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.248 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.248 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.248 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135028+0000", 2024-09-13T15:04:47.248 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.248 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.249 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.249 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.249 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135028+0000", 2024-09-13T15:04:47.249 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.249 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.249 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.249 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.249 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135027+0000", 2024-09-13T15:04:47.249 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:04:47.249 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.249 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.249 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.249 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135035+0000", 2024-09-13T15:04:47.249 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4770000000000003e-06 2024-09-13T15:04:47.250 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.250 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.250 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.250 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135046+0000", 2024-09-13T15:04:47.250 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1158999999999999e-05 2024-09-13T15:04:47.250 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.250 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.250 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.250 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135052+0000", 2024-09-13T15:04:47.250 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.074e-06 2024-09-13T15:04:47.250 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.250 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.250 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.250 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.251127+0000", 2024-09-13T15:04:47.250 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.116075096 2024-09-13T15:04:47.251 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.251 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.251 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.251 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.251133+0000", 2024-09-13T15:04:47.251 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.6919999999999996e-06 2024-09-13T15:04:47.251 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.251 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.251 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.251 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.251142+0000", 2024-09-13T15:04:47.251 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3550000000000005e-06 2024-09-13T15:04:47.251 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.251 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.251 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.251 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.251 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.252 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.36552.0:69689 10.a e362/355 ondisk, result = 0)", 2024-09-13T15:04:47.252 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:03:23.135097+0000", 2024-09-13T15:04:47.252 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 84.016016022000002, 2024-09-13T15:04:47.252 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.113765962, 2024-09-13T15:04:47.252 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.252 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:04:47.252 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.252 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.252 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.252 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135097+0000", 2024-09-13T15:04:47.252 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.252 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.252 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.252 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.253 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135097+0000", 2024-09-13T15:04:47.253 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.253 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.253 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.253 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.253 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135097+0000", 2024-09-13T15:04:47.253 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:47.253 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.253 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.253 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.253 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135100+0000", 2024-09-13T15:04:47.253 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.3629999999999998e-06 2024-09-13T15:04:47.253 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.253 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.253 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.254 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135108+0000", 2024-09-13T15:04:47.254 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0469999999999994e-06 2024-09-13T15:04:47.254 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.254 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.254 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.254 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135113+0000", 2024-09-13T15:04:47.254 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.3019999999999996e-06 2024-09-13T15:04:47.254 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.254 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.254 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.254 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248835+0000", 2024-09-13T15:04:47.254 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.11372236199999999 2024-09-13T15:04:47.254 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.254 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.254 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.255 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248843+0000", 2024-09-13T15:04:47.255 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6650000000000003e-06 2024-09-13T15:04:47.255 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.255 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.255 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.255 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248863+0000", 2024-09-13T15:04:47.255 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0713e-05 2024-09-13T15:04:47.255 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.255 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.255 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.255 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.255 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.255 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:69710 10.f 10:fe7e8c5d:::benchmark_data_smithi096_288650_object69709:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2024-09-13T15:04:47.255 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:03:23.135482+0000", 2024-09-13T15:04:47.256 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 84.015631095000003, 2024-09-13T15:04:47.256 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.127934509, 2024-09-13T15:04:47.256 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.256 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.256 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.256 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:47.256 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:47.256 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 69710 2024-09-13T15:04:47.256 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.256 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.256 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.256 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.256 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135482+0000", 2024-09-13T15:04:47.256 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.256 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.257 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.257 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.257 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135482+0000", 2024-09-13T15:04:47.257 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.257 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.257 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.257 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.257 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135474+0000", 2024-09-13T15:04:47.257 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999914 2024-09-13T15:04:47.257 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.257 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.257 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.257 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.135523+0000", 2024-09-13T15:04:47.258 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9697000000000002e-05 2024-09-13T15:04:47.258 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.258 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.258 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.258 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.247678+0000", 2024-09-13T15:04:47.258 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.112155276 2024-09-13T15:04:47.258 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.258 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.258 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.258 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.247696+0000", 2024-09-13T15:04:47.258 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.7711999999999999e-05 2024-09-13T15:04:47.258 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.258 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.258 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.259 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.251376+0000", 2024-09-13T15:04:47.259 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0036794150000000001 2024-09-13T15:04:47.259 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.259 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.259 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.259 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.251486+0000", 2024-09-13T15:04:47.259 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000110569 2024-09-13T15:04:47.259 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.259 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.259 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,4", 2024-09-13T15:04:47.259 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.251610+0000", 2024-09-13T15:04:47.259 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012345699999999999 2024-09-13T15:04:47.259 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.259 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.260 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.260 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.262008+0000", 2024-09-13T15:04:47.260 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010398515000000001 2024-09-13T15:04:47.260 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.260 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.260 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.260 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.262190+0000", 2024-09-13T15:04:47.260 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018208300000000001 2024-09-13T15:04:47.260 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.260 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.260 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.260 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.263301+0000", 2024-09-13T15:04:47.260 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.001111157 2024-09-13T15:04:47.260 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.261 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.261 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.261 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.263380+0000", 2024-09-13T15:04:47.261 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8997000000000003e-05 2024-09-13T15:04:47.261 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.261 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.261 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.261 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.263417+0000", 2024-09-13T15:04:47.261 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6378000000000003e-05 2024-09-13T15:04:47.261 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.261 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.261 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.261 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.261 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.262 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_repop_reply(client.36552.0:69689 10.a e362/355 ondisk, result = 0)", 2024-09-13T15:04:47.262 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:03:23.136161+0000", 2024-09-13T15:04:47.262 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 84.014952140000005, 2024-09-13T15:04:47.262 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.112845297, 2024-09-13T15:04:47.262 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.262 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "started", 2024-09-13T15:04:47.262 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.262 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.262 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.262 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.136161+0000", 2024-09-13T15:04:47.262 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.262 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.262 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.263 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.263 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.136161+0000", 2024-09-13T15:04:47.263 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.263 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.263 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.263 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.263 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.136161+0000", 2024-09-13T15:04:47.263 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:47.263 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.263 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.263 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.263 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.136165+0000", 2024-09-13T15:04:47.263 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7720000000000002e-06 2024-09-13T15:04:47.263 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.264 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.264 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.264 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248539+0000", 2024-09-13T15:04:47.264 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.112373378 2024-09-13T15:04:47.264 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.264 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.264 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.264 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248549+0000", 2024-09-13T15:04:47.264 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.8570000000000007e-06 2024-09-13T15:04:47.264 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.264 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.264 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.264 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248900+0000", 2024-09-13T15:04:47.264 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00035176700000000001 2024-09-13T15:04:47.265 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.265 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.265 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.265 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.248907+0000", 2024-09-13T15:04:47.265 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4370000000000004e-06 2024-09-13T15:04:47.265 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.265 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.265 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.265 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:03:23.249007+0000", 2024-09-13T15:04:47.265 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.9730999999999998e-05 2024-09-13T15:04:47.265 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.265 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.265 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.265 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.266 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.266 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:142390 10.7 10:e2bd146a:::benchmark_data_smithi096_288650_object6698:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2024-09-13T15:04:47.266 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:26.887109+0000", 2024-09-13T15:04:47.266 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 20.264004552999999, 2024-09-13T15:04:47.266 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.113763738, 2024-09-13T15:04:47.266 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:04:47.266 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:47.266 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:04:47.266 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:04:47.266 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:04:47.266 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142390 2024-09-13T15:04:47.266 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.266 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:04:47.266 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.266 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:04:47.267 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:26.887109+0000", 2024-09-13T15:04:47.267 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.267 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.267 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.267 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:04:47.267 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:26.887109+0000", 2024-09-13T15:04:47.267 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:04:47.267 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.267 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.267 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:04:47.267 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:26.887108+0000", 2024-09-13T15:04:47.267 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:47.267 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.267 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.268 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:04:47.268 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:26.887112+0000", 2024-09-13T15:04:47.268 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5049999999999999e-06 2024-09-13T15:04:47.268 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.268 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.268 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:04:47.268 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:26.887119+0000", 2024-09-13T15:04:47.268 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7209999999999994e-06 2024-09-13T15:04:47.268 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.268 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.268 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:04:47.268 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:26.887124+0000", 2024-09-13T15:04:47.268 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4460000000000003e-06 2024-09-13T15:04:47.268 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.268 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.269 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:04:47.269 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:26.892382+0000", 2024-09-13T15:04:47.269 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0052578410000000001 2024-09-13T15:04:47.269 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.269 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.269 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:04:47.269 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:26.892485+0000", 2024-09-13T15:04:47.269 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000103247 2024-09-13T15:04:47.269 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.269 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.269 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:04:47.269 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:26.892596+0000", 2024-09-13T15:04:47.269 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011137999999999999 2024-09-13T15:04:47.269 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:26.895785+0000", 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0031891380000000002 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:26.896029+0000", 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000243611 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:27.000800+0000", 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.104770918 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.270 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.271 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:04:47.271 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:27.000840+0000", 2024-09-13T15:04:47.271 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9821999999999999e-05 2024-09-13T15:04:47.271 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:04:47.271 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:04:47.271 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:04:47.271 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:27.000873+0000", 2024-09-13T15:04:47.271 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2704999999999999e-05 2024-09-13T15:04:47.271 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.271 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.271 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.271 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:04:47.271 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:04:47.272 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:04:47.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.273 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:47.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:47.374 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:47.398 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T15:04:47.408 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:47.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:47.511 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:47.549 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:47.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:47.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:47 smithi096 ceph-mon[98901]: pgmap v1259: 119 pgs: 119 active+clean; 6.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 771 op/s 2024-09-13T15:04:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:47 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:47 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:04:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:47 smithi096 ceph-mon[98901]: osdmap e374: 8 total, 8 up, 7 in 2024-09-13T15:04:47.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:47 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3597269215' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:04:47.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:47 smithi096 ceph-mon[102928]: pgmap v1259: 119 pgs: 119 active+clean; 6.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 771 op/s 2024-09-13T15:04:47.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:47 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:47.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:47 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:04:47.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:47 smithi096 ceph-mon[102928]: osdmap e374: 8 total, 8 up, 7 in 2024-09-13T15:04:47.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:47 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3597269215' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:04:47.651 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:47.674 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:47.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:47 smithi204 ceph-mon[87071]: pgmap v1259: 119 pgs: 119 active+clean; 6.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 771 op/s 2024-09-13T15:04:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:47 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:47 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:04:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:47 smithi204 ceph-mon[87071]: osdmap e374: 8 total, 8 up, 7 in 2024-09-13T15:04:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:47 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3597269215' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:04:47.775 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:47.807 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:47.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:47.909 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:47.942 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:47.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:48.044 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:48.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.084 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:48.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:48.185 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:48.220 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:48.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:48.321 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:48.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:04:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:04:47] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.51.0" 2024-09-13T15:04:48.353 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:48.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:48.455 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:48.488 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:48.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:48.590 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:48.611 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:48.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:48.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:48.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:48 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3597269215' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:04:48.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:48 smithi204 ceph-mon[87071]: osdmap e375: 8 total, 8 up, 7 in 2024-09-13T15:04:48.734 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:48.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:48.778 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:48 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3597269215' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:04:48.779 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:48 smithi096 ceph-mon[98901]: osdmap e375: 8 total, 8 up, 7 in 2024-09-13T15:04:48.779 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:48 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3597269215' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:04:48.779 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:48 smithi096 ceph-mon[102928]: osdmap e375: 8 total, 8 up, 7 in 2024-09-13T15:04:48.836 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:48.863 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:48.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:48.965 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:48.997 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:48.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:49.099 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:49.121 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:49.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:49.130 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T15:04:49.225 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:49.262 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:49.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:49.364 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:49.398 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:49.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:49.500 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:49.548 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:49.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:49.650 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:49.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.686 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:49.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:49.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:49 smithi204 ceph-mon[87071]: pgmap v1262: 119 pgs: 119 active+clean; 6.6 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.24k op/s 2024-09-13T15:04:49.721 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:49 smithi096 ceph-mon[98901]: pgmap v1262: 119 pgs: 119 active+clean; 6.6 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.24k op/s 2024-09-13T15:04:49.721 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:49 smithi096 ceph-mon[102928]: pgmap v1262: 119 pgs: 119 active+clean; 6.6 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.24k op/s 2024-09-13T15:04:49.786 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:49.822 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:49.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:49.924 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:49.944 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:49.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:50.046 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:50.085 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:50.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:50.110 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:50.187 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:50.223 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:50.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:50.326 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:50.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.359 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:50.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:50.460 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:04:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:04:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:04:50.492 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:50.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:50.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:50.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.647 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:50.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:50.748 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:50.789 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:50.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:50.891 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:50.915 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:50.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:51.017 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:51.056 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:51.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:51.158 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:51.181 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:51.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:51.262 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:04:51.283 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:51.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.320 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:51.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:51.421 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:51.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.466 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:51.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:51.567 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:51.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:51 smithi096 ceph-mon[98901]: pgmap v1263: 119 pgs: 119 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 870 op/s 2024-09-13T15:04:51.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:51.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:51 smithi096 ceph-mon[102928]: pgmap v1263: 119 pgs: 119 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 870 op/s 2024-09-13T15:04:51.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:51.594 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:51.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:51.696 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:51 smithi204 ceph-mon[87071]: pgmap v1263: 119 pgs: 119 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 870 op/s 2024-09-13T15:04:51.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:51.725 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:51.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:51.826 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:51.855 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:51.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:51.957 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:51.985 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:51.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:52.087 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:52.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.144 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:52.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:52.245 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:52.275 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:52.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:52.376 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:52.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.412 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:52.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:52.414 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:04:52.415 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:04:52.450 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:04:52.450 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:04:52.482 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:04:52.482 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:04:52.512 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:04:52.513 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:04:52.513 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:52.543 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:04:52.543 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:04:52.544 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:52.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:52.571 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:04:52.572 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:04:52.600 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:04:52.600 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:04:52.630 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:52 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3190406124' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:52.630 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:52 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3190406124' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:52.631 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:04:52.631 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:04:52.646 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:52.675 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:52.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:52.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:52 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3190406124' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:52.777 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:52.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.811 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:52.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:52.912 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:52.947 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:52.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:53.049 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:53.077 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:53.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:53.179 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:53.219 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:53.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:53.321 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:53.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.357 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:53.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:53.458 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:53.503 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:53.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:53.528 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T15:04:53.606 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:53.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.669 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:53.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:53.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:53 smithi204 ceph-mon[87071]: pgmap v1264: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.31k op/s 2024-09-13T15:04:53.770 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:53.796 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:53.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:53.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:53 smithi096 ceph-mon[98901]: pgmap v1264: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.31k op/s 2024-09-13T15:04:53.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:53 smithi096 ceph-mon[102928]: pgmap v1264: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.31k op/s 2024-09-13T15:04:53.898 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:53.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.953 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:53.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:54.042 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:54.054 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:54.082 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:54.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:54.184 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:54.214 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:54.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:54.316 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:54.357 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:54.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:54.446 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T15:04:54.458 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:54.492 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:54.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:54.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:54.628 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:54.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:54.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:54 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2089332146' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:04:54.730 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:54.756 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:54.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:54.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:54 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2089332146' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:04:54.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:54 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2089332146' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:04:54.858 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:54.892 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:54.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:54.993 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:55.035 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:55.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:55.137 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:55.162 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:55.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:55.203 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T15:04:55.264 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:55.281 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:55.318 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:55.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:55.419 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:55.462 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:55.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:55.564 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:55.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.603 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:55.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:55.704 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:55 smithi204 ceph-mon[87071]: pgmap v1265: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s 2024-09-13T15:04:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:55 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2089332146' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:04:55.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:55 smithi204 ceph-mon[87071]: osdmap e376: 8 total, 8 up, 7 in 2024-09-13T15:04:55.735 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:55.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:55.788 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:55 smithi096 ceph-mon[98901]: pgmap v1265: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s 2024-09-13T15:04:55.788 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:55 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2089332146' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:04:55.788 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:55 smithi096 ceph-mon[98901]: osdmap e376: 8 total, 8 up, 7 in 2024-09-13T15:04:55.788 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:55 smithi096 ceph-mon[102928]: pgmap v1265: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s 2024-09-13T15:04:55.789 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:55 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2089332146' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:04:55.789 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:55 smithi096 ceph-mon[102928]: osdmap e376: 8 total, 8 up, 7 in 2024-09-13T15:04:55.836 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:55.866 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:55.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:55.968 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:56.010 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:56.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:56.112 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:56.143 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:56.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:56.245 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:56.271 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:56.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:56.372 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:56.408 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:56.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:56.510 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:56.538 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:56.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:56.566 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:04:56.639 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:56.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.677 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:56.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:56.778 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:56.785 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:04:56.785 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:04:56.811 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:56.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:56.913 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:56.946 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:56.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:57.047 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:57.084 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:57.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:57.186 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:57.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.224 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:57.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:57.253 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:57.323 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:57.325 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:57.371 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:57.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:57.473 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:57.548 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T15:04:57.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.627 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:57.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:57.694 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:57 smithi096 ceph-mon[98901]: pgmap v1267: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 734 op/s 2024-09-13T15:04:57.694 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:57 smithi096 ceph-mon[102928]: pgmap v1267: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 734 op/s 2024-09-13T15:04:57.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:57 smithi204 ceph-mon[87071]: pgmap v1267: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 734 op/s 2024-09-13T15:04:57.728 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:57.755 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:57.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:57.857 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:57.907 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:57.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:58.009 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:58.036 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:58.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:58.059 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:58.138 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:58.178 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:58.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:58.280 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:58.321 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:58.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:58.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:04:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:04:57] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-09-13T15:04:58.424 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:58.466 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:58.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:58.472 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T15:04:58.568 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:58.600 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:58.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:58.702 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:58.717 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:58 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2770564283' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:58.717 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:58 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:58.718 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:58 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2770564283' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:58.718 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:58 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:58.720 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:58 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2770564283' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:58.721 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:58 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:58.739 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:58.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:58.841 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:58.867 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:58.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:58.969 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:58.999 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:59.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:59.101 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:59.138 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:59.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:59.239 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:59.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.275 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:59.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:59.375 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:59.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.416 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:59.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:59.517 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:59.550 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:59.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:59.652 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:59.685 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:59.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:59 smithi204 ceph-mon[87071]: pgmap v1268: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T15:04:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:59 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:04:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:59 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T15:04:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:59 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:04:59 smithi204 ceph-mon[87071]: osdmap e377: 8 total, 8 up, 7 in 2024-09-13T15:04:59.753 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:59 smithi096 ceph-mon[98901]: pgmap v1268: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T15:04:59.753 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:59 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:04:59.753 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:59 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T15:04:59.753 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:59 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:59.753 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:04:59 smithi096 ceph-mon[98901]: osdmap e377: 8 total, 8 up, 7 in 2024-09-13T15:04:59.754 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:59 smithi096 ceph-mon[102928]: pgmap v1268: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T15:04:59.754 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:59 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:04:59.754 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:59 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T15:04:59.754 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:59 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:59.754 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:04:59 smithi096 ceph-mon[102928]: osdmap e377: 8 total, 8 up, 7 in 2024-09-13T15:04:59.787 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:59.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.825 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:04:59.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:59.926 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:59.972 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:04:59.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:00.074 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:00.100 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:00.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:00.202 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:00.229 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:00.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:00.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:00.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.359 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:00.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:00.460 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:00.465 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:05:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:05:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:05:00.479 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T15:05:00.505 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:00.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:00.607 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:00.649 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:00.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:00.658 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:05:00.659 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:05:00.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:00 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3799956055' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:05:00.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:00 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:05:00.751 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:00.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.786 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:00.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:00.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:00 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3799956055' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:05:00.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:00 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:05:00.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:00 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3799956055' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:05:00.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:00 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:05:00.887 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:00.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.926 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:00.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:01.027 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:01.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.062 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:01.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:01.163 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:01.186 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:05:01.204 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:01.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:01.306 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:01.348 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:01.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:01.450 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:01.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:01 smithi096 ceph-mon[98901]: pgmap v1270: 119 pgs: 119 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 837 op/s 2024-09-13T15:05:01.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:01 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:05:01.474 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:01 smithi096 ceph-mon[98901]: osdmap e378: 8 total, 8 up, 7 in 2024-09-13T15:05:01.478 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:01.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:01.483 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T15:05:01.581 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:01.627 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:01.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:01 smithi204 ceph-mon[87071]: pgmap v1270: 119 pgs: 119 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 837 op/s 2024-09-13T15:05:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:01 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:05:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:01 smithi204 ceph-mon[87071]: osdmap e378: 8 total, 8 up, 7 in 2024-09-13T15:05:01.728 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:01.755 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:01 smithi096 ceph-mon[102928]: pgmap v1270: 119 pgs: 119 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 837 op/s 2024-09-13T15:05:01.755 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:01 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:05:01.755 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:01 smithi096 ceph-mon[102928]: osdmap e378: 8 total, 8 up, 7 in 2024-09-13T15:05:01.758 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:01.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:01.792 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T15:05:01.792 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:05:01.860 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:01.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.901 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:01.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:01.925 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:05:02.002 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:02.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.049 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:02.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:02.150 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:02.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.190 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:02.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:02.291 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:02.325 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:02.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:02.346 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:05:02.346 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:05:02.354 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:02.392 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:05:02.392 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:05:02.427 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:02.431 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:05:02.431 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:05:02.460 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:05:02.460 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:05:02.467 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:02.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:02.495 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:05:02.495 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:05:02.522 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:05:02.522 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:05:02.553 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:05:02.553 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:05:02.569 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:02.589 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:05:02.589 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:05:02.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:02 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:05:02.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:02 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3810863543' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:02.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:02 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:02.598 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:02 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:02.599 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:02 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:02.599 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:02 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:02.601 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:02 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:05:02.601 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:02 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3810863543' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:02.601 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:02 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:02.601 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:02 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:02.601 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:02 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:02.601 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:02 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:02.606 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:02.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:02.708 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:02 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:05:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:02 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3810863543' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:02 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:02 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:02 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:02 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:02.772 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:02.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:02.874 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:02.895 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:02.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:02.997 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:03.031 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:03.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:03.133 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:03.173 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:03.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:03.274 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:03.310 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:03.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:03.412 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:03.448 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:03.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:03.549 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:03.581 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:03.683 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:03.711 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:03.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:03 smithi204 ceph-mon[87071]: pgmap v1272: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.38k op/s 2024-09-13T15:05:03.812 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:03.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:03 smithi096 ceph-mon[98901]: pgmap v1272: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.38k op/s 2024-09-13T15:05:03.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:03 smithi096 ceph-mon[102928]: pgmap v1272: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.38k op/s 2024-09-13T15:05:03.851 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:03.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:03.944 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:03.953 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:04.001 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:04.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:04.103 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:04.132 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:04.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:04.234 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:04.291 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:04.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:04.394 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:04.445 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:04.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:04.549 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:04.590 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:04.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:04.592 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:05:04.592 INFO:teuthology.orchestra.run.smithi204.stdout: "size": 20, 2024-09-13T15:05:04.592 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 600, 2024-09-13T15:05:04.592 INFO:teuthology.orchestra.run.smithi204.stdout: "ops": [ 2024-09-13T15:05:04.592 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.592 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:113296 9.3 9:c73004b8:::benchmark_data_smithi096_256909_object113295:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-09-13T15:05:04.592 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:11.915762+0000", 2024-09-13T15:05:04.592 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 352.67064617699998, 2024-09-13T15:05:04.592 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.98619730999999999, 2024-09-13T15:05:04.593 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.593 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.593 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.593 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.593 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.593 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 113296 2024-09-13T15:05:04.593 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.593 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.593 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.593 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.593 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915762+0000", 2024-09-13T15:05:04.593 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.593 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.593 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.594 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.594 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915762+0000", 2024-09-13T15:05:04.594 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.594 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.594 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.594 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.594 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915760+0000", 2024-09-13T15:05:04.594 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:05:04.594 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.594 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.594 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.594 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915786+0000", 2024-09-13T15:05:04.594 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5636e-05 2024-09-13T15:05:04.594 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.594 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.595 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.595 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915801+0000", 2024-09-13T15:05:04.595 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5174000000000001e-05 2024-09-13T15:05:04.595 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.595 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.595 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.595 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915808+0000", 2024-09-13T15:05:04.595 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8379999999999996e-06 2024-09-13T15:05:04.595 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.595 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.595 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.595 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915849+0000", 2024-09-13T15:05:04.595 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1340000000000001e-05 2024-09-13T15:05:04.595 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.596 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.596 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.596 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.915856+0000", 2024-09-13T15:05:04.596 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.0770000000000002e-06 2024-09-13T15:05:04.596 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.596 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.596 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.596 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.888274+0000", 2024-09-13T15:05:04.596 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.972418381 2024-09-13T15:05:04.596 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.596 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.596 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.596 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.888439+0000", 2024-09-13T15:05:04.596 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000164365 2024-09-13T15:05:04.597 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.597 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.597 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.597 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.888633+0000", 2024-09-13T15:05:04.597 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000194286 2024-09-13T15:05:04.597 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.597 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.597 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.597 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.897978+0000", 2024-09-13T15:05:04.597 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0093450700000000005 2024-09-13T15:05:04.597 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.597 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.597 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.597 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.898169+0000", 2024-09-13T15:05:04.597 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00019047199999999999 2024-09-13T15:05:04.598 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.598 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.598 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.598 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.901811+0000", 2024-09-13T15:05:04.598 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003641886 2024-09-13T15:05:04.598 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.598 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.598 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.598 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.901858+0000", 2024-09-13T15:05:04.598 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7889999999999997e-05 2024-09-13T15:05:04.598 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.598 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.598 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.598 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.901959+0000", 2024-09-13T15:05:04.598 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00010039099999999999 2024-09-13T15:05:04.599 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.599 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.599 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.599 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.599 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.599 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:113301 9.3 9:c3749cfb:::benchmark_data_smithi096_256909_object113300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-09-13T15:05:04.599 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:11.920727+0000", 2024-09-13T15:05:04.599 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 352.665680415, 2024-09-13T15:05:04.599 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.99153909699999998, 2024-09-13T15:05:04.599 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.599 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.599 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.600 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.600 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.600 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 113301 2024-09-13T15:05:04.600 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.600 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.600 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.600 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.600 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920727+0000", 2024-09-13T15:05:04.600 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.600 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.600 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.600 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.600 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920727+0000", 2024-09-13T15:05:04.600 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.600 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.601 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.601 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.601 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920726+0000", 2024-09-13T15:05:04.601 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:05:04.601 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.601 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.601 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.601 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920767+0000", 2024-09-13T15:05:04.601 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1270000000000003e-05 2024-09-13T15:05:04.601 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.601 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.601 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.601 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920783+0000", 2024-09-13T15:05:04.601 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6081000000000001e-05 2024-09-13T15:05:04.602 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.602 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.602 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.602 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920790+0000", 2024-09-13T15:05:04.602 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1910000000000003e-06 2024-09-13T15:05:04.602 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.602 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.602 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.602 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920839+0000", 2024-09-13T15:05:04.602 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9172999999999998e-05 2024-09-13T15:05:04.602 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.602 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.602 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.603 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:11.920853+0000", 2024-09-13T15:05:04.603 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4098e-05 2024-09-13T15:05:04.603 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.603 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.603 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.603 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.892743+0000", 2024-09-13T15:05:04.603 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.97188978699999995 2024-09-13T15:05:04.603 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.603 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.603 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.603 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.892878+0000", 2024-09-13T15:05:04.603 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000134657 2024-09-13T15:05:04.603 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.603 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.603 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.604 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.893051+0000", 2024-09-13T15:05:04.604 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00017329199999999999 2024-09-13T15:05:04.604 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.604 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.604 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.604 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.898204+0000", 2024-09-13T15:05:04.604 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0051531320000000004 2024-09-13T15:05:04.604 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.604 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.604 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.604 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.901779+0000", 2024-09-13T15:05:04.604 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0035747069999999999 2024-09-13T15:05:04.604 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.604 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.605 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.605 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.912119+0000", 2024-09-13T15:05:04.605 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010340293 2024-09-13T15:05:04.605 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.605 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.605 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.605 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.912170+0000", 2024-09-13T15:05:04.605 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0917000000000001e-05 2024-09-13T15:05:04.605 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.605 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.605 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.605 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:12.912266+0000", 2024-09-13T15:05:04.606 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.6180000000000004e-05 2024-09-13T15:05:04.606 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.606 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.606 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.606 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.606 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.606 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142789 9.3 9:c5ee02be:::benchmark_data_smithi096_256909_object23381:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:05:04.606 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.187568+0000", 2024-09-13T15:05:04.606 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.39884021799998, 2024-09-13T15:05:04.606 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.458558638, 2024-09-13T15:05:04.606 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.606 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "reached pg", 2024-09-13T15:05:04.606 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.606 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142789 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.187568+0000", 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.187568+0000", 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.607 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.608 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.187566+0000", 2024-09-13T15:05:04.608 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:05:04.608 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.608 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.608 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.608 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.187571+0000", 2024-09-13T15:05:04.608 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0270000000000003e-06 2024-09-13T15:05:04.608 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.608 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.608 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.608 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.187583+0000", 2024-09-13T15:05:04.608 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1603999999999999e-05 2024-09-13T15:05:04.608 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.608 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.609 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.609 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.187589+0000", 2024-09-13T15:05:04.609 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.3169999999999998e-06 2024-09-13T15:05:04.609 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.609 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.609 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.609 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.187747+0000", 2024-09-13T15:05:04.609 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000157529 2024-09-13T15:05:04.609 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.609 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.609 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.609 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.188591+0000", 2024-09-13T15:05:04.609 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00084404 2024-09-13T15:05:04.609 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.609 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.610 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.610 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.188709+0000", 2024-09-13T15:05:04.610 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00011758500000000001 2024-09-13T15:05:04.610 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.610 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.610 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.610 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.198395+0000", 2024-09-13T15:05:04.610 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0096860690000000003 2024-09-13T15:05:04.610 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.610 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.610 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.610 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.198401+0000", 2024-09-13T15:05:04.610 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9210000000000003e-06 2024-09-13T15:05:04.611 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.611 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.611 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.611 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.603902+0000", 2024-09-13T15:05:04.611 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4055006640000001 2024-09-13T15:05:04.611 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.611 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.611 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for missing object", 2024-09-13T15:05:04.611 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.604030+0000", 2024-09-13T15:05:04.611 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012781500000000001 2024-09-13T15:05:04.611 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.611 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.611 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.612 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646040+0000", 2024-09-13T15:05:04.612 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.042009791999999997 2024-09-13T15:05:04.612 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.612 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.612 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.612 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646126+0000", 2024-09-13T15:05:04.612 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.6382000000000001e-05 2024-09-13T15:05:04.612 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.612 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.612 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.612 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.612 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.612 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142798 9.3 9:c534ab8f:::benchmark_data_smithi096_256909_object23390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:05:04.613 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.201181+0000", 2024-09-13T15:05:04.613 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.38522633999997, 2024-09-13T15:05:04.613 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4451804939999999, 2024-09-13T15:05:04.613 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.613 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.613 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.613 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.613 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.613 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142798 2024-09-13T15:05:04.614 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.614 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.614 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.614 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.614 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201181+0000", 2024-09-13T15:05:04.614 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.614 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.614 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.614 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.614 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201181+0000", 2024-09-13T15:05:04.614 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.614 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.614 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.615 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.615 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201179+0000", 2024-09-13T15:05:04.615 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:05:04.615 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.615 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.615 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.615 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201186+0000", 2024-09-13T15:05:04.615 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9009999999999997e-06 2024-09-13T15:05:04.615 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.615 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.615 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.615 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201204+0000", 2024-09-13T15:05:04.615 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.8158000000000001e-05 2024-09-13T15:05:04.616 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.616 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.616 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.616 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201210+0000", 2024-09-13T15:05:04.616 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1210000000000003e-06 2024-09-13T15:05:04.616 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.616 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.616 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.616 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201461+0000", 2024-09-13T15:05:04.616 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00025028499999999999 2024-09-13T15:05:04.616 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.616 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.616 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.617 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.201469+0000", 2024-09-13T15:05:04.617 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8669999999999994e-06 2024-09-13T15:05:04.617 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.617 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.617 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.617 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.604066+0000", 2024-09-13T15:05:04.617 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.402596897 2024-09-13T15:05:04.617 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.617 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.617 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.617 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.604204+0000", 2024-09-13T15:05:04.617 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013892900000000001 2024-09-13T15:05:04.617 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.618 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.618 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.618 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.604352+0000", 2024-09-13T15:05:04.618 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000147119 2024-09-13T15:05:04.618 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.618 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.618 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.618 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.616334+0000", 2024-09-13T15:05:04.618 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011981921 2024-09-13T15:05:04.618 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.618 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.618 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.618 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.645720+0000", 2024-09-13T15:05:04.618 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.029386374 2024-09-13T15:05:04.619 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.619 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.619 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.619 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646210+0000", 2024-09-13T15:05:04.619 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000489684 2024-09-13T15:05:04.619 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.619 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.619 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.619 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646260+0000", 2024-09-13T15:05:04.619 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.9963999999999998e-05 2024-09-13T15:05:04.619 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.619 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.619 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.620 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646362+0000", 2024-09-13T15:05:04.620 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000102352 2024-09-13T15:05:04.620 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.620 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.620 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.621 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.621 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.621 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142804 9.3 9:c741bee0:::benchmark_data_smithi096_256909_object23396:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:05:04.621 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.203140+0000", 2024-09-13T15:05:04.621 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.38326809, 2024-09-13T15:05:04.621 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.443340676, 2024-09-13T15:05:04.621 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.621 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.621 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.621 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.621 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.621 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142804 2024-09-13T15:05:04.621 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.621 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.622 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.622 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.622 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203140+0000", 2024-09-13T15:05:04.622 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.622 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.622 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.622 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.622 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203140+0000", 2024-09-13T15:05:04.622 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.622 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.622 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.622 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.622 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203139+0000", 2024-09-13T15:05:04.622 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:05:04.623 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.623 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.623 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.623 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203143+0000", 2024-09-13T15:05:04.623 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.642e-06 2024-09-13T15:05:04.623 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.623 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.623 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.623 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203153+0000", 2024-09-13T15:05:04.623 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0023999999999999e-05 2024-09-13T15:05:04.623 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.623 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.623 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.623 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203157+0000", 2024-09-13T15:05:04.624 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1740000000000002e-06 2024-09-13T15:05:04.624 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.624 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.624 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.624 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203184+0000", 2024-09-13T15:05:04.624 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7155999999999999e-05 2024-09-13T15:05:04.624 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.624 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.624 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.624 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.203188+0000", 2024-09-13T15:05:04.624 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6070000000000001e-06 2024-09-13T15:05:04.624 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.624 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.624 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.625 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.607722+0000", 2024-09-13T15:05:04.625 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.404533799 2024-09-13T15:05:04.625 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.625 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.625 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.625 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.607849+0000", 2024-09-13T15:05:04.625 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012717499999999999 2024-09-13T15:05:04.625 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.625 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.625 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.625 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.608000+0000", 2024-09-13T15:05:04.625 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00015092 2024-09-13T15:05:04.626 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.626 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.626 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.626 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.621314+0000", 2024-09-13T15:05:04.626 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013313503000000001 2024-09-13T15:05:04.626 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.626 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.626 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.626 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646164+0000", 2024-09-13T15:05:04.626 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.024850395000000001 2024-09-13T15:05:04.626 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.626 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.626 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.626 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646400+0000", 2024-09-13T15:05:04.627 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00023617199999999999 2024-09-13T15:05:04.627 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.627 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.627 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.627 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646429+0000", 2024-09-13T15:05:04.627 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8323e-05 2024-09-13T15:05:04.627 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.627 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.627 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.627 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646480+0000", 2024-09-13T15:05:04.627 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1530999999999998e-05 2024-09-13T15:05:04.628 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.628 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.628 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.628 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.628 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.628 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142822 9.3 9:c7fbd599:::benchmark_data_smithi096_256909_object23414:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:05:04.628 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.232068+0000", 2024-09-13T15:05:04.628 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.35433999600002, 2024-09-13T15:05:04.628 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.414564594, 2024-09-13T15:05:04.628 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.628 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.629 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.629 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.629 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.629 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142822 2024-09-13T15:05:04.629 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.629 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.629 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.629 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.629 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232068+0000", 2024-09-13T15:05:04.629 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.629 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.629 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.629 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.630 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232068+0000", 2024-09-13T15:05:04.630 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.630 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.630 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.630 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.630 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232067+0000", 2024-09-13T15:05:04.630 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:05:04.630 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.630 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.630 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.630 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232071+0000", 2024-09-13T15:05:04.630 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4159999999999997e-06 2024-09-13T15:05:04.630 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.631 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.631 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.631 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232087+0000", 2024-09-13T15:05:04.631 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.5184000000000001e-05 2024-09-13T15:05:04.631 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.631 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.631 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.631 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232092+0000", 2024-09-13T15:05:04.631 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6899999999999997e-06 2024-09-13T15:05:04.631 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.631 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.631 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.631 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232659+0000", 2024-09-13T15:05:04.632 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00056658499999999998 2024-09-13T15:05:04.632 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.632 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.632 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.632 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.232667+0000", 2024-09-13T15:05:04.632 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6799999999999993e-06 2024-09-13T15:05:04.632 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.632 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.632 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.632 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.611147+0000", 2024-09-13T15:05:04.632 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3784807080000001 2024-09-13T15:05:04.632 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.632 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.632 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.633 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.611228+0000", 2024-09-13T15:05:04.633 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1085e-05 2024-09-13T15:05:04.633 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.633 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.633 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.633 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.611303+0000", 2024-09-13T15:05:04.633 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.4412000000000005e-05 2024-09-13T15:05:04.633 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.633 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.633 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.633 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.621322+0000", 2024-09-13T15:05:04.633 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010018872 2024-09-13T15:05:04.634 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.634 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.634 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.634 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646521+0000", 2024-09-13T15:05:04.634 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.025199784999999999 2024-09-13T15:05:04.634 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.634 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.634 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.634 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646538+0000", 2024-09-13T15:05:04.634 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.6949999999999999e-05 2024-09-13T15:05:04.634 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.634 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.634 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.635 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646564+0000", 2024-09-13T15:05:04.635 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5083999999999999e-05 2024-09-13T15:05:04.635 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.635 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.635 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.635 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646632+0000", 2024-09-13T15:05:04.635 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8813000000000007e-05 2024-09-13T15:05:04.635 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.635 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.635 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.635 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.635 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.635 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142836 9.3 9:c3b86387:::benchmark_data_smithi096_256909_object23428:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:05:04.636 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.245149+0000", 2024-09-13T15:05:04.636 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.34125838, 2024-09-13T15:05:04.636 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4016361989999999, 2024-09-13T15:05:04.636 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.636 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.636 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.636 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.636 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.636 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142836 2024-09-13T15:05:04.636 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.636 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.636 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.637 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.637 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245149+0000", 2024-09-13T15:05:04.637 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.637 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.637 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.637 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.637 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245149+0000", 2024-09-13T15:05:04.637 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.637 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.637 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.637 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.637 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245148+0000", 2024-09-13T15:05:04.637 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:05:04.637 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.638 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.638 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.638 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245155+0000", 2024-09-13T15:05:04.638 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.187e-06 2024-09-13T15:05:04.638 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.638 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.638 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.638 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245167+0000", 2024-09-13T15:05:04.638 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2513e-05 2024-09-13T15:05:04.638 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.638 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.638 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.638 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245173+0000", 2024-09-13T15:05:04.639 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5890000000000002e-06 2024-09-13T15:05:04.639 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.639 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.639 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.639 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245241+0000", 2024-09-13T15:05:04.639 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.8044000000000001e-05 2024-09-13T15:05:04.639 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.639 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.639 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.639 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.245248+0000", 2024-09-13T15:05:04.639 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1439999999999997e-06 2024-09-13T15:05:04.639 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.640 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.640 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.640 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.613589+0000", 2024-09-13T15:05:04.640 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.36834093 2024-09-13T15:05:04.640 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.640 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.640 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.640 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.613714+0000", 2024-09-13T15:05:04.640 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012501299999999999 2024-09-13T15:05:04.640 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.640 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.640 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.640 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.613852+0000", 2024-09-13T15:05:04.641 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013756200000000001 2024-09-13T15:05:04.641 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.641 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.641 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.641 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.626398+0000", 2024-09-13T15:05:04.641 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012546639 2024-09-13T15:05:04.641 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.641 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.641 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.641 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646665+0000", 2024-09-13T15:05:04.641 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.020266998000000001 2024-09-13T15:05:04.641 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.641 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.642 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.642 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646688+0000", 2024-09-13T15:05:04.642 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.2529999999999999e-05 2024-09-13T15:05:04.642 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.642 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.642 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.642 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646713+0000", 2024-09-13T15:05:04.642 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.5193000000000001e-05 2024-09-13T15:05:04.642 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.642 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.642 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.642 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646786+0000", 2024-09-13T15:05:04.642 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.2516999999999997e-05 2024-09-13T15:05:04.643 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.643 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.643 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.643 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.643 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.643 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142851 9.3 9:c9bb45f3:::benchmark_data_smithi096_256909_object23443:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:05:04.643 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.267504+0000", 2024-09-13T15:05:04.643 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.318904041, 2024-09-13T15:05:04.643 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.379523225, 2024-09-13T15:05:04.643 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.643 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.643 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.643 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.644 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.644 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142851 2024-09-13T15:05:04.644 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.644 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.644 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.644 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.644 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267504+0000", 2024-09-13T15:05:04.644 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.644 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.644 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.644 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.644 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267504+0000", 2024-09-13T15:05:04.644 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.645 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.645 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.645 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.645 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267502+0000", 2024-09-13T15:05:04.645 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:05:04.645 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.645 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.645 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.645 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267509+0000", 2024-09-13T15:05:04.645 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.7800000000000003e-06 2024-09-13T15:05:04.645 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.645 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.645 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.646 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267522+0000", 2024-09-13T15:05:04.646 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2819e-05 2024-09-13T15:05:04.646 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.646 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.646 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.646 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267526+0000", 2024-09-13T15:05:04.646 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.887e-06 2024-09-13T15:05:04.646 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.646 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.646 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.646 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267574+0000", 2024-09-13T15:05:04.646 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7586000000000001e-05 2024-09-13T15:05:04.646 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.647 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.647 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.647 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.267583+0000", 2024-09-13T15:05:04.647 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.9269999999999994e-06 2024-09-13T15:05:04.647 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.647 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.647 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.647 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.616374+0000", 2024-09-13T15:05:04.647 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3487907029999999 2024-09-13T15:05:04.647 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.647 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.647 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.647 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.616499+0000", 2024-09-13T15:05:04.648 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012532299999999999 2024-09-13T15:05:04.648 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.648 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.648 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.648 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.616633+0000", 2024-09-13T15:05:04.648 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013409899999999999 2024-09-13T15:05:04.648 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.648 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.648 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.648 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.626407+0000", 2024-09-13T15:05:04.648 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0097741200000000007 2024-09-13T15:05:04.648 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.648 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.648 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.648 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646812+0000", 2024-09-13T15:05:04.649 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.020404723999999999 2024-09-13T15:05:04.649 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.649 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.649 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.649 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646871+0000", 2024-09-13T15:05:04.649 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8733999999999997e-05 2024-09-13T15:05:04.649 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.649 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.649 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.649 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646913+0000", 2024-09-13T15:05:04.649 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1918000000000001e-05 2024-09-13T15:05:04.649 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.649 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.652 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.652 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647027+0000", 2024-09-13T15:05:04.652 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000114276 2024-09-13T15:05:04.652 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.652 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.653 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.653 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.653 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.653 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142853 9.3 9:ccfce356:::benchmark_data_smithi096_256909_object23445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:05:04.653 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.268877+0000", 2024-09-13T15:05:04.653 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.317531105, 2024-09-13T15:05:04.653 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.378334902, 2024-09-13T15:05:04.653 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.653 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.653 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.653 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.653 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.653 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142853 2024-09-13T15:05:04.653 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.654 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.654 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.654 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.654 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268877+0000", 2024-09-13T15:05:04.654 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.654 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.654 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.654 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.654 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268877+0000", 2024-09-13T15:05:04.654 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.654 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.654 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.654 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.654 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268876+0000", 2024-09-13T15:05:04.655 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:05:04.655 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.655 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.655 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.655 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268879+0000", 2024-09-13T15:05:04.655 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8609999999999998e-06 2024-09-13T15:05:04.655 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.655 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.655 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.655 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268884+0000", 2024-09-13T15:05:04.656 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5879999999999999e-06 2024-09-13T15:05:04.656 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.656 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.656 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.656 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268886+0000", 2024-09-13T15:05:04.656 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4729999999999999e-06 2024-09-13T15:05:04.656 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.656 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.656 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.656 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268918+0000", 2024-09-13T15:05:04.656 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.2483000000000001e-05 2024-09-13T15:05:04.656 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.656 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.656 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.657 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.268923+0000", 2024-09-13T15:05:04.657 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1450000000000001e-06 2024-09-13T15:05:04.657 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.657 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.657 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.657 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.618816+0000", 2024-09-13T15:05:04.657 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3498932770000001 2024-09-13T15:05:04.657 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.657 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.657 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.657 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.618914+0000", 2024-09-13T15:05:04.657 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.8220000000000005e-05 2024-09-13T15:05:04.657 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.658 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.658 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.658 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.618996+0000", 2024-09-13T15:05:04.658 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1409000000000006e-05 2024-09-13T15:05:04.658 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.658 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.658 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.658 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.632113+0000", 2024-09-13T15:05:04.658 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.013117403 2024-09-13T15:05:04.658 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.658 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.658 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.658 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.646842+0000", 2024-09-13T15:05:04.658 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.014728774 2024-09-13T15:05:04.659 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.659 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.659 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.659 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647095+0000", 2024-09-13T15:05:04.659 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00025298900000000001 2024-09-13T15:05:04.659 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.659 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.659 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.659 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647134+0000", 2024-09-13T15:05:04.659 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8890999999999997e-05 2024-09-13T15:05:04.659 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.659 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.659 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.659 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647212+0000", 2024-09-13T15:05:04.660 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7939e-05 2024-09-13T15:05:04.660 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.660 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.660 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.660 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.660 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.660 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142872 9.3 9:c83c4353:::benchmark_data_smithi096_256909_object23464:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:05:04.660 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.294230+0000", 2024-09-13T15:05:04.660 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.29217776000002, 2024-09-13T15:05:04.660 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.353197628, 2024-09-13T15:05:04.660 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.660 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.660 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.661 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.661 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.661 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142872 2024-09-13T15:05:04.661 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.661 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.661 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.661 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.661 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294230+0000", 2024-09-13T15:05:04.661 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.661 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.661 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.661 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.661 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294230+0000", 2024-09-13T15:05:04.662 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.662 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.662 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.662 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.662 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294228+0000", 2024-09-13T15:05:04.662 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:05:04.662 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.662 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.662 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.662 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294236+0000", 2024-09-13T15:05:04.662 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1640000000000004e-06 2024-09-13T15:05:04.662 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.662 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.663 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.663 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294250+0000", 2024-09-13T15:05:04.663 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4419999999999999e-05 2024-09-13T15:05:04.663 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.663 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.663 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.663 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294256+0000", 2024-09-13T15:05:04.663 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6590000000000004e-06 2024-09-13T15:05:04.663 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.663 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.663 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.663 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294304+0000", 2024-09-13T15:05:04.663 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7942999999999998e-05 2024-09-13T15:05:04.664 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.664 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.664 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.664 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.294312+0000", 2024-09-13T15:05:04.664 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5690000000000005e-06 2024-09-13T15:05:04.664 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.664 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.664 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.664 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.621367+0000", 2024-09-13T15:05:04.665 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.327055071 2024-09-13T15:05:04.665 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.665 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.665 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.665 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.621487+0000", 2024-09-13T15:05:04.665 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000119842 2024-09-13T15:05:04.665 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.665 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.665 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.665 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.621613+0000", 2024-09-13T15:05:04.665 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000125585 2024-09-13T15:05:04.666 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.666 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.666 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.666 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.632118+0000", 2024-09-13T15:05:04.666 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010505504000000001 2024-09-13T15:05:04.666 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.666 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.666 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.666 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647246+0000", 2024-09-13T15:05:04.666 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.015128252999999999 2024-09-13T15:05:04.666 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.666 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.666 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.667 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647305+0000", 2024-09-13T15:05:04.667 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.8207e-05 2024-09-13T15:05:04.667 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.667 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.667 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.667 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647344+0000", 2024-09-13T15:05:04.668 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.9419999999999999e-05 2024-09-13T15:05:04.668 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.668 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.668 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.668 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647428+0000", 2024-09-13T15:05:04.668 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.3546000000000003e-05 2024-09-13T15:05:04.668 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.669 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.669 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.669 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.669 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.669 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142877 9.3 9:cb7a6d5e:::benchmark_data_smithi096_256909_object23469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:05:04.669 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.300920+0000", 2024-09-13T15:05:04.669 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.28548752699999, 2024-09-13T15:05:04.669 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3466536950000001, 2024-09-13T15:05:04.669 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.669 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.669 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.669 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.669 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.670 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142877 2024-09-13T15:05:04.670 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.670 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.670 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.670 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.670 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.300920+0000", 2024-09-13T15:05:04.670 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.670 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.670 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.670 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.670 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.300920+0000", 2024-09-13T15:05:04.670 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.670 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.670 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.670 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.671 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.300919+0000", 2024-09-13T15:05:04.671 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:05:04.671 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.671 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.671 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.671 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.300925+0000", 2024-09-13T15:05:04.671 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.4690000000000002e-06 2024-09-13T15:05:04.671 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.671 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.671 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.671 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.300947+0000", 2024-09-13T15:05:04.671 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1826000000000001e-05 2024-09-13T15:05:04.671 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.671 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.671 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.672 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.300952+0000", 2024-09-13T15:05:04.672 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4199999999999998e-06 2024-09-13T15:05:04.672 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.672 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.672 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.672 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.301014+0000", 2024-09-13T15:05:04.672 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.1659e-05 2024-09-13T15:05:04.672 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.672 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.672 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.672 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.301023+0000", 2024-09-13T15:05:04.672 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4470000000000006e-06 2024-09-13T15:05:04.672 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.672 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.673 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.673 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.623982+0000", 2024-09-13T15:05:04.673 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3229591359999999 2024-09-13T15:05:04.673 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.673 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.673 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.673 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.624062+0000", 2024-09-13T15:05:04.673 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0673000000000005e-05 2024-09-13T15:05:04.673 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.673 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.673 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.673 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.624150+0000", 2024-09-13T15:05:04.673 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7698e-05 2024-09-13T15:05:04.673 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.673 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.674 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.674 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.636952+0000", 2024-09-13T15:05:04.674 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012801619 2024-09-13T15:05:04.674 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.674 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.674 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.674 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647278+0000", 2024-09-13T15:05:04.674 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010326633 2024-09-13T15:05:04.674 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.674 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.674 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.674 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647461+0000", 2024-09-13T15:05:04.674 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018296100000000001 2024-09-13T15:05:04.674 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.675 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.675 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.675 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647497+0000", 2024-09-13T15:05:04.675 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6089e-05 2024-09-13T15:05:04.675 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.675 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.675 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.675 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647574+0000", 2024-09-13T15:05:04.675 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6555000000000001e-05 2024-09-13T15:05:04.675 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.675 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.675 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.675 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.675 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.675 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142896 9.3 9:cf57ea2d:::benchmark_data_smithi096_256909_object23488:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:05:04.676 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.329447+0000", 2024-09-13T15:05:04.676 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.25696049700002, 2024-09-13T15:05:04.676 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.3183072769999999, 2024-09-13T15:05:04.676 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.676 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.676 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.676 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.676 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.676 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142896 2024-09-13T15:05:04.676 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.676 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.676 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.677 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.677 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329447+0000", 2024-09-13T15:05:04.677 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.677 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.677 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.677 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.677 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329447+0000", 2024-09-13T15:05:04.677 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.677 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.677 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.677 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.677 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329446+0000", 2024-09-13T15:05:04.677 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:05:04.677 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.678 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.678 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.678 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329453+0000", 2024-09-13T15:05:04.678 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.6909999999999997e-06 2024-09-13T15:05:04.678 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.678 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.678 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.678 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329466+0000", 2024-09-13T15:05:04.678 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2835e-05 2024-09-13T15:05:04.678 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.678 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.678 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.678 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329471+0000", 2024-09-13T15:05:04.679 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0089999999999998e-06 2024-09-13T15:05:04.679 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.679 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.679 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.679 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329522+0000", 2024-09-13T15:05:04.679 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0895e-05 2024-09-13T15:05:04.679 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.679 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.679 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.679 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.329529+0000", 2024-09-13T15:05:04.679 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.435e-06 2024-09-13T15:05:04.679 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.679 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.679 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.680 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.626453+0000", 2024-09-13T15:05:04.680 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.296923563 2024-09-13T15:05:04.680 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.680 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.680 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.680 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.626576+0000", 2024-09-13T15:05:04.680 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012282499999999999 2024-09-13T15:05:04.680 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.680 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.680 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.680 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.626781+0000", 2024-09-13T15:05:04.680 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00020488400000000001 2024-09-13T15:05:04.680 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.680 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.681 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.681 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.636955+0000", 2024-09-13T15:05:04.681 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010174325 2024-09-13T15:05:04.681 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.681 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.681 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.681 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647609+0000", 2024-09-13T15:05:04.681 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010653635 2024-09-13T15:05:04.681 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.681 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.681 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.681 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647637+0000", 2024-09-13T15:05:04.681 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8668999999999999e-05 2024-09-13T15:05:04.681 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.681 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.682 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.682 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647674+0000", 2024-09-13T15:05:04.682 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.7128000000000001e-05 2024-09-13T15:05:04.682 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.682 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.682 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.682 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647755+0000", 2024-09-13T15:05:04.682 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0012000000000006e-05 2024-09-13T15:05:04.682 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.682 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.682 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.682 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.682 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.683 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142921 9.3 9:c414e2ce:::benchmark_data_smithi096_256909_object23513:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:05:04.683 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.368915+0000", 2024-09-13T15:05:04.683 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.217493037, 2024-09-13T15:05:04.683 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2790645789999999, 2024-09-13T15:05:04.683 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.683 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.683 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.683 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.683 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.683 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142921 2024-09-13T15:05:04.683 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.683 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.683 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.683 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.683 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.368915+0000", 2024-09-13T15:05:04.684 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.684 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.684 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.684 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.684 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.368915+0000", 2024-09-13T15:05:04.684 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.684 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.684 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.684 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.684 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.368914+0000", 2024-09-13T15:05:04.684 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:05:04.684 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.684 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.684 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.685 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.368921+0000", 2024-09-13T15:05:04.685 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.3300000000000001e-06 2024-09-13T15:05:04.685 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.685 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.685 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.685 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.368941+0000", 2024-09-13T15:05:04.685 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.0217e-05 2024-09-13T15:05:04.685 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.685 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.685 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.685 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.368947+0000", 2024-09-13T15:05:04.685 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.7860000000000002e-06 2024-09-13T15:05:04.685 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.369000+0000", 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2757999999999999e-05 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.369008+0000", 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0879999999999995e-06 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.629551+0000", 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2605434019999999 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.686 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.687 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.687 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.629673+0000", 2024-09-13T15:05:04.687 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012139 2024-09-13T15:05:04.687 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.687 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.687 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.687 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.629802+0000", 2024-09-13T15:05:04.687 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000129729 2024-09-13T15:05:04.687 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.687 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.687 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.687 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.642703+0000", 2024-09-13T15:05:04.687 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.012900688 2024-09-13T15:05:04.688 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.688 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.688 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.688 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647789+0000", 2024-09-13T15:05:04.688 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0050858270000000002 2024-09-13T15:05:04.688 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.688 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.688 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.688 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647858+0000", 2024-09-13T15:05:04.688 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9139000000000003e-05 2024-09-13T15:05:04.688 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.688 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.688 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.689 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647895+0000", 2024-09-13T15:05:04.689 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6597999999999998e-05 2024-09-13T15:05:04.689 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.689 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.689 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.689 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647979+0000", 2024-09-13T15:05:04.689 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.454e-05 2024-09-13T15:05:04.689 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.689 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.689 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.689 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.689 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.689 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142944 9.3 9:ca2419a8:::benchmark_data_smithi096_256909_object23536:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:05:04.689 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.407675+0000", 2024-09-13T15:05:04.690 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.17873229899999, 2024-09-13T15:05:04.690 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.240453279, 2024-09-13T15:05:04.690 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.690 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.690 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.690 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.690 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.690 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142944 2024-09-13T15:05:04.690 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.690 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.690 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.690 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.690 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407675+0000", 2024-09-13T15:05:04.690 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.691 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.691 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.691 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.691 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407675+0000", 2024-09-13T15:05:04.691 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.691 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.691 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.691 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.691 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407675+0000", 2024-09-13T15:05:04.691 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999995 2024-09-13T15:05:04.691 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.691 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.691 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.691 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407678+0000", 2024-09-13T15:05:04.692 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.1350000000000001e-06 2024-09-13T15:05:04.692 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.692 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.692 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.692 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407683+0000", 2024-09-13T15:05:04.692 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.4939999999999997e-06 2024-09-13T15:05:04.692 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.692 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.692 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.692 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407687+0000", 2024-09-13T15:05:04.692 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.014e-06 2024-09-13T15:05:04.692 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.692 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.692 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.692 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407738+0000", 2024-09-13T15:05:04.693 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.1585999999999997e-05 2024-09-13T15:05:04.693 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.693 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.693 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.693 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.407745+0000", 2024-09-13T15:05:04.693 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.1169999999999998e-06 2024-09-13T15:05:04.693 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.693 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.693 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.693 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.632152+0000", 2024-09-13T15:05:04.693 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2244062090000001 2024-09-13T15:05:04.693 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.693 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.693 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.694 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.632227+0000", 2024-09-13T15:05:04.694 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.5765000000000004e-05 2024-09-13T15:05:04.694 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.694 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.694 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.694 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.632313+0000", 2024-09-13T15:05:04.694 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.5833000000000003e-05 2024-09-13T15:05:04.694 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.694 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.694 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.694 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.642711+0000", 2024-09-13T15:05:04.694 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010397805 2024-09-13T15:05:04.694 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.695 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.695 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.695 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.647828+0000", 2024-09-13T15:05:04.695 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0051175079999999998 2024-09-13T15:05:04.695 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.695 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.695 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.695 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648011+0000", 2024-09-13T15:05:04.695 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00018260600000000001 2024-09-13T15:05:04.695 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.695 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.695 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.695 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648047+0000", 2024-09-13T15:05:04.696 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5902999999999998e-05 2024-09-13T15:05:04.696 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.696 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.696 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.696 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648129+0000", 2024-09-13T15:05:04.696 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1773000000000005e-05 2024-09-13T15:05:04.696 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.696 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.696 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.696 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.696 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.696 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142964 9.3 9:c3b98c18:::benchmark_data_smithi096_256909_object23556:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:05:04.696 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.448630+0000", 2024-09-13T15:05:04.697 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.13777806299998, 2024-09-13T15:05:04.697 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.199687605, 2024-09-13T15:05:04.697 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.697 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.697 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.697 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.697 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.697 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142964 2024-09-13T15:05:04.697 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.697 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.697 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.697 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.697 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448630+0000", 2024-09-13T15:05:04.698 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.698 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.698 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.698 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.698 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448630+0000", 2024-09-13T15:05:04.698 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.698 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.698 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.698 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.698 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448629+0000", 2024-09-13T15:05:04.698 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.999999 2024-09-13T15:05:04.698 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.699 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.699 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.699 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448637+0000", 2024-09-13T15:05:04.699 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4489999999999999e-06 2024-09-13T15:05:04.699 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.699 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.699 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.699 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448647+0000", 2024-09-13T15:05:04.699 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7040000000000006e-06 2024-09-13T15:05:04.699 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.699 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.699 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.699 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448653+0000", 2024-09-13T15:05:04.700 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.6629999999999998e-06 2024-09-13T15:05:04.700 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.700 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.700 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.700 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448699+0000", 2024-09-13T15:05:04.700 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.6468999999999998e-05 2024-09-13T15:05:04.700 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.700 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.700 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.700 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.448707+0000", 2024-09-13T15:05:04.700 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.8939999999999993e-06 2024-09-13T15:05:04.700 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.700 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.701 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.701 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.634596+0000", 2024-09-13T15:05:04.701 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.185889134 2024-09-13T15:05:04.701 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.701 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.701 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.701 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.634677+0000", 2024-09-13T15:05:04.701 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.1176999999999999e-05 2024-09-13T15:05:04.701 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.701 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.701 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.701 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.634764+0000", 2024-09-13T15:05:04.701 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.7028999999999996e-05 2024-09-13T15:05:04.702 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.702 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.702 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.702 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.645444+0000", 2024-09-13T15:05:04.702 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.010679711 2024-09-13T15:05:04.702 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.702 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.702 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.702 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648162+0000", 2024-09-13T15:05:04.702 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0027181900000000001 2024-09-13T15:05:04.702 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.702 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.702 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.703 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648189+0000", 2024-09-13T15:05:04.703 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.7257999999999999e-05 2024-09-13T15:05:04.703 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.703 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.703 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.703 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648228+0000", 2024-09-13T15:05:04.703 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.8260000000000003e-05 2024-09-13T15:05:04.703 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.703 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.703 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.703 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648317+0000", 2024-09-13T15:05:04.703 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.9621000000000001e-05 2024-09-13T15:05:04.703 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.704 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.704 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.704 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.704 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.704 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:142987 9.3 9:c82bbcac:::benchmark_data_smithi096_256909_object23579:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:05:04.704 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.519042+0000", 2024-09-13T15:05:04.704 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 314.06736597000003, 2024-09-13T15:05:04.704 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.1294554480000001, 2024-09-13T15:05:04.704 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.704 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.704 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.704 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.704 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.704 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 142987 2024-09-13T15:05:04.705 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.705 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.705 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.705 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.705 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519042+0000", 2024-09-13T15:05:04.705 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.705 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.705 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.705 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.705 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519042+0000", 2024-09-13T15:05:04.705 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.705 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.705 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.706 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.706 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519040+0000", 2024-09-13T15:05:04.706 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999986 2024-09-13T15:05:04.706 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.706 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.706 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.706 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519046+0000", 2024-09-13T15:05:04.706 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.9780000000000002e-06 2024-09-13T15:05:04.706 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.706 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.706 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.706 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519059+0000", 2024-09-13T15:05:04.706 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2181000000000001e-05 2024-09-13T15:05:04.707 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.707 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.707 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.707 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519064+0000", 2024-09-13T15:05:04.707 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.5799999999999999e-06 2024-09-13T15:05:04.707 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.707 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.707 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.707 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519127+0000", 2024-09-13T15:05:04.707 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2706999999999994e-05 2024-09-13T15:05:04.707 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.707 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.707 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.708 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.519137+0000", 2024-09-13T15:05:04.708 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.7170000000000003e-06 2024-09-13T15:05:04.708 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.708 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.708 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.708 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.636985+0000", 2024-09-13T15:05:04.708 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.11784839 2024-09-13T15:05:04.708 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.708 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.708 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.708 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.637113+0000", 2024-09-13T15:05:04.708 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012767599999999999 2024-09-13T15:05:04.708 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.708 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.708 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.709 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.637249+0000", 2024-09-13T15:05:04.709 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000136668 2024-09-13T15:05:04.709 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.709 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.709 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.709 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.645451+0000", 2024-09-13T15:05:04.709 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0082014380000000001 2024-09-13T15:05:04.709 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.709 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.709 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.709 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648351+0000", 2024-09-13T15:05:04.709 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.002900026 2024-09-13T15:05:04.709 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.710 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.710 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.710 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648380+0000", 2024-09-13T15:05:04.710 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8717e-05 2024-09-13T15:05:04.710 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.710 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.710 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.710 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648414+0000", 2024-09-13T15:05:04.710 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.4921e-05 2024-09-13T15:05:04.710 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.710 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.710 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.711 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648497+0000", 2024-09-13T15:05:04.711 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.2725999999999994e-05 2024-09-13T15:05:04.711 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.711 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.711 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.711 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.711 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.711 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:143035 9.3 9:c1b6d37f:::benchmark_data_smithi096_256909_object23627:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:05:04.711 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:50.658945+0000", 2024-09-13T15:05:04.711 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 313.927463042, 2024-09-13T15:05:04.711 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.98972997100000004, 2024-09-13T15:05:04.711 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.711 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.712 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.712 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.712 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.712 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 143035 2024-09-13T15:05:04.712 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.712 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.712 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.712 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.712 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.658945+0000", 2024-09-13T15:05:04.712 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.712 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.712 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.712 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.712 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.658945+0000", 2024-09-13T15:05:04.713 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.713 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.713 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.713 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.713 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.658943+0000", 2024-09-13T15:05:04.713 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:05:04.713 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.713 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.713 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.714 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.658951+0000", 2024-09-13T15:05:04.714 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9889999999999992e-06 2024-09-13T15:05:04.714 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.714 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.714 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.714 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.658964+0000", 2024-09-13T15:05:04.714 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.2819999999999999e-05 2024-09-13T15:05:04.714 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.714 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.714 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.714 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.658969+0000", 2024-09-13T15:05:04.714 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.2510000000000002e-06 2024-09-13T15:05:04.714 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.715 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.715 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.715 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.659017+0000", 2024-09-13T15:05:04.715 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7599999999999998e-05 2024-09-13T15:05:04.715 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.715 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.715 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.715 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:50.659023+0000", 2024-09-13T15:05:04.715 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.9079999999999998e-06 2024-09-13T15:05:04.715 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.715 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.715 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.715 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.640211+0000", 2024-09-13T15:05:04.715 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.98118727900000002 2024-09-13T15:05:04.716 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.716 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.716 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.716 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.640301+0000", 2024-09-13T15:05:04.716 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.0450000000000003e-05 2024-09-13T15:05:04.716 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.716 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.716 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.716 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.640380+0000", 2024-09-13T15:05:04.716 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.9170000000000003e-05 2024-09-13T15:05:04.716 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.716 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.716 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.716 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.645658+0000", 2024-09-13T15:05:04.717 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0052772649999999997 2024-09-13T15:05:04.717 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.717 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.717 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.717 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648530+0000", 2024-09-13T15:05:04.717 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0028719829999999998 2024-09-13T15:05:04.717 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.717 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.717 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.717 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648558+0000", 2024-09-13T15:05:04.717 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.8326e-05 2024-09-13T15:05:04.717 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.717 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.717 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.717 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648594+0000", 2024-09-13T15:05:04.718 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.6508999999999999e-05 2024-09-13T15:05:04.718 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.718 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.718 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.718 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648675+0000", 2024-09-13T15:05:04.718 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.0189000000000001e-05 2024-09-13T15:05:04.718 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.718 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.718 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.718 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.718 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.718 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.46040.0:143100 9.3 9:c04321e8:::benchmark_data_smithi096_256909_object23692:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-13T15:05:04.718 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T14:59:51.036355+0000", 2024-09-13T15:05:04.718 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 313.55005302400002, 2024-09-13T15:05:04.718 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.61255426899999998, 2024-09-13T15:05:04.719 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.719 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.719 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.719 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.46040", 2024-09-13T15:05:04.719 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/2793840793", 2024-09-13T15:05:04.719 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 143100 2024-09-13T15:05:04.719 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.719 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.719 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.719 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.719 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036355+0000", 2024-09-13T15:05:04.719 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.719 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.719 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.720 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.720 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036355+0000", 2024-09-13T15:05:04.720 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.720 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.720 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.720 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.720 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036352+0000", 2024-09-13T15:05:04.720 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999976 2024-09-13T15:05:04.720 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.720 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.720 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.720 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036363+0000", 2024-09-13T15:05:04.720 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0889e-05 2024-09-13T15:05:04.721 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.721 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.721 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.721 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036385+0000", 2024-09-13T15:05:04.721 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.1515e-05 2024-09-13T15:05:04.721 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.721 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.721 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.721 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036395+0000", 2024-09-13T15:05:04.721 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0142e-05 2024-09-13T15:05:04.721 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.721 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.721 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.721 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036441+0000", 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5670000000000002e-05 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for peered", 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.036449+0000", 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.4719999999999995e-06 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.642757+0000", 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.60630773500000001 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.642878+0000", 2024-09-13T15:05:04.722 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00012154099999999999 2024-09-13T15:05:04.723 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.723 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.723 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:05:04.723 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.643019+0000", 2024-09-13T15:05:04.723 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00014106700000000001 2024-09-13T15:05:04.723 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.723 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.723 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.723 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.645995+0000", 2024-09-13T15:05:04.723 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0029752060000000002 2024-09-13T15:05:04.723 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.723 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.723 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.723 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648705+0000", 2024-09-13T15:05:04.723 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0027105470000000002 2024-09-13T15:05:04.724 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.724 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.724 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:04.724 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.724 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648796+0000", 2024-09-13T15:05:04.725 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.1189999999999999e-05 2024-09-13T15:05:04.725 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.725 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.725 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.725 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648832+0000", 2024-09-13T15:05:04.725 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 3.5352000000000003e-05 2024-09-13T15:05:04.725 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.725 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.725 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.725 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T14:59:51.648909+0000", 2024-09-13T15:05:04.725 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.7176999999999996e-05 2024-09-13T15:05:04.725 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.725 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.725 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.726 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.726 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.726 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:166645 10.c 10:30951665:::benchmark_data_smithi096_288650_object30953:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2024-09-13T15:05:04.726 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:48.506180+0000", 2024-09-13T15:05:04.726 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 16.080227375, 2024-09-13T15:05:04.726 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.32862386300000002, 2024-09-13T15:05:04.726 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.726 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.726 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.726 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:05:04.726 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:05:04.726 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 166645 2024-09-13T15:05:04.726 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.727 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.727 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.727 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.727 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.506180+0000", 2024-09-13T15:05:04.727 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.727 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.727 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.727 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.727 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.506180+0000", 2024-09-13T15:05:04.727 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.727 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.727 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.727 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.728 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.506179+0000", 2024-09-13T15:05:04.728 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:05:04.728 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.728 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.728 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.728 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.506186+0000", 2024-09-13T15:05:04.728 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 7.3679999999999996e-06 2024-09-13T15:05:04.728 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.728 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.729 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.729 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.506200+0000", 2024-09-13T15:05:04.729 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.4251e-05 2024-09-13T15:05:04.729 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.729 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.729 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.729 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.506206+0000", 2024-09-13T15:05:04.729 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 6.2280000000000003e-06 2024-09-13T15:05:04.729 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.729 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.729 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.729 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.506252+0000", 2024-09-13T15:05:04.729 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.5283e-05 2024-09-13T15:05:04.730 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.730 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.730 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.730 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.506352+0000", 2024-09-13T15:05:04.730 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000100264 2024-09-13T15:05:04.730 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.730 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.730 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,4", 2024-09-13T15:05:04.730 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.506451+0000", 2024-09-13T15:05:04.730 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.8737e-05 2024-09-13T15:05:04.730 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.730 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.730 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.730 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.831000+0000", 2024-09-13T15:05:04.731 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.32454933400000002 2024-09-13T15:05:04.731 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.731 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.731 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.731 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.831025+0000", 2024-09-13T15:05:04.731 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 2.4627e-05 2024-09-13T15:05:04.731 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.731 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.731 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.731 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.834678+0000", 2024-09-13T15:05:04.731 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.0036531689999999999 2024-09-13T15:05:04.731 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.732 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.732 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.732 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.834764+0000", 2024-09-13T15:05:04.732 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 8.6222000000000002e-05 2024-09-13T15:05:04.732 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.732 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.732 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.732 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.834804+0000", 2024-09-13T15:05:04.732 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.0225999999999997e-05 2024-09-13T15:05:04.732 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.732 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.732 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.732 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.733 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.733 INFO:teuthology.orchestra.run.smithi204.stdout: "description": "osd_op(client.36552.0:166653 10.c 10:3a71c85a:::benchmark_data_smithi096_288650_object30961:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2024-09-13T15:05:04.733 INFO:teuthology.orchestra.run.smithi204.stdout: "initiated_at": "2024-09-13T15:04:48.512474+0000", 2024-09-13T15:05:04.733 INFO:teuthology.orchestra.run.smithi204.stdout: "age": 16.073933576999998, 2024-09-13T15:05:04.733 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.33411767399999998, 2024-09-13T15:05:04.733 INFO:teuthology.orchestra.run.smithi204.stdout: "type_data": { 2024-09-13T15:05:04.733 INFO:teuthology.orchestra.run.smithi204.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:04.733 INFO:teuthology.orchestra.run.smithi204.stdout: "client_info": { 2024-09-13T15:05:04.733 INFO:teuthology.orchestra.run.smithi204.stdout: "client": "client.36552", 2024-09-13T15:05:04.733 INFO:teuthology.orchestra.run.smithi204.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:05:04.733 INFO:teuthology.orchestra.run.smithi204.stdout: "tid": 166653 2024-09-13T15:05:04.733 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.734 INFO:teuthology.orchestra.run.smithi204.stdout: "events": [ 2024-09-13T15:05:04.734 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.734 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "initiated", 2024-09-13T15:05:04.734 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.512474+0000", 2024-09-13T15:05:04.734 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.734 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.734 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.734 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "throttled", 2024-09-13T15:05:04.734 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.512474+0000", 2024-09-13T15:05:04.734 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0 2024-09-13T15:05:04.734 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.734 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.735 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "header_read", 2024-09-13T15:05:04.735 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.512472+0000", 2024-09-13T15:05:04.735 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4294967295.9999981 2024-09-13T15:05:04.735 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.735 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.735 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "all_read", 2024-09-13T15:05:04.735 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.512482+0000", 2024-09-13T15:05:04.735 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 1.0040999999999999e-05 2024-09-13T15:05:04.735 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.735 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.735 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "dispatched", 2024-09-13T15:05:04.735 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.831026+0000", 2024-09-13T15:05:04.735 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.318543464 2024-09-13T15:05:04.735 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.736 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.736 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "queued_for_pg", 2024-09-13T15:05:04.736 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.831035+0000", 2024-09-13T15:05:04.736 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 9.4339999999999998e-06 2024-09-13T15:05:04.736 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.736 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.736 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "reached_pg", 2024-09-13T15:05:04.736 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.831076+0000", 2024-09-13T15:05:04.736 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.1146000000000002e-05 2024-09-13T15:05:04.736 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.736 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.736 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "started", 2024-09-13T15:05:04.736 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.831211+0000", 2024-09-13T15:05:04.736 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.00013516600000000001 2024-09-13T15:05:04.737 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.737 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.737 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "waiting for subops from 0,4", 2024-09-13T15:05:04.737 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.831334+0000", 2024-09-13T15:05:04.737 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.000122731 2024-09-13T15:05:04.737 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.737 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.737 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.737 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.834846+0000", 2024-09-13T15:05:04.737 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.003511494 2024-09-13T15:05:04.737 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.737 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.737 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:04.738 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.846439+0000", 2024-09-13T15:05:04.738 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 0.011593365 2024-09-13T15:05:04.738 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.738 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.738 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "op_commit", 2024-09-13T15:05:04.738 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.846494+0000", 2024-09-13T15:05:04.738 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.4766999999999999e-05 2024-09-13T15:05:04.738 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.738 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.738 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "commit_sent", 2024-09-13T15:05:04.738 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.846542+0000", 2024-09-13T15:05:04.738 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 4.7979999999999998e-05 2024-09-13T15:05:04.738 INFO:teuthology.orchestra.run.smithi204.stdout: }, 2024-09-13T15:05:04.739 INFO:teuthology.orchestra.run.smithi204.stdout: { 2024-09-13T15:05:04.739 INFO:teuthology.orchestra.run.smithi204.stdout: "event": "done", 2024-09-13T15:05:04.739 INFO:teuthology.orchestra.run.smithi204.stdout: "time": "2024-09-13T15:04:48.846592+0000", 2024-09-13T15:05:04.739 INFO:teuthology.orchestra.run.smithi204.stdout: "duration": 5.0127000000000003e-05 2024-09-13T15:05:04.739 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.739 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.739 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.739 INFO:teuthology.orchestra.run.smithi204.stdout: } 2024-09-13T15:05:04.739 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-09-13T15:05:04.739 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:05:04.777 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:04.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:04.878 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:04.907 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:04.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:05.009 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:05.056 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:05.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:05.158 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:05.194 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:05.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:05.296 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:05.338 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:05.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:05.440 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:05.478 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:05.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:05.580 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:05.612 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:05.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:05.713 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:05.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:05 smithi204 ceph-mon[87071]: pgmap v1273: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s 2024-09-13T15:05:05.743 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:05.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:05.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:05 smithi096 ceph-mon[98901]: pgmap v1273: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s 2024-09-13T15:05:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:05 smithi096 ceph-mon[102928]: pgmap v1273: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s 2024-09-13T15:05:05.845 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:05.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.893 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:05.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:05.950 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:05:05.994 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:06.026 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:06.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:06.128 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:06.166 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:06.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:06.268 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:06.305 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:06.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:06.406 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:06.448 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:06.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:06.467 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T15:05:06.485 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:06.550 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:06.600 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:06.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:06.702 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:06.743 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:06.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:06.845 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:06.871 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T15:05:06.890 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:06.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:06.906 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:06.906 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:06.907 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:06 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1154780680' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:05:06.907 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:06.907 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:06.907 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:06 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1154780680' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:05:06.993 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:07.026 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:07.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:07.128 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:07.160 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:07.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:06 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1154780680' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:05:07.262 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:07.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.317 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:07.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:07.418 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:07.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.468 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:07.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:07.570 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:07.594 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:07.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:07.696 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:07.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.751 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:07.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:07.852 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:07.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.892 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:07.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:07.993 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:08.032 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:08.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:08.134 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:08.181 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:08.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:08.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:07 smithi204 ceph-mon[87071]: pgmap v1274: 119 pgs: 119 active+clean; 5.5 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 881 op/s 2024-09-13T15:05:08.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:07 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:08.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:07 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1154780680' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:05:08.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:07 smithi204 ceph-mon[87071]: osdmap e379: 8 total, 8 up, 7 in 2024-09-13T15:05:08.283 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:08.313 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:08.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:08.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:05:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:05:07] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.51.0" 2024-09-13T15:05:08.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:07 smithi096 ceph-mon[98901]: pgmap v1274: 119 pgs: 119 active+clean; 5.5 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 881 op/s 2024-09-13T15:05:08.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:07 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:08.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:07 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1154780680' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:05:08.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:07 smithi096 ceph-mon[98901]: osdmap e379: 8 total, 8 up, 7 in 2024-09-13T15:05:08.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:07 smithi096 ceph-mon[102928]: pgmap v1274: 119 pgs: 119 active+clean; 5.5 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 881 op/s 2024-09-13T15:05:08.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:07 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:08.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:07 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1154780680' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:05:08.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:07 smithi096 ceph-mon[102928]: osdmap e379: 8 total, 8 up, 7 in 2024-09-13T15:05:08.415 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:08.439 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:08.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:08.541 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:08.573 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:08.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:08.675 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:08.718 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:08.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:08.820 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:08.860 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:08.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:08.962 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:08.996 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:08.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:09.098 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:09.143 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:09.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:09.245 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:09.275 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:09.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:09.377 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:09.405 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:09.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:09.506 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:09.540 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:09.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:09.642 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:09.671 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:09.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:09.773 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:09.803 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:09.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:09.905 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:09.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.958 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:09.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:10.059 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:10.087 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:10.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:10.189 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:10.190 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:09 smithi096 ceph-mon[98901]: pgmap v1276: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-13T15:05:10.190 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:09 smithi096 ceph-mon[102928]: pgmap v1276: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-13T15:05:10.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:09 smithi204 ceph-mon[87071]: pgmap v1276: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-13T15:05:10.223 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:10.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:10.325 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:10.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.372 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:10.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:05:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:05:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:05:10.473 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:10.506 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:10.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:10.609 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:10.644 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:10.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:10.746 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:10.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.784 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:10.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:10.885 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:10.921 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:10.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:11.022 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:11.062 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:11.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:11.163 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:11.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.208 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:11.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:11.309 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:11.316 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:05:11.353 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:11.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:11.455 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:11.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.489 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:11.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:11.590 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:11.626 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:11.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:11.728 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:11.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.776 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:11.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:11.801 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:05:11.801 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:05:11.833 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:05:11.833 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:05:11.861 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:05:11.861 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:05:11.877 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:11.890 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:11 smithi096 ceph-mon[98901]: pgmap v1277: 119 pgs: 119 active+clean; 5.2 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s 2024-09-13T15:05:11.890 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:11 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:11.890 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:11 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:11.890 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:11 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2352468009' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:11.896 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:05:11.896 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:05:11.909 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:11.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:11.929 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:05:11.930 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:05:11.970 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:05:11.971 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:05:12.003 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:05:12.004 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:05:12.012 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:12.034 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:05:12.034 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:05:12.045 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:12.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:12.146 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:12.172 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:12.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:12.182 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:11 smithi096 ceph-mon[102928]: pgmap v1277: 119 pgs: 119 active+clean; 5.2 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s 2024-09-13T15:05:12.182 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:11 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:12.182 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:11 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:12.182 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:11 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2352468009' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:12.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:11 smithi204 ceph-mon[87071]: pgmap v1277: 119 pgs: 119 active+clean; 5.2 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s 2024-09-13T15:05:12.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:11 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:12.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:11 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:12.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:11 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2352468009' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:12.275 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:12.304 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:12.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:12.406 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:12.432 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:12.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:12.526 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T15:05:12.527 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T15:05:12.534 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:12.584 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:12.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:12.686 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:12.734 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:12.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:12.836 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:12.866 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:12.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:12.967 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:12.986 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:13.003 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:13.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:13.105 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:13.142 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:13.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:13.244 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:13.279 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:13.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:13.381 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:13.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.410 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:13.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:13.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:13 smithi204 ceph-mon[87071]: from='client.36705 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:05:13.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:13 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:13.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:13 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:13.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:13 smithi204 ceph-mon[87071]: pgmap v1278: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T15:05:13.510 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:13.539 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:13 smithi096 ceph-mon[98901]: from='client.36705 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:05:13.539 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:13 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:13.539 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:13 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:13.539 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:13 smithi096 ceph-mon[98901]: pgmap v1278: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T15:05:13.539 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:13 smithi096 ceph-mon[102928]: from='client.36705 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:05:13.539 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:13 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:13.539 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:13 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:13.539 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:13 smithi096 ceph-mon[102928]: pgmap v1278: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T15:05:13.553 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:13.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:13.573 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":1277,"stamp":"2024-09-13T15:05:10.211321+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5551101925,"num_objects":84659,"num_object_clones":0,"num_object_copies":253977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":84659,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":187656,"num_write_kb":8696720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":109,"num_bytes_recovered":8404818,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96381,"ondisk_log_size":96381,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":22495648,"kb_used_data":16311780,"kb_used_omap":29,"kb_used_meta":6183650,"kb_avail":633720416,"statfs":{"total":671965249536,"available":648929705984,"internally_reserved":0,"allocated":16703262720,"data_stored":16684882789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30654,"internal_metadata":6332057666},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,0,5,5,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-718798848,"num_objects":-10968,"num_object_clones":0,"num_object_copies":-32904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-10968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10968,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.739512"},"pg_stats":[{"pgid":"1.11","version":"82'100","reported_seq":362,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.891121+0000","last_change":"2024-09-13T14:59:00.929470+0000","last_active":"2024-09-13T15:05:06.891121+0000","last_peered":"2024-09-13T15:05:06.891121+0000","last_clean":"2024-09-13T15:05:06.891121+0000","last_became_active":"2024-09-13T14:59:00.928726+0000","last_became_peered":"2024-09-13T14:59:00.928726+0000","last_unstale":"2024-09-13T15:05:06.891121+0000","last_undegraded":"2024-09-13T15:05:06.891121+0000","last_fullsized":"2024-09-13T15:05:06.891121+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.087994+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.087994+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:13.136818+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":204,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.893243+0000","last_change":"2024-09-13T14:59:00.935629+0000","last_active":"2024-09-13T15:05:06.893243+0000","last_peered":"2024-09-13T15:05:06.893243+0000","last_clean":"2024-09-13T15:05:06.893243+0000","last_became_active":"2024-09-13T14:59:00.935106+0000","last_became_peered":"2024-09-13T14:59:00.935106+0000","last_unstale":"2024-09-13T15:05:06.893243+0000","last_undegraded":"2024-09-13T15:05:06.893243+0000","last_fullsized":"2024-09-13T15:05:06.893243+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:12.481049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":344,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.471841+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T15:04:58.471841+0000","last_peered":"2024-09-13T15:04:58.471841+0000","last_clean":"2024-09-13T15:04:58.471841+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T15:04:58.471841+0000","last_undegraded":"2024-09-13T15:04:58.471841+0000","last_fullsized":"2024-09-13T15:04:58.471841+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":108,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.473977+0000","last_change":"2024-09-13T14:56:04.643672+0000","last_active":"2024-09-13T15:04:58.473977+0000","last_peered":"2024-09-13T15:04:58.473977+0000","last_clean":"2024-09-13T15:04:58.473977+0000","last_became_active":"2024-09-13T14:55:55.185995+0000","last_became_peered":"2024-09-13T14:55:55.185995+0000","last_unstale":"2024-09-13T15:04:58.473977+0000","last_undegraded":"2024-09-13T15:04:58.473977+0000","last_fullsized":"2024-09-13T15:04:58.473977+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.643614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.643614+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:36.495546+0000","scrub_duration":0.001441681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":351,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.474963+0000","last_change":"2024-09-13T14:59:00.926231+0000","last_active":"2024-09-13T15:04:58.474963+0000","last_peered":"2024-09-13T15:04:58.474963+0000","last_clean":"2024-09-13T15:04:58.474963+0000","last_became_active":"2024-09-13T14:59:00.925808+0000","last_became_peered":"2024-09-13T14:59:00.925808+0000","last_unstale":"2024-09-13T15:04:58.474963+0000","last_undegraded":"2024-09-13T15:04:58.474963+0000","last_fullsized":"2024-09-13T15:04:58.474963+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:39.672143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":653,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.890407+0000","last_change":"2024-09-13T14:59:00.923848+0000","last_active":"2024-09-13T15:05:06.890407+0000","last_peered":"2024-09-13T15:05:06.890407+0000","last_clean":"2024-09-13T15:05:06.890407+0000","last_became_active":"2024-09-13T14:59:00.923522+0000","last_became_peered":"2024-09-13T14:59:00.923522+0000","last_unstale":"2024-09-13T15:05:06.890407+0000","last_undegraded":"2024-09-13T15:05:06.890407+0000","last_fullsized":"2024-09-13T15:05:06.890407+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:35:35.267527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":793,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.889498+0000","last_change":"2024-09-13T14:59:00.930294+0000","last_active":"2024-09-13T15:05:06.889498+0000","last_peered":"2024-09-13T15:05:06.889498+0000","last_clean":"2024-09-13T15:05:06.889498+0000","last_became_active":"2024-09-13T14:59:00.929860+0000","last_became_peered":"2024-09-13T14:59:00.929860+0000","last_unstale":"2024-09-13T15:05:06.889498+0000","last_undegraded":"2024-09-13T15:05:06.889498+0000","last_fullsized":"2024-09-13T15:05:06.889498+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.240882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.240882+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:51.950106+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":647,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.897066+0000","last_change":"2024-09-13T14:59:00.925448+0000","last_active":"2024-09-13T15:05:06.897066+0000","last_peered":"2024-09-13T15:05:06.897066+0000","last_clean":"2024-09-13T15:05:06.897066+0000","last_became_active":"2024-09-13T14:59:00.925152+0000","last_became_peered":"2024-09-13T14:59:00.925152+0000","last_unstale":"2024-09-13T15:05:06.897066+0000","last_undegraded":"2024-09-13T15:05:06.897066+0000","last_fullsized":"2024-09-13T15:05:06.897066+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:06.703642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":345,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.472623+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T15:04:58.472623+0000","last_peered":"2024-09-13T15:04:58.472623+0000","last_clean":"2024-09-13T15:04:58.472623+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T15:04:58.472623+0000","last_undegraded":"2024-09-13T15:04:58.472623+0000","last_fullsized":"2024-09-13T15:04:58.472623+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":108,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.475366+0000","last_change":"2024-09-13T14:56:03.605447+0000","last_active":"2024-09-13T15:04:58.475366+0000","last_peered":"2024-09-13T15:04:58.475366+0000","last_clean":"2024-09-13T15:04:58.475366+0000","last_became_active":"2024-09-13T14:55:55.218939+0000","last_became_peered":"2024-09-13T14:55:55.218939+0000","last_unstale":"2024-09-13T15:04:58.475366+0000","last_undegraded":"2024-09-13T15:04:58.475366+0000","last_fullsized":"2024-09-13T15:04:58.475366+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.605356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.605356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:12.467371+0000","scrub_duration":0.0020269620000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":100,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.481793+0000","last_change":"2024-09-13T14:59:00.925287+0000","last_active":"2024-09-13T15:04:58.481793+0000","last_peered":"2024-09-13T15:04:58.481793+0000","last_clean":"2024-09-13T15:04:58.481793+0000","last_became_active":"2024-09-13T14:59:00.923868+0000","last_became_peered":"2024-09-13T14:59:00.923868+0000","last_unstale":"2024-09-13T15:04:58.481793+0000","last_undegraded":"2024-09-13T15:04:58.481793+0000","last_fullsized":"2024-09-13T15:04:58.481793+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:28.106847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":172,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.489352+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T15:04:58.489352+0000","last_peered":"2024-09-13T15:04:58.489352+0000","last_clean":"2024-09-13T15:04:58.489352+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T15:04:58.489352+0000","last_undegraded":"2024-09-13T15:04:58.489352+0000","last_fullsized":"2024-09-13T15:04:58.489352+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"378'11570","reported_seq":11601,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.667524+0000","last_change":"2024-09-13T15:02:39.884666+0000","last_active":"2024-09-13T15:05:06.667524+0000","last_peered":"2024-09-13T15:05:06.667524+0000","last_clean":"2024-09-13T15:05:06.667524+0000","last_became_active":"2024-09-13T15:02:39.884349+0000","last_became_peered":"2024-09-13T15:02:39.884349+0000","last_unstale":"2024-09-13T15:05:06.667524+0000","last_undegraded":"2024-09-13T15:05:06.667524+0000","last_fullsized":"2024-09-13T15:05:06.667524+0000","mapping_epoch":355,"log_start":"362'4900","ondisk_log_start":"362'4900","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6670,"log_dups_size":0,"ondisk_log_size":6670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350879744,"num_objects":5354,"num_object_clones":0,"num_object_copies":16062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11570,"num_write_kb":541568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":128,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.482057+0000","last_change":"2024-09-13T15:02:53.820519+0000","last_active":"2024-09-13T15:04:58.482057+0000","last_peered":"2024-09-13T15:04:58.482057+0000","last_clean":"2024-09-13T15:04:58.482057+0000","last_became_active":"2024-09-13T14:59:53.222306+0000","last_became_peered":"2024-09-13T14:59:53.222306+0000","last_unstale":"2024-09-13T15:04:58.482057+0000","last_undegraded":"2024-09-13T15:04:58.482057+0000","last_fullsized":"2024-09-13T15:04:58.482057+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:53.820435+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:53.820435+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:40.108691+0000","scrub_duration":0.0030873739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":617,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.894322+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T15:05:06.894322+0000","last_peered":"2024-09-13T15:05:06.894322+0000","last_clean":"2024-09-13T15:05:06.894322+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T15:05:06.894322+0000","last_undegraded":"2024-09-13T15:05:06.894322+0000","last_fullsized":"2024-09-13T15:05:06.894322+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":364,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.474304+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T15:04:58.474304+0000","last_peered":"2024-09-13T15:04:58.474304+0000","last_clean":"2024-09-13T15:04:58.474304+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T15:04:58.474304+0000","last_undegraded":"2024-09-13T15:04:58.474304+0000","last_fullsized":"2024-09-13T15:04:58.474304+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"378'11695","reported_seq":11726,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.767807+0000","last_change":"2024-09-13T15:02:39.921810+0000","last_active":"2024-09-13T15:05:06.767807+0000","last_peered":"2024-09-13T15:05:06.767807+0000","last_clean":"2024-09-13T15:05:06.767807+0000","last_became_active":"2024-09-13T15:02:39.921510+0000","last_became_peered":"2024-09-13T15:02:39.921510+0000","last_unstale":"2024-09-13T15:05:06.767807+0000","last_undegraded":"2024-09-13T15:05:06.767807+0000","last_fullsized":"2024-09-13T15:05:06.767807+0000","mapping_epoch":355,"log_start":"365'6300","ondisk_log_start":"365'6300","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5395,"log_dups_size":0,"ondisk_log_size":5395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:53:53.564891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353697792,"num_objects":5397,"num_object_clones":0,"num_object_copies":16191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11695,"num_write_kb":546944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":577,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.891733+0000","last_change":"2024-09-13T15:00:10.016332+0000","last_active":"2024-09-13T15:05:06.891733+0000","last_peered":"2024-09-13T15:05:06.891733+0000","last_clean":"2024-09-13T15:05:06.891733+0000","last_became_active":"2024-09-13T14:59:53.227343+0000","last_became_peered":"2024-09-13T14:59:53.227343+0000","last_unstale":"2024-09-13T15:05:06.891733+0000","last_undegraded":"2024-09-13T15:05:06.891733+0000","last_fullsized":"2024-09-13T15:05:06.891733+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.016260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.016260+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:49:12.716184+0000","scrub_duration":71,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":344,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.481555+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T15:04:58.481555+0000","last_peered":"2024-09-13T15:04:58.481555+0000","last_clean":"2024-09-13T15:04:58.481555+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T15:04:58.481555+0000","last_undegraded":"2024-09-13T15:04:58.481555+0000","last_fullsized":"2024-09-13T15:04:58.481555+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":409,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.482457+0000","last_change":"2024-09-13T14:59:00.929566+0000","last_active":"2024-09-13T15:04:58.482457+0000","last_peered":"2024-09-13T15:04:58.482457+0000","last_clean":"2024-09-13T15:04:58.482457+0000","last_became_active":"2024-09-13T14:59:00.929271+0000","last_became_peered":"2024-09-13T14:59:00.929271+0000","last_unstale":"2024-09-13T15:04:58.482457+0000","last_undegraded":"2024-09-13T15:04:58.482457+0000","last_fullsized":"2024-09-13T15:04:58.482457+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:25.655536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"379'11695","reported_seq":11755,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:09.505200+0000","last_change":"2024-09-13T15:02:39.880701+0000","last_active":"2024-09-13T15:05:09.505200+0000","last_peered":"2024-09-13T15:05:09.505200+0000","last_clean":"2024-09-13T15:05:09.505200+0000","last_became_active":"2024-09-13T15:02:39.880319+0000","last_became_peered":"2024-09-13T15:02:39.880319+0000","last_unstale":"2024-09-13T15:05:09.505200+0000","last_undegraded":"2024-09-13T15:05:09.505200+0000","last_fullsized":"2024-09-13T15:05:09.505200+0000","mapping_epoch":355,"log_start":"364'6000","ondisk_log_start":"364'6000","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5695,"log_dups_size":0,"ondisk_log_size":5695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:25:44.594262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330760192,"num_objects":5047,"num_object_clones":0,"num_object_copies":15141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11695,"num_write_kb":535744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":211,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.486242+0000","last_change":"2024-09-13T14:56:08.889974+0000","last_active":"2024-09-13T15:04:58.486242+0000","last_peered":"2024-09-13T15:04:58.486242+0000","last_clean":"2024-09-13T15:04:58.486242+0000","last_became_active":"2024-09-13T14:55:55.185755+0000","last_became_peered":"2024-09-13T14:55:55.185755+0000","last_unstale":"2024-09-13T15:04:58.486242+0000","last_undegraded":"2024-09-13T15:04:58.486242+0000","last_fullsized":"2024-09-13T15:04:58.486242+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:08.889814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:08.889814+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:02.911564+0000","scrub_duration":0.0053731569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":354,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.482492+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T15:04:58.482492+0000","last_peered":"2024-09-13T15:04:58.482492+0000","last_clean":"2024-09-13T15:04:58.482492+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T15:04:58.482492+0000","last_undegraded":"2024-09-13T15:04:58.482492+0000","last_fullsized":"2024-09-13T15:04:58.482492+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":182,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.484964+0000","last_change":"2024-09-13T15:02:38.875727+0000","last_active":"2024-09-13T15:04:58.484964+0000","last_peered":"2024-09-13T15:04:58.484964+0000","last_clean":"2024-09-13T15:04:58.484964+0000","last_became_active":"2024-09-13T15:02:38.875372+0000","last_became_peered":"2024-09-13T15:02:38.875372+0000","last_unstale":"2024-09-13T15:04:58.484964+0000","last_undegraded":"2024-09-13T15:04:58.484964+0000","last_fullsized":"2024-09-13T15:04:58.484964+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:08.504810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"379'11700","reported_seq":11760,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:09.513932+0000","last_change":"2024-09-13T15:02:39.921711+0000","last_active":"2024-09-13T15:05:09.513932+0000","last_peered":"2024-09-13T15:05:09.513932+0000","last_clean":"2024-09-13T15:05:09.513932+0000","last_became_active":"2024-09-13T15:02:39.921415+0000","last_became_peered":"2024-09-13T15:02:39.921415+0000","last_unstale":"2024-09-13T15:05:09.513932+0000","last_undegraded":"2024-09-13T15:05:09.513932+0000","last_fullsized":"2024-09-13T15:05:09.513932+0000","mapping_epoch":355,"log_start":"364'6000","ondisk_log_start":"364'6000","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5700,"log_dups_size":0,"ondisk_log_size":5700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:22.478391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330039296,"num_objects":5036,"num_object_clones":0,"num_object_copies":15108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11700,"num_write_kb":535552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":138,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.482461+0000","last_change":"2024-09-13T15:02:55.358839+0000","last_active":"2024-09-13T15:04:58.482461+0000","last_peered":"2024-09-13T15:04:58.482461+0000","last_clean":"2024-09-13T15:04:58.482461+0000","last_became_active":"2024-09-13T14:59:53.220175+0000","last_became_peered":"2024-09-13T14:59:53.220175+0000","last_unstale":"2024-09-13T15:04:58.482461+0000","last_undegraded":"2024-09-13T15:04:58.482461+0000","last_fullsized":"2024-09-13T15:04:58.482461+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.358699+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.358699+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:13:15.538248+0000","scrub_duration":0.0023994049999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":372,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.474778+0000","last_change":"2024-09-13T14:59:00.930126+0000","last_active":"2024-09-13T15:04:58.474778+0000","last_peered":"2024-09-13T15:04:58.474778+0000","last_clean":"2024-09-13T15:04:58.474778+0000","last_became_active":"2024-09-13T14:59:00.929803+0000","last_became_peered":"2024-09-13T14:59:00.929803+0000","last_unstale":"2024-09-13T15:04:58.474778+0000","last_undegraded":"2024-09-13T15:04:58.474778+0000","last_fullsized":"2024-09-13T15:04:58.474778+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:44.456770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":627,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.890860+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T15:05:06.890860+0000","last_peered":"2024-09-13T15:05:06.890860+0000","last_clean":"2024-09-13T15:05:06.890860+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T15:05:06.890860+0000","last_undegraded":"2024-09-13T15:05:06.890860+0000","last_fullsized":"2024-09-13T15:05:06.890860+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"379'11616","reported_seq":11676,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:08.948443+0000","last_change":"2024-09-13T15:02:39.881732+0000","last_active":"2024-09-13T15:05:08.948443+0000","last_peered":"2024-09-13T15:05:08.948443+0000","last_clean":"2024-09-13T15:05:08.948443+0000","last_became_active":"2024-09-13T15:02:39.880853+0000","last_became_peered":"2024-09-13T15:02:39.880853+0000","last_unstale":"2024-09-13T15:05:08.948443+0000","last_undegraded":"2024-09-13T15:05:08.948443+0000","last_fullsized":"2024-09-13T15:05:08.948443+0000","mapping_epoch":355,"log_start":"364'5900","ondisk_log_start":"364'5900","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5716,"log_dups_size":0,"ondisk_log_size":5716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:19.523510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333709312,"num_objects":5092,"num_object_clones":0,"num_object_copies":15276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11616,"num_write_kb":534656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":555,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.472413+0000","last_change":"2024-09-13T15:02:50.810133+0000","last_active":"2024-09-13T15:04:58.472413+0000","last_peered":"2024-09-13T15:04:58.472413+0000","last_clean":"2024-09-13T15:04:58.472413+0000","last_became_active":"2024-09-13T14:59:53.234208+0000","last_became_peered":"2024-09-13T14:59:53.234208+0000","last_unstale":"2024-09-13T15:04:58.472413+0000","last_undegraded":"2024-09-13T15:04:58.472413+0000","last_fullsized":"2024-09-13T15:04:58.472413+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:50.810045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:50.810045+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:37:54.116248+0000","scrub_duration":0.0042318529999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":166,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.890727+0000","last_change":"2024-09-13T14:59:00.936078+0000","last_active":"2024-09-13T15:05:06.890727+0000","last_peered":"2024-09-13T15:05:06.890727+0000","last_clean":"2024-09-13T15:05:06.890727+0000","last_became_active":"2024-09-13T14:59:00.925997+0000","last_became_peered":"2024-09-13T14:59:00.925997+0000","last_unstale":"2024-09-13T15:05:06.890727+0000","last_undegraded":"2024-09-13T15:05:06.890727+0000","last_fullsized":"2024-09-13T15:05:06.890727+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:43.716136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":372,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.485881+0000","last_change":"2024-09-13T14:59:00.934706+0000","last_active":"2024-09-13T15:04:58.485881+0000","last_peered":"2024-09-13T15:04:58.485881+0000","last_clean":"2024-09-13T15:04:58.485881+0000","last_became_active":"2024-09-13T14:59:00.934367+0000","last_became_peered":"2024-09-13T14:59:00.934367+0000","last_unstale":"2024-09-13T15:04:58.485881+0000","last_undegraded":"2024-09-13T15:04:58.485881+0000","last_fullsized":"2024-09-13T15:04:58.485881+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:37:52.860543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"378'11835","reported_seq":11866,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.655325+0000","last_change":"2024-09-13T15:02:39.923388+0000","last_active":"2024-09-13T15:05:06.655325+0000","last_peered":"2024-09-13T15:05:06.655325+0000","last_clean":"2024-09-13T15:05:06.655325+0000","last_became_active":"2024-09-13T15:02:39.921428+0000","last_became_peered":"2024-09-13T15:02:39.921428+0000","last_unstale":"2024-09-13T15:05:06.655325+0000","last_undegraded":"2024-09-13T15:05:06.655325+0000","last_fullsized":"2024-09-13T15:05:06.655325+0000","mapping_epoch":355,"log_start":"363'5100","ondisk_log_start":"363'5100","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6735,"log_dups_size":0,"ondisk_log_size":6735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:55.689072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355139584,"num_objects":5419,"num_object_clones":0,"num_object_copies":16257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11835,"num_write_kb":552128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":319,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.474053+0000","last_change":"2024-09-13T15:02:57.138892+0000","last_active":"2024-09-13T15:04:58.474053+0000","last_peered":"2024-09-13T15:04:58.474053+0000","last_clean":"2024-09-13T15:04:58.474053+0000","last_became_active":"2024-09-13T14:59:53.223473+0000","last_became_peered":"2024-09-13T14:59:53.223473+0000","last_unstale":"2024-09-13T15:04:58.474053+0000","last_undegraded":"2024-09-13T15:04:58.474053+0000","last_fullsized":"2024-09-13T15:04:58.474053+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:57.138789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:57.138789+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:17.296460+0000","scrub_duration":0.0036861670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":162,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.485033+0000","last_change":"2024-09-13T14:48:32.200521+0000","last_active":"2024-09-13T15:04:58.485033+0000","last_peered":"2024-09-13T15:04:58.485033+0000","last_clean":"2024-09-13T15:04:58.485033+0000","last_became_active":"2024-09-13T14:48:32.200256+0000","last_became_peered":"2024-09-13T14:48:32.200256+0000","last_unstale":"2024-09-13T15:04:58.485033+0000","last_undegraded":"2024-09-13T15:04:58.485033+0000","last_fullsized":"2024-09-13T15:04:58.485033+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:38.899629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":359,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.485075+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T15:04:58.485075+0000","last_peered":"2024-09-13T15:04:58.485075+0000","last_clean":"2024-09-13T15:04:58.485075+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T15:04:58.485075+0000","last_undegraded":"2024-09-13T15:04:58.485075+0000","last_fullsized":"2024-09-13T15:04:58.485075+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"378'11564","reported_seq":11595,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.791879+0000","last_change":"2024-09-13T15:02:39.881185+0000","last_active":"2024-09-13T15:05:06.791879+0000","last_peered":"2024-09-13T15:05:06.791879+0000","last_clean":"2024-09-13T15:05:06.791879+0000","last_became_active":"2024-09-13T15:02:39.880841+0000","last_became_peered":"2024-09-13T15:02:39.880841+0000","last_unstale":"2024-09-13T15:05:06.791879+0000","last_undegraded":"2024-09-13T15:05:06.791879+0000","last_fullsized":"2024-09-13T15:05:06.791879+0000","mapping_epoch":355,"log_start":"365'6200","ondisk_log_start":"365'6200","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5364,"log_dups_size":0,"ondisk_log_size":5364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:49:37.338743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353894400,"num_objects":5400,"num_object_clones":0,"num_object_copies":16200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11564,"num_write_kb":542848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":810,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.889655+0000","last_change":"2024-09-13T15:00:10.659910+0000","last_active":"2024-09-13T15:05:06.889655+0000","last_peered":"2024-09-13T15:05:06.889655+0000","last_clean":"2024-09-13T15:05:06.889655+0000","last_became_active":"2024-09-13T14:59:53.216739+0000","last_became_peered":"2024-09-13T14:59:53.216739+0000","last_unstale":"2024-09-13T15:05:06.889655+0000","last_undegraded":"2024-09-13T15:05:06.889655+0000","last_fullsized":"2024-09-13T15:05:06.889655+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.659838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.659838+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:23:09.068190+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":613,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.890930+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T15:05:06.890930+0000","last_peered":"2024-09-13T15:05:06.890930+0000","last_clean":"2024-09-13T15:05:06.890930+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T15:05:06.890930+0000","last_undegraded":"2024-09-13T15:05:06.890930+0000","last_fullsized":"2024-09-13T15:05:06.890930+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":362,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.482666+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T15:04:58.482666+0000","last_peered":"2024-09-13T15:04:58.482666+0000","last_clean":"2024-09-13T15:04:58.482666+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T15:04:58.482666+0000","last_undegraded":"2024-09-13T15:04:58.482666+0000","last_fullsized":"2024-09-13T15:04:58.482666+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"378'11724","reported_seq":11755,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.668345+0000","last_change":"2024-09-13T15:02:39.881232+0000","last_active":"2024-09-13T15:05:06.668345+0000","last_peered":"2024-09-13T15:05:06.668345+0000","last_clean":"2024-09-13T15:05:06.668345+0000","last_became_active":"2024-09-13T15:02:39.880899+0000","last_became_peered":"2024-09-13T15:02:39.880899+0000","last_unstale":"2024-09-13T15:05:06.668345+0000","last_undegraded":"2024-09-13T15:05:06.668345+0000","last_fullsized":"2024-09-13T15:05:06.668345+0000","mapping_epoch":355,"log_start":"365'6300","ondisk_log_start":"365'6300","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5424,"log_dups_size":0,"ondisk_log_size":5424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:53:52.712556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352845824,"num_objects":5384,"num_object_clones":0,"num_object_copies":16152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11724,"num_write_kb":547456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":308,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.475188+0000","last_change":"2024-09-13T15:02:58.197427+0000","last_active":"2024-09-13T15:04:58.475188+0000","last_peered":"2024-09-13T15:04:58.475188+0000","last_clean":"2024-09-13T15:04:58.475188+0000","last_became_active":"2024-09-13T14:59:53.232987+0000","last_became_peered":"2024-09-13T14:59:53.232987+0000","last_unstale":"2024-09-13T15:04:58.475188+0000","last_undegraded":"2024-09-13T15:04:58.475188+0000","last_fullsized":"2024-09-13T15:04:58.475188+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:58.195731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:58.195731+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:51.565489+0000","scrub_duration":0.011077536000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":367,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.890389+0000","last_change":"2024-09-13T14:59:00.931433+0000","last_active":"2024-09-13T15:05:06.890389+0000","last_peered":"2024-09-13T15:05:06.890389+0000","last_clean":"2024-09-13T15:05:06.890389+0000","last_became_active":"2024-09-13T14:59:00.928906+0000","last_became_peered":"2024-09-13T14:59:00.928906+0000","last_unstale":"2024-09-13T15:05:06.890389+0000","last_undegraded":"2024-09-13T15:05:06.890389+0000","last_fullsized":"2024-09-13T15:05:06.890389+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:12.786568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":621,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.892807+0000","last_change":"2024-09-13T14:59:00.925277+0000","last_active":"2024-09-13T15:05:06.892807+0000","last_peered":"2024-09-13T15:05:06.892807+0000","last_clean":"2024-09-13T15:05:06.892807+0000","last_became_active":"2024-09-13T14:59:00.924952+0000","last_became_peered":"2024-09-13T14:59:00.924952+0000","last_unstale":"2024-09-13T15:05:06.892807+0000","last_undegraded":"2024-09-13T15:05:06.892807+0000","last_fullsized":"2024-09-13T15:05:06.892807+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:08.888187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"378'11478","reported_seq":11509,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.759155+0000","last_change":"2024-09-13T15:02:39.880337+0000","last_active":"2024-09-13T15:05:06.759155+0000","last_peered":"2024-09-13T15:05:06.759155+0000","last_clean":"2024-09-13T15:05:06.759155+0000","last_became_active":"2024-09-13T15:02:39.880044+0000","last_became_peered":"2024-09-13T15:02:39.880044+0000","last_unstale":"2024-09-13T15:05:06.759155+0000","last_undegraded":"2024-09-13T15:05:06.759155+0000","last_fullsized":"2024-09-13T15:05:06.759155+0000","mapping_epoch":355,"log_start":"365'6100","ondisk_log_start":"365'6100","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5378,"log_dups_size":0,"ondisk_log_size":5378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:38.582990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347996160,"num_objects":5310,"num_object_clones":0,"num_object_copies":15930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11478,"num_write_kb":537216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":526,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.476467+0000","last_change":"2024-09-13T14:55:58.568852+0000","last_active":"2024-09-13T15:04:58.476467+0000","last_peered":"2024-09-13T15:04:58.476467+0000","last_clean":"2024-09-13T15:04:58.476467+0000","last_became_active":"2024-09-13T14:55:51.169987+0000","last_became_peered":"2024-09-13T14:55:51.169987+0000","last_unstale":"2024-09-13T15:04:58.476467+0000","last_undegraded":"2024-09-13T15:04:58.476467+0000","last_fullsized":"2024-09-13T15:04:58.476467+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.568750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.568750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:56.273429+0000","scrub_duration":0.0074886340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":904,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.477095+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T15:04:58.477095+0000","last_peered":"2024-09-13T15:04:58.477095+0000","last_clean":"2024-09-13T15:04:58.477095+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T15:04:58.477095+0000","last_undegraded":"2024-09-13T15:04:58.477095+0000","last_fullsized":"2024-09-13T15:04:58.477095+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":198,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.475290+0000","last_change":"2024-09-13T14:59:00.930790+0000","last_active":"2024-09-13T15:04:58.475290+0000","last_peered":"2024-09-13T15:04:58.475290+0000","last_clean":"2024-09-13T15:04:58.475290+0000","last_became_active":"2024-09-13T14:59:00.930439+0000","last_became_peered":"2024-09-13T14:59:00.930439+0000","last_unstale":"2024-09-13T15:04:58.475290+0000","last_undegraded":"2024-09-13T15:04:58.475290+0000","last_fullsized":"2024-09-13T15:04:58.475290+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:15.636095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":162,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.484253+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T15:04:58.484253+0000","last_peered":"2024-09-13T15:04:58.484253+0000","last_clean":"2024-09-13T15:04:58.484253+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T15:04:58.484253+0000","last_undegraded":"2024-09-13T15:04:58.484253+0000","last_fullsized":"2024-09-13T15:04:58.484253+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"379'11834","reported_seq":11894,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:09.526424+0000","last_change":"2024-09-13T15:02:39.921581+0000","last_active":"2024-09-13T15:05:09.526424+0000","last_peered":"2024-09-13T15:05:09.526424+0000","last_clean":"2024-09-13T15:05:09.526424+0000","last_became_active":"2024-09-13T15:02:39.921286+0000","last_became_peered":"2024-09-13T15:02:39.921286+0000","last_unstale":"2024-09-13T15:05:09.526424+0000","last_undegraded":"2024-09-13T15:05:09.526424+0000","last_fullsized":"2024-09-13T15:05:09.526424+0000","mapping_epoch":355,"log_start":"365'6100","ondisk_log_start":"365'6100","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5734,"log_dups_size":0,"ondisk_log_size":5734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:54.996704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332005376,"num_objects":5066,"num_object_clones":0,"num_object_copies":15198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11834,"num_write_kb":540800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":781,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.889348+0000","last_change":"2024-09-13T14:55:59.245841+0000","last_active":"2024-09-13T15:05:06.889348+0000","last_peered":"2024-09-13T15:05:06.889348+0000","last_clean":"2024-09-13T15:05:06.889348+0000","last_became_active":"2024-09-13T14:55:51.159487+0000","last_became_peered":"2024-09-13T14:55:51.159487+0000","last_unstale":"2024-09-13T15:05:06.889348+0000","last_undegraded":"2024-09-13T15:05:06.889348+0000","last_fullsized":"2024-09-13T15:05:06.889348+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.245213+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.245213+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:28.202646+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":589,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.476384+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T15:04:58.476384+0000","last_peered":"2024-09-13T15:04:58.476384+0000","last_clean":"2024-09-13T15:04:58.476384+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T15:04:58.476384+0000","last_undegraded":"2024-09-13T15:04:58.476384+0000","last_fullsized":"2024-09-13T15:04:58.476384+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":656,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.891020+0000","last_change":"2024-09-13T14:59:00.928965+0000","last_active":"2024-09-13T15:05:06.891020+0000","last_peered":"2024-09-13T15:05:06.891020+0000","last_clean":"2024-09-13T15:05:06.891020+0000","last_became_active":"2024-09-13T14:59:00.928166+0000","last_became_peered":"2024-09-13T14:59:00.928166+0000","last_unstale":"2024-09-13T15:05:06.891020+0000","last_undegraded":"2024-09-13T15:05:06.891020+0000","last_fullsized":"2024-09-13T15:05:06.891020+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:34.878172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":652,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.897244+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T15:05:06.897244+0000","last_peered":"2024-09-13T15:05:06.897244+0000","last_clean":"2024-09-13T15:05:06.897244+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T15:05:06.897244+0000","last_undegraded":"2024-09-13T15:05:06.897244+0000","last_fullsized":"2024-09-13T15:05:06.897244+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"378'11458","reported_seq":11489,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.659030+0000","last_change":"2024-09-13T15:02:39.921711+0000","last_active":"2024-09-13T15:05:06.659030+0000","last_peered":"2024-09-13T15:05:06.659030+0000","last_clean":"2024-09-13T15:05:06.659030+0000","last_became_active":"2024-09-13T15:02:39.921395+0000","last_became_peered":"2024-09-13T15:02:39.921395+0000","last_unstale":"2024-09-13T15:05:06.659030+0000","last_undegraded":"2024-09-13T15:05:06.659030+0000","last_fullsized":"2024-09-13T15:05:06.659030+0000","mapping_epoch":355,"log_start":"362'4700","ondisk_log_start":"362'4700","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6758,"log_dups_size":0,"ondisk_log_size":6758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:12.076600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347602944,"num_objects":5304,"num_object_clones":0,"num_object_copies":15912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11458,"num_write_kb":536384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":155,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.472913+0000","last_change":"2024-09-13T14:55:59.574827+0000","last_active":"2024-09-13T15:04:58.472913+0000","last_peered":"2024-09-13T15:04:58.472913+0000","last_clean":"2024-09-13T15:04:58.472913+0000","last_became_active":"2024-09-13T14:55:51.162903+0000","last_became_peered":"2024-09-13T14:55:51.162903+0000","last_unstale":"2024-09-13T15:04:58.472913+0000","last_undegraded":"2024-09-13T15:04:58.472913+0000","last_fullsized":"2024-09-13T15:04:58.472913+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.574773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.574773+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:39.074072+0000","scrub_duration":0.001553708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":577,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.891312+0000","last_change":"2024-09-13T14:59:00.950349+0000","last_active":"2024-09-13T15:05:06.891312+0000","last_peered":"2024-09-13T15:05:06.891312+0000","last_clean":"2024-09-13T15:05:06.891312+0000","last_became_active":"2024-09-13T14:59:00.925703+0000","last_became_peered":"2024-09-13T14:59:00.925703+0000","last_unstale":"2024-09-13T15:05:06.891312+0000","last_undegraded":"2024-09-13T15:05:06.891312+0000","last_fullsized":"2024-09-13T15:05:06.891312+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:43.851359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":630,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.890506+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T15:05:06.890506+0000","last_peered":"2024-09-13T15:05:06.890506+0000","last_clean":"2024-09-13T15:05:06.890506+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T15:05:06.890506+0000","last_undegraded":"2024-09-13T15:05:06.890506+0000","last_fullsized":"2024-09-13T15:05:06.890506+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":647,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.892711+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T15:05:06.892711+0000","last_peered":"2024-09-13T15:05:06.892711+0000","last_clean":"2024-09-13T15:05:06.892711+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T15:05:06.892711+0000","last_undegraded":"2024-09-13T15:05:06.892711+0000","last_fullsized":"2024-09-13T15:05:06.892711+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"378'11710","reported_seq":11741,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.801332+0000","last_change":"2024-09-13T15:02:39.882940+0000","last_active":"2024-09-13T15:05:06.801332+0000","last_peered":"2024-09-13T15:05:06.801332+0000","last_clean":"2024-09-13T15:05:06.801332+0000","last_became_active":"2024-09-13T15:02:39.882641+0000","last_became_peered":"2024-09-13T15:02:39.882641+0000","last_unstale":"2024-09-13T15:05:06.801332+0000","last_undegraded":"2024-09-13T15:05:06.801332+0000","last_fullsized":"2024-09-13T15:05:06.801332+0000","mapping_epoch":355,"log_start":"365'6300","ondisk_log_start":"365'6300","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5410,"log_dups_size":0,"ondisk_log_size":5410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:10.481971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348127232,"num_objects":5312,"num_object_clones":0,"num_object_copies":15936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11710,"num_write_kb":544704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":298,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.474590+0000","last_change":"2024-09-13T14:56:01.658240+0000","last_active":"2024-09-13T15:04:58.474590+0000","last_peered":"2024-09-13T15:04:58.474590+0000","last_clean":"2024-09-13T15:04:58.474590+0000","last_became_active":"2024-09-13T14:55:51.160480+0000","last_became_peered":"2024-09-13T14:55:51.160480+0000","last_unstale":"2024-09-13T15:04:58.474590+0000","last_undegraded":"2024-09-13T15:04:58.474590+0000","last_fullsized":"2024-09-13T15:04:58.474590+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.658143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.658143+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:19.262955+0000","scrub_duration":0.0038903079999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":1004,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.475545+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T15:04:58.475545+0000","last_peered":"2024-09-13T15:04:58.475545+0000","last_clean":"2024-09-13T15:04:58.475545+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T15:04:58.475545+0000","last_undegraded":"2024-09-13T15:04:58.475545+0000","last_fullsized":"2024-09-13T15:04:58.475545+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":363,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.485600+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T15:04:58.485600+0000","last_peered":"2024-09-13T15:04:58.485600+0000","last_clean":"2024-09-13T15:04:58.485600+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T15:04:58.485600+0000","last_undegraded":"2024-09-13T15:04:58.485600+0000","last_fullsized":"2024-09-13T15:04:58.485600+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":654,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.892923+0000","last_change":"2024-09-13T14:59:00.930713+0000","last_active":"2024-09-13T15:05:06.892923+0000","last_peered":"2024-09-13T15:05:06.892923+0000","last_clean":"2024-09-13T15:05:06.892923+0000","last_became_active":"2024-09-13T14:59:00.928260+0000","last_became_peered":"2024-09-13T14:59:00.928260+0000","last_unstale":"2024-09-13T15:05:06.892923+0000","last_undegraded":"2024-09-13T15:05:06.892923+0000","last_fullsized":"2024-09-13T15:05:06.892923+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:07.048853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"378'11619","reported_seq":11650,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.592789+0000","last_change":"2024-09-13T15:02:39.882990+0000","last_active":"2024-09-13T15:05:06.592789+0000","last_peered":"2024-09-13T15:05:06.592789+0000","last_clean":"2024-09-13T15:05:06.592789+0000","last_became_active":"2024-09-13T15:02:39.882663+0000","last_became_peered":"2024-09-13T15:02:39.882663+0000","last_unstale":"2024-09-13T15:05:06.592789+0000","last_undegraded":"2024-09-13T15:05:06.592789+0000","last_fullsized":"2024-09-13T15:05:06.592789+0000","mapping_epoch":355,"log_start":"363'5600","ondisk_log_start":"363'5600","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6019,"log_dups_size":0,"ondisk_log_size":6019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:20.368697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350945280,"num_objects":5355,"num_object_clones":0,"num_object_copies":16065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11619,"num_write_kb":543168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":241,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.486161+0000","last_change":"2024-09-13T15:02:49.771643+0000","last_active":"2024-09-13T15:04:58.486161+0000","last_peered":"2024-09-13T15:04:58.486161+0000","last_clean":"2024-09-13T15:04:58.486161+0000","last_became_active":"2024-09-13T14:59:53.249243+0000","last_became_peered":"2024-09-13T14:59:53.249243+0000","last_unstale":"2024-09-13T15:04:58.486161+0000","last_undegraded":"2024-09-13T15:04:58.486161+0000","last_fullsized":"2024-09-13T15:04:58.486161+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:49.771572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:49.771572+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:11:27.741449+0000","scrub_duration":0.0035237329999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":369,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.475095+0000","last_change":"2024-09-13T14:59:00.931326+0000","last_active":"2024-09-13T15:04:58.475095+0000","last_peered":"2024-09-13T15:04:58.475095+0000","last_clean":"2024-09-13T15:04:58.475095+0000","last_became_active":"2024-09-13T14:59:00.931011+0000","last_became_peered":"2024-09-13T14:59:00.931011+0000","last_unstale":"2024-09-13T15:04:58.475095+0000","last_undegraded":"2024-09-13T15:04:58.475095+0000","last_fullsized":"2024-09-13T15:04:58.475095+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:24:41.833263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":1132,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.893099+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T15:05:06.893099+0000","last_peered":"2024-09-13T15:05:06.893099+0000","last_clean":"2024-09-13T15:05:06.893099+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T15:05:06.893099+0000","last_undegraded":"2024-09-13T15:05:06.893099+0000","last_fullsized":"2024-09-13T15:05:06.893099+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":604,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.889868+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T15:05:06.889868+0000","last_peered":"2024-09-13T15:05:06.889868+0000","last_clean":"2024-09-13T15:05:06.889868+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T15:05:06.889868+0000","last_undegraded":"2024-09-13T15:05:06.889868+0000","last_fullsized":"2024-09-13T15:05:06.889868+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"378'11710","reported_seq":11742,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.795727+0000","last_change":"2024-09-13T15:02:39.881285+0000","last_active":"2024-09-13T15:05:06.795727+0000","last_peered":"2024-09-13T15:05:06.795727+0000","last_clean":"2024-09-13T15:05:06.795727+0000","last_became_active":"2024-09-13T15:02:39.880984+0000","last_became_peered":"2024-09-13T15:02:39.880984+0000","last_unstale":"2024-09-13T15:05:06.795727+0000","last_undegraded":"2024-09-13T15:05:06.795727+0000","last_fullsized":"2024-09-13T15:05:06.795727+0000","mapping_epoch":355,"log_start":"365'6300","ondisk_log_start":"365'6300","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5410,"log_dups_size":0,"ondisk_log_size":5410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:31.193802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356188184,"num_objects":5436,"num_object_clones":0,"num_object_copies":16308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5436,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11710,"num_write_kb":548609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":231,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.484689+0000","last_change":"2024-09-13T15:02:51.748425+0000","last_active":"2024-09-13T15:04:58.484689+0000","last_peered":"2024-09-13T15:04:58.484689+0000","last_clean":"2024-09-13T15:04:58.484689+0000","last_became_active":"2024-09-13T14:59:53.226288+0000","last_became_peered":"2024-09-13T14:59:53.226288+0000","last_unstale":"2024-09-13T15:04:58.484689+0000","last_undegraded":"2024-09-13T15:04:58.484689+0000","last_fullsized":"2024-09-13T15:04:58.484689+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:51.748121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:51.748121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:17:47.226084+0000","scrub_duration":0.0026525849999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":629,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.893006+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T15:05:06.893006+0000","last_peered":"2024-09-13T15:05:06.893006+0000","last_clean":"2024-09-13T15:05:06.893006+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T15:05:06.893006+0000","last_undegraded":"2024-09-13T15:05:06.893006+0000","last_fullsized":"2024-09-13T15:05:06.893006+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":591,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.482666+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T15:04:58.482666+0000","last_peered":"2024-09-13T15:04:58.482666+0000","last_clean":"2024-09-13T15:04:58.482666+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T15:04:58.482666+0000","last_undegraded":"2024-09-13T15:04:58.482666+0000","last_fullsized":"2024-09-13T15:04:58.482666+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":617,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.890826+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T15:05:06.890826+0000","last_peered":"2024-09-13T15:05:06.890826+0000","last_clean":"2024-09-13T15:05:06.890826+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T15:05:06.890826+0000","last_undegraded":"2024-09-13T15:05:06.890826+0000","last_fullsized":"2024-09-13T15:05:06.890826+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"378'11837","reported_seq":11868,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.672123+0000","last_change":"2024-09-13T15:02:39.921655+0000","last_active":"2024-09-13T15:05:06.672123+0000","last_peered":"2024-09-13T15:05:06.672123+0000","last_clean":"2024-09-13T15:05:06.672123+0000","last_became_active":"2024-09-13T15:02:39.921331+0000","last_became_peered":"2024-09-13T15:02:39.921331+0000","last_unstale":"2024-09-13T15:05:06.672123+0000","last_undegraded":"2024-09-13T15:05:06.672123+0000","last_fullsized":"2024-09-13T15:05:06.672123+0000","mapping_epoch":355,"log_start":"365'6400","ondisk_log_start":"365'6400","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5437,"log_dups_size":0,"ondisk_log_size":5437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:48.004179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351731712,"num_objects":5367,"num_object_clones":0,"num_object_copies":16101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11837,"num_write_kb":550528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":349,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.484762+0000","last_change":"2024-09-13T15:02:52.361444+0000","last_active":"2024-09-13T15:04:58.484762+0000","last_peered":"2024-09-13T15:04:58.484762+0000","last_clean":"2024-09-13T15:04:58.484762+0000","last_became_active":"2024-09-13T14:59:53.220135+0000","last_became_peered":"2024-09-13T14:59:53.220135+0000","last_unstale":"2024-09-13T15:04:58.484762+0000","last_undegraded":"2024-09-13T15:04:58.484762+0000","last_fullsized":"2024-09-13T15:04:58.484762+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.361359+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.361359+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:35.003140+0000","scrub_duration":0.005845154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":99,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.482034+0000","last_change":"2024-09-13T14:59:00.936064+0000","last_active":"2024-09-13T15:04:58.482034+0000","last_peered":"2024-09-13T15:04:58.482034+0000","last_clean":"2024-09-13T15:04:58.482034+0000","last_became_active":"2024-09-13T14:59:00.932311+0000","last_became_peered":"2024-09-13T14:59:00.932311+0000","last_unstale":"2024-09-13T15:04:58.482034+0000","last_undegraded":"2024-09-13T15:04:58.482034+0000","last_fullsized":"2024-09-13T15:04:58.482034+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:16.456004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":986,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.888756+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T15:05:06.888756+0000","last_peered":"2024-09-13T15:05:06.888756+0000","last_clean":"2024-09-13T15:05:06.888756+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T15:05:06.888756+0000","last_undegraded":"2024-09-13T15:05:06.888756+0000","last_fullsized":"2024-09-13T15:05:06.888756+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":100,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.471954+0000","last_change":"2024-09-13T14:59:00.924324+0000","last_active":"2024-09-13T15:04:58.471954+0000","last_peered":"2024-09-13T15:04:58.471954+0000","last_clean":"2024-09-13T15:04:58.471954+0000","last_became_active":"2024-09-13T14:59:00.923798+0000","last_became_peered":"2024-09-13T14:59:00.923798+0000","last_unstale":"2024-09-13T15:04:58.471954+0000","last_undegraded":"2024-09-13T15:04:58.471954+0000","last_fullsized":"2024-09-13T15:04:58.471954+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:10.994877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"378'11704","reported_seq":11735,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.781190+0000","last_change":"2024-09-13T15:02:39.922368+0000","last_active":"2024-09-13T15:05:06.781190+0000","last_peered":"2024-09-13T15:05:06.781190+0000","last_clean":"2024-09-13T15:05:06.781190+0000","last_became_active":"2024-09-13T15:02:39.922106+0000","last_became_peered":"2024-09-13T15:02:39.922106+0000","last_unstale":"2024-09-13T15:05:06.781190+0000","last_undegraded":"2024-09-13T15:05:06.781190+0000","last_fullsized":"2024-09-13T15:05:06.781190+0000","mapping_epoch":355,"log_start":"365'6300","ondisk_log_start":"365'6300","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5404,"log_dups_size":0,"ondisk_log_size":5404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:13:52.416513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350748672,"num_objects":5352,"num_object_clones":0,"num_object_copies":16056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11704,"num_write_kb":545792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":86,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.474629+0000","last_change":"2024-09-13T15:02:54.193233+0000","last_active":"2024-09-13T15:04:58.474629+0000","last_peered":"2024-09-13T15:04:58.474629+0000","last_clean":"2024-09-13T15:04:58.474629+0000","last_became_active":"2024-09-13T14:59:00.925167+0000","last_became_peered":"2024-09-13T14:59:00.925167+0000","last_unstale":"2024-09-13T15:04:58.474629+0000","last_undegraded":"2024-09-13T15:04:58.474629+0000","last_fullsized":"2024-09-13T15:04:58.474629+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:54.193156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:54.193156+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:16.851666+0000","scrub_duration":0.0040069900000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":638,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.892858+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T15:05:06.892858+0000","last_peered":"2024-09-13T15:05:06.892858+0000","last_clean":"2024-09-13T15:05:06.892858+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T15:05:06.892858+0000","last_undegraded":"2024-09-13T15:05:06.892858+0000","last_fullsized":"2024-09-13T15:05:06.892858+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1191,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.486480+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T15:04:58.486480+0000","last_peered":"2024-09-13T15:04:58.486480+0000","last_clean":"2024-09-13T15:04:58.486480+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T15:04:58.486480+0000","last_undegraded":"2024-09-13T15:04:58.486480+0000","last_fullsized":"2024-09-13T15:04:58.486480+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":345,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.482333+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T15:04:58.482333+0000","last_peered":"2024-09-13T15:04:58.482333+0000","last_clean":"2024-09-13T15:04:58.482333+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T15:04:58.482333+0000","last_undegraded":"2024-09-13T15:04:58.482333+0000","last_fullsized":"2024-09-13T15:04:58.482333+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"82'100","reported_seq":91,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.888603+0000","last_change":"2024-09-13T15:02:51.192587+0000","last_active":"2024-09-13T15:05:06.888603+0000","last_peered":"2024-09-13T15:05:06.888603+0000","last_clean":"2024-09-13T15:05:06.888603+0000","last_became_active":"2024-09-13T15:02:23.878033+0000","last_became_peered":"2024-09-13T15:02:23.878033+0000","last_unstale":"2024-09-13T15:05:06.888603+0000","last_undegraded":"2024-09-13T15:05:06.888603+0000","last_fullsized":"2024-09-13T15:05:06.888603+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:51.192503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:51.192503+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:38.276910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":362,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.485456+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T15:04:58.485456+0000","last_peered":"2024-09-13T15:04:58.485456+0000","last_clean":"2024-09-13T15:04:58.485456+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T15:04:58.485456+0000","last_undegraded":"2024-09-13T15:04:58.485456+0000","last_fullsized":"2024-09-13T15:04:58.485456+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":645,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.891836+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T15:05:06.891836+0000","last_peered":"2024-09-13T15:05:06.891836+0000","last_clean":"2024-09-13T15:05:06.891836+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T15:05:06.891836+0000","last_undegraded":"2024-09-13T15:05:06.891836+0000","last_fullsized":"2024-09-13T15:05:06.891836+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.17","version":"82'100","reported_seq":190,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.475451+0000","last_change":"2024-09-13T15:03:03.108031+0000","last_active":"2024-09-13T15:04:58.475451+0000","last_peered":"2024-09-13T15:04:58.475451+0000","last_clean":"2024-09-13T15:04:58.475451+0000","last_became_active":"2024-09-13T15:02:23.883373+0000","last_became_peered":"2024-09-13T15:02:23.883373+0000","last_unstale":"2024-09-13T15:04:58.475451+0000","last_undegraded":"2024-09-13T15:04:58.475451+0000","last_fullsized":"2024-09-13T15:04:58.475451+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:03.107786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:03.107786+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:17:10.041468+0000","scrub_duration":0.0049024139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":628,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.889707+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T15:05:06.889707+0000","last_peered":"2024-09-13T15:05:06.889707+0000","last_clean":"2024-09-13T15:05:06.889707+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T15:05:06.889707+0000","last_undegraded":"2024-09-13T15:05:06.889707+0000","last_fullsized":"2024-09-13T15:05:06.889707+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":368,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.482195+0000","last_change":"2024-09-13T14:59:00.929406+0000","last_active":"2024-09-13T15:04:58.482195+0000","last_peered":"2024-09-13T15:04:58.482195+0000","last_clean":"2024-09-13T15:04:58.482195+0000","last_became_active":"2024-09-13T14:59:00.928524+0000","last_became_peered":"2024-09-13T14:59:00.928524+0000","last_unstale":"2024-09-13T15:04:58.482195+0000","last_undegraded":"2024-09-13T15:04:58.482195+0000","last_fullsized":"2024-09-13T15:04:58.482195+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:10.609448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":209,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.893649+0000","last_change":"2024-09-13T14:56:01.235655+0000","last_active":"2024-09-13T15:05:06.893649+0000","last_peered":"2024-09-13T15:05:06.893649+0000","last_clean":"2024-09-13T15:05:06.893649+0000","last_became_active":"2024-09-13T14:55:55.195703+0000","last_became_peered":"2024-09-13T14:55:55.195703+0000","last_unstale":"2024-09-13T15:05:06.893649+0000","last_undegraded":"2024-09-13T15:05:06.893649+0000","last_fullsized":"2024-09-13T15:05:06.893649+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.235497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.235497+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:36:33.861613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":342,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.476734+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T15:04:58.476734+0000","last_peered":"2024-09-13T15:04:58.476734+0000","last_clean":"2024-09-13T15:04:58.476734+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T15:04:58.476734+0000","last_undegraded":"2024-09-13T15:04:58.476734+0000","last_fullsized":"2024-09-13T15:04:58.476734+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":349,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.472487+0000","last_change":"2024-09-13T14:59:00.926586+0000","last_active":"2024-09-13T15:04:58.472487+0000","last_peered":"2024-09-13T15:04:58.472487+0000","last_clean":"2024-09-13T15:04:58.472487+0000","last_became_active":"2024-09-13T14:59:00.926206+0000","last_became_peered":"2024-09-13T14:59:00.926206+0000","last_unstale":"2024-09-13T15:04:58.472487+0000","last_undegraded":"2024-09-13T15:04:58.472487+0000","last_fullsized":"2024-09-13T15:04:58.472487+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:17:18.805856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.15","version":"82'100","reported_seq":46,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.482577+0000","last_change":"2024-09-13T15:02:55.870615+0000","last_active":"2024-09-13T15:04:58.482577+0000","last_peered":"2024-09-13T15:04:58.482577+0000","last_clean":"2024-09-13T15:04:58.482577+0000","last_became_active":"2024-09-13T15:02:23.883217+0000","last_became_peered":"2024-09-13T15:02:23.883217+0000","last_unstale":"2024-09-13T15:04:58.482577+0000","last_undegraded":"2024-09-13T15:04:58.482577+0000","last_fullsized":"2024-09-13T15:04:58.482577+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.870528+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.870528+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:10.023787+0000","scrub_duration":0.0047551850000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":632,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.892519+0000","last_change":"2024-09-13T14:59:00.932503+0000","last_active":"2024-09-13T15:05:06.892519+0000","last_peered":"2024-09-13T15:05:06.892519+0000","last_clean":"2024-09-13T15:05:06.892519+0000","last_became_active":"2024-09-13T14:59:00.931358+0000","last_became_peered":"2024-09-13T14:59:00.931358+0000","last_unstale":"2024-09-13T15:05:06.892519+0000","last_undegraded":"2024-09-13T15:05:06.892519+0000","last_fullsized":"2024-09-13T15:05:06.892519+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:19.226182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":464,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.890201+0000","last_change":"2024-09-13T14:59:00.927060+0000","last_active":"2024-09-13T15:05:06.890201+0000","last_peered":"2024-09-13T15:05:06.890201+0000","last_clean":"2024-09-13T15:05:06.890201+0000","last_became_active":"2024-09-13T14:59:00.925402+0000","last_became_peered":"2024-09-13T14:59:00.925402+0000","last_unstale":"2024-09-13T15:05:06.890201+0000","last_undegraded":"2024-09-13T15:05:06.890201+0000","last_fullsized":"2024-09-13T15:05:06.890201+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:31:08.336446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"82'100","reported_seq":46,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.482451+0000","last_change":"2024-09-13T15:02:56.319389+0000","last_active":"2024-09-13T15:04:58.482451+0000","last_peered":"2024-09-13T15:04:58.482451+0000","last_clean":"2024-09-13T15:04:58.482451+0000","last_became_active":"2024-09-13T15:02:23.876739+0000","last_became_peered":"2024-09-13T15:02:23.876739+0000","last_unstale":"2024-09-13T15:04:58.482451+0000","last_undegraded":"2024-09-13T15:04:58.482451+0000","last_fullsized":"2024-09-13T15:04:58.482451+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:56.319302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:56.319302+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:58.230158+0000","scrub_duration":0.0031182190000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":162,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.486326+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T15:04:58.486326+0000","last_peered":"2024-09-13T15:04:58.486326+0000","last_clean":"2024-09-13T15:04:58.486326+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T15:04:58.486326+0000","last_undegraded":"2024-09-13T15:04:58.486326+0000","last_fullsized":"2024-09-13T15:04:58.486326+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":222,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.897508+0000","last_change":"2024-09-13T14:59:00.924196+0000","last_active":"2024-09-13T15:05:06.897508+0000","last_peered":"2024-09-13T15:05:06.897508+0000","last_clean":"2024-09-13T15:05:06.897508+0000","last_became_active":"2024-09-13T14:59:00.923901+0000","last_became_peered":"2024-09-13T14:59:00.923901+0000","last_unstale":"2024-09-13T15:05:06.897508+0000","last_undegraded":"2024-09-13T15:05:06.897508+0000","last_fullsized":"2024-09-13T15:05:06.897508+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:01.913804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"82'100","reported_seq":628,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.476908+0000","last_change":"2024-09-13T15:03:07.166808+0000","last_active":"2024-09-13T15:04:58.476908+0000","last_peered":"2024-09-13T15:04:58.476908+0000","last_clean":"2024-09-13T15:04:58.476908+0000","last_became_active":"2024-09-13T15:02:23.883387+0000","last_became_peered":"2024-09-13T15:02:23.883387+0000","last_unstale":"2024-09-13T15:04:58.476908+0000","last_undegraded":"2024-09-13T15:04:58.476908+0000","last_fullsized":"2024-09-13T15:04:58.476908+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:07.166034+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:07.166034+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:32:42.605595+0000","scrub_duration":0.01094734,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":656,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.897167+0000","last_change":"2024-09-13T14:59:00.924814+0000","last_active":"2024-09-13T15:05:06.897167+0000","last_peered":"2024-09-13T15:05:06.897167+0000","last_clean":"2024-09-13T15:05:06.897167+0000","last_became_active":"2024-09-13T14:59:00.924501+0000","last_became_peered":"2024-09-13T14:59:00.924501+0000","last_unstale":"2024-09-13T15:05:06.897167+0000","last_undegraded":"2024-09-13T15:05:06.897167+0000","last_fullsized":"2024-09-13T15:05:06.897167+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:11:15.382207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":620,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.887139+0000","last_change":"2024-09-13T14:59:00.925431+0000","last_active":"2024-09-13T15:05:06.887139+0000","last_peered":"2024-09-13T15:05:06.887139+0000","last_clean":"2024-09-13T15:05:06.887139+0000","last_became_active":"2024-09-13T14:59:00.925023+0000","last_became_peered":"2024-09-13T14:59:00.925023+0000","last_unstale":"2024-09-13T15:05:06.887139+0000","last_undegraded":"2024-09-13T15:05:06.887139+0000","last_fullsized":"2024-09-13T15:05:06.887139+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:38:18.686253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"82'100","reported_seq":333,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.474230+0000","last_change":"2024-09-13T15:03:04.101898+0000","last_active":"2024-09-13T15:04:58.474230+0000","last_peered":"2024-09-13T15:04:58.474230+0000","last_clean":"2024-09-13T15:04:58.474230+0000","last_became_active":"2024-09-13T15:02:23.884406+0000","last_became_peered":"2024-09-13T15:02:23.884406+0000","last_unstale":"2024-09-13T15:04:58.474230+0000","last_undegraded":"2024-09-13T15:04:58.474230+0000","last_fullsized":"2024-09-13T15:04:58.474230+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:04.101802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:04.101802+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:34.459623+0000","scrub_duration":0.003875819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":152,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.484891+0000","last_change":"2024-09-13T14:59:00.927250+0000","last_active":"2024-09-13T15:04:58.484891+0000","last_peered":"2024-09-13T15:04:58.484891+0000","last_clean":"2024-09-13T15:04:58.484891+0000","last_became_active":"2024-09-13T14:59:00.926953+0000","last_became_peered":"2024-09-13T14:59:00.926953+0000","last_unstale":"2024-09-13T15:04:58.484891+0000","last_undegraded":"2024-09-13T15:04:58.484891+0000","last_fullsized":"2024-09-13T15:04:58.484891+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:58.306547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":361,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.482772+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T15:04:58.482772+0000","last_peered":"2024-09-13T15:04:58.482772+0000","last_clean":"2024-09-13T15:04:58.482772+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T15:04:58.482772+0000","last_undegraded":"2024-09-13T15:04:58.482772+0000","last_fullsized":"2024-09-13T15:04:58.482772+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"82'100","reported_seq":91,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.890768+0000","last_change":"2024-09-13T15:02:52.429623+0000","last_active":"2024-09-13T15:05:06.890768+0000","last_peered":"2024-09-13T15:05:06.890768+0000","last_clean":"2024-09-13T15:05:06.890768+0000","last_became_active":"2024-09-13T15:02:23.880201+0000","last_became_peered":"2024-09-13T15:02:23.880201+0000","last_unstale":"2024-09-13T15:05:06.890768+0000","last_undegraded":"2024-09-13T15:05:06.890768+0000","last_fullsized":"2024-09-13T15:05:06.890768+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.429543+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.429543+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:32.778195+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":454,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.893327+0000","last_change":"2024-09-13T14:59:00.934412+0000","last_active":"2024-09-13T15:05:06.893327+0000","last_peered":"2024-09-13T15:05:06.893327+0000","last_clean":"2024-09-13T15:05:06.893327+0000","last_became_active":"2024-09-13T14:59:00.932986+0000","last_became_peered":"2024-09-13T14:59:00.932986+0000","last_unstale":"2024-09-13T15:05:06.893327+0000","last_undegraded":"2024-09-13T15:05:06.893327+0000","last_fullsized":"2024-09-13T15:05:06.893327+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:21:44.285324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":617,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.890760+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T15:05:06.890760+0000","last_peered":"2024-09-13T15:05:06.890760+0000","last_clean":"2024-09-13T15:05:06.890760+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T15:05:06.890760+0000","last_undegraded":"2024-09-13T15:05:06.890760+0000","last_fullsized":"2024-09-13T15:05:06.890760+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"82'100","reported_seq":46,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.484390+0000","last_change":"2024-09-13T15:02:55.679993+0000","last_active":"2024-09-13T15:04:58.484390+0000","last_peered":"2024-09-13T15:04:58.484390+0000","last_clean":"2024-09-13T15:04:58.484390+0000","last_became_active":"2024-09-13T15:02:23.878770+0000","last_became_peered":"2024-09-13T15:02:23.878770+0000","last_unstale":"2024-09-13T15:04:58.484390+0000","last_undegraded":"2024-09-13T15:04:58.484390+0000","last_fullsized":"2024-09-13T15:04:58.484390+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.679893+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.679893+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:25.717099+0000","scrub_duration":0.0050514899999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":369,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.485392+0000","last_change":"2024-09-13T14:59:00.929751+0000","last_active":"2024-09-13T15:04:58.485392+0000","last_peered":"2024-09-13T15:04:58.485392+0000","last_clean":"2024-09-13T15:04:58.485392+0000","last_became_active":"2024-09-13T14:59:00.929456+0000","last_became_peered":"2024-09-13T14:59:00.929456+0000","last_unstale":"2024-09-13T15:04:58.485392+0000","last_undegraded":"2024-09-13T15:04:58.485392+0000","last_fullsized":"2024-09-13T15:04:58.485392+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:11.462853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":344,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.474710+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T15:04:58.474710+0000","last_peered":"2024-09-13T15:04:58.474710+0000","last_clean":"2024-09-13T15:04:58.474710+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T15:04:58.474710+0000","last_undegraded":"2024-09-13T15:04:58.474710+0000","last_fullsized":"2024-09-13T15:04:58.474710+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":369,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.482697+0000","last_change":"2024-09-13T14:59:00.928898+0000","last_active":"2024-09-13T15:04:58.482697+0000","last_peered":"2024-09-13T15:04:58.482697+0000","last_clean":"2024-09-13T15:04:58.482697+0000","last_became_active":"2024-09-13T14:59:00.928585+0000","last_became_peered":"2024-09-13T14:59:00.928585+0000","last_unstale":"2024-09-13T15:04:58.482697+0000","last_undegraded":"2024-09-13T15:04:58.482697+0000","last_fullsized":"2024-09-13T15:04:58.482697+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:26:57.155149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":383,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.475068+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T15:04:58.475068+0000","last_peered":"2024-09-13T15:04:58.475068+0000","last_clean":"2024-09-13T15:04:58.475068+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T15:04:58.475068+0000","last_undegraded":"2024-09-13T15:04:58.475068+0000","last_fullsized":"2024-09-13T15:04:58.475068+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"82'100","reported_seq":36,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.481197+0000","last_change":"2024-09-13T15:02:23.879862+0000","last_active":"2024-09-13T15:04:58.481197+0000","last_peered":"2024-09-13T15:04:58.481197+0000","last_clean":"2024-09-13T15:04:58.481197+0000","last_became_active":"2024-09-13T15:02:23.879170+0000","last_became_peered":"2024-09-13T15:02:23.879170+0000","last_unstale":"2024-09-13T15:04:58.481197+0000","last_undegraded":"2024-09-13T15:04:58.481197+0000","last_fullsized":"2024-09-13T15:04:58.481197+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:06.945525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:06.945525+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:48.496118+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":188,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.891219+0000","last_change":"2024-09-13T14:59:00.929835+0000","last_active":"2024-09-13T15:05:06.891219+0000","last_peered":"2024-09-13T15:05:06.891219+0000","last_clean":"2024-09-13T15:05:06.891219+0000","last_became_active":"2024-09-13T14:59:00.929514+0000","last_became_peered":"2024-09-13T14:59:00.929514+0000","last_unstale":"2024-09-13T15:05:06.891219+0000","last_undegraded":"2024-09-13T15:05:06.891219+0000","last_fullsized":"2024-09-13T15:05:06.891219+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:48.981240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":405,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.484570+0000","last_change":"2024-09-13T14:59:00.925895+0000","last_active":"2024-09-13T15:04:58.484570+0000","last_peered":"2024-09-13T15:04:58.484570+0000","last_clean":"2024-09-13T15:04:58.484570+0000","last_became_active":"2024-09-13T14:59:00.925564+0000","last_became_peered":"2024-09-13T14:59:00.925564+0000","last_unstale":"2024-09-13T15:04:58.484570+0000","last_undegraded":"2024-09-13T15:04:58.484570+0000","last_fullsized":"2024-09-13T15:04:58.484570+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:23.283927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1d","version":"82'100","reported_seq":93,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.888089+0000","last_change":"2024-09-13T15:02:52.149390+0000","last_active":"2024-09-13T15:05:06.888089+0000","last_peered":"2024-09-13T15:05:06.888089+0000","last_clean":"2024-09-13T15:05:06.888089+0000","last_became_active":"2024-09-13T15:02:23.914809+0000","last_became_peered":"2024-09-13T15:02:23.914809+0000","last_unstale":"2024-09-13T15:05:06.888089+0000","last_undegraded":"2024-09-13T15:05:06.888089+0000","last_fullsized":"2024-09-13T15:05:06.888089+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.149290+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.149290+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:50:54.160344+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":365,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-13T15:04:58.482194+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T15:04:58.482194+0000","last_peered":"2024-09-13T15:04:58.482194+0000","last_clean":"2024-09-13T15:04:58.482194+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T15:04:58.482194+0000","last_undegraded":"2024-09-13T15:04:58.482194+0000","last_fullsized":"2024-09-13T15:04:58.482194+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":670,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.892420+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T15:05:06.892420+0000","last_peered":"2024-09-13T15:05:06.892420+0000","last_clean":"2024-09-13T15:05:06.892420+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T15:05:06.892420+0000","last_undegraded":"2024-09-13T15:05:06.892420+0000","last_fullsized":"2024-09-13T15:05:06.892420+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5546311704,"num_objects":84631,"num_object_clones":0,"num_object_copies":253893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":84631,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":186749,"num_write_kb":8684097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16537300992,"data_stored":16537288776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92249,"ondisk_log_size":92249,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":16172,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397974,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2600136,"kb_used_data":1719040,"kb_used_omap":1,"kb_used_meta":881086,"kb_avail":91145016,"statfs":{"total":95995035648,"available":93332496384,"internally_reserved":0,"allocated":1760296960,"data_stored":1757666446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":902232408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594203,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2731264,"kb_used_data":2421432,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91013888,"statfs":{"total":95995035648,"available":93198221312,"internally_reserved":0,"allocated":2479546368,"data_stored":2476923098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790432,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2930488,"kb_used_data":1749232,"kb_used_omap":1,"kb_used_meta":1181246,"kb_avail":90814664,"statfs":{"total":95995035648,"available":92994215936,"internally_reserved":0,"allocated":1791213568,"data_stored":1788594907,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1209596248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986661,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4346748,"kb_used_data":3457716,"kb_used_omap":1,"kb_used_meta":888894,"kb_avail":89398404,"statfs":{"total":95995035648,"available":91543965696,"internally_reserved":0,"allocated":3540701184,"data_stored":3538077638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910227800},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":0,"seq":1438814044208,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762636,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3331080,"kb_used_data":2322048,"kb_used_omap":8,"kb_used_meta":1009015,"kb_avail":90414072,"statfs":{"total":95995035648,"available":92584009728,"internally_reserved":0,"allocated":2377777152,"data_stored":2375150697,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":1033231818},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926162,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2311684,"kb_used_data":1332984,"kb_used_omap":7,"kb_used_meta":978680,"kb_avail":91433468,"statfs":{"total":95995035648,"available":93627871232,"internally_reserved":0,"allocated":1364975616,"data_stored":1362356446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":1002169033},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122402,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4244248,"kb_used_data":3309328,"kb_used_omap":7,"kb_used_meta":934904,"kb_avail":89500904,"statfs":{"total":95995035648,"available":91648925696,"internally_reserved":0,"allocated":3388751872,"data_stored":3386113557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":957342287},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3354595328,"data_stored":3354591256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1347026944,"data_stored":1347026944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2367225856,"data_stored":2367225856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3510833152,"data_stored":3510829080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1756561408,"data_stored":1756561408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2461601792,"data_stored":2461597720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1739456512,"data_stored":1739456512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:05:13.579 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:05:13.654 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:13.696 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:13.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:13.798 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:13.833 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:13.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:13.935 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:13.972 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:13.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:14.073 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:14.103 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:14.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:14.205 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:14.233 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:14.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:14.335 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:14.365 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:14.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:14.467 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:14.501 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:14.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:14.603 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:14.630 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:14.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:14.731 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:14.771 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:14.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:14.813 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:14.873 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:14.911 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:14.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:15.013 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:15.044 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:15.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:15.146 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:15.172 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:15.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:15.274 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:15.294 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:15 smithi204 ceph-mon[87071]: pgmap v1279: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T15:05:15.302 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:15.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:15.404 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:15.446 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:15.524 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:15 smithi096 ceph-mon[98901]: pgmap v1279: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T15:05:15.524 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:15 smithi096 ceph-mon[102928]: pgmap v1279: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T15:05:15.547 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:15.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.583 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:15.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:15.684 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:15.713 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:15.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:15.815 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:15.853 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:15.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:15.955 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:15.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.985 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:15.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:16.086 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:16.114 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:05:16.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.121 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:16.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:16.222 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:16.262 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:16.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:16.364 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:16.413 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:16.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:16.514 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:16.527 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:05:16.527 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:05:16.538 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:16.543 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:16.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:16.646 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:16.683 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:16.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:16.785 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:16.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.821 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:16.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:16.922 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:16.957 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:16.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:17.059 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:17.085 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:17.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:17.187 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:17.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.222 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:17.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:17.323 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:17.348 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:17.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:17.450 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:17.488 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:17.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:17.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:17 smithi096 ceph-mon[98901]: pgmap v1280: 119 pgs: 119 active+clean; 4.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T15:05:17.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:17 smithi096 ceph-mon[102928]: pgmap v1280: 119 pgs: 119 active+clean; 4.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T15:05:17.589 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:17.617 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:17.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:17.719 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:17.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:17 smithi204 ceph-mon[87071]: pgmap v1280: 119 pgs: 119 active+clean; 4.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T15:05:17.752 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:17.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:17.854 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:17.888 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:18.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:18.105 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:18.144 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:18.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:18.246 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:18.267 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:05:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:05:17] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.51.0" 2024-09-13T15:05:18.276 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:18.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:18.378 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:18.414 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:18.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:18.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:18.551 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:18.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:18.652 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:18.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.684 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:18.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:18.785 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:18.822 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:18.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:18.924 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:18.961 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:18.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:19.063 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:19.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.096 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:19.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:19.197 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:19.232 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:19.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:19.334 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:19.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.379 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:19.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:19.480 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:19.513 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:19.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:19.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:19 smithi096 ceph-mon[98901]: pgmap v1281: 119 pgs: 119 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 977 op/s 2024-09-13T15:05:19.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:19 smithi096 ceph-mon[102928]: pgmap v1281: 119 pgs: 119 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 977 op/s 2024-09-13T15:05:19.615 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:19.651 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:19.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:19.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:19 smithi204 ceph-mon[87071]: pgmap v1281: 119 pgs: 119 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 977 op/s 2024-09-13T15:05:19.752 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:19.791 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:19.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:19.893 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:19.929 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:19.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:20.031 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:20.070 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:20.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:20.172 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:20.215 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:20.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:20.317 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:20.347 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:20.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:20.448 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:05:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:05:20.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:05:20.487 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:20.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:20.589 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:20.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.628 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:20.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:20.729 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:20.763 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:20.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:20.865 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:20.896 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:20.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:20.998 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:21.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.034 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:21.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:21.135 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:21.172 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:21.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:21.274 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:21.308 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:21.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:21.410 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:21.444 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:21.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:21.546 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:21.574 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:21.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:21.676 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:21.697 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:21.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:21.799 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:21.824 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:21.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:21.926 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:21.947 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:21.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:22.049 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:22.072 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:22.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:22.174 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:22.203 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:22.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:21 smithi204 ceph-mon[87071]: pgmap v1282: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T15:05:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:22.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:22.305 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:22.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:21 smithi096 ceph-mon[98901]: pgmap v1282: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T15:05:22.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:22.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:22.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:22.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:21 smithi096 ceph-mon[102928]: pgmap v1282: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T15:05:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:22.344 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:22.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:22.440 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T15:05:22.446 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:22.515 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:22.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:22.616 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:22.655 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:22.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:22.754 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:22.756 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:22.803 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:22.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:22.905 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:22.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.948 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:22.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:23.049 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:23.089 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:23.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:23.190 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:23.225 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:23.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:23.327 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:23.369 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:23.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:23.471 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:23.503 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:23.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:23.605 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:23.632 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:23.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:23.734 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:23.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.772 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:23.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:23.874 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:23.886 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T15:05:23.915 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:23.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:24.017 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:24.044 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:24.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:24.146 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:24.171 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[98901]: pgmap v1283: 119 pgs: 119 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.28k op/s 2024-09-13T15:05:24.171 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4172506030' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:24.171 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:24.171 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:24.171 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:24.171 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:05:24.171 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:05:24.171 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:24.171 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[102928]: pgmap v1283: 119 pgs: 119 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.28k op/s 2024-09-13T15:05:24.172 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4172506030' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:24.172 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:24.172 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:24.172 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:24.172 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:05:24.172 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:05:24.172 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:23 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:24.187 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:24.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:24.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:23 smithi204 ceph-mon[87071]: pgmap v1283: 119 pgs: 119 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.28k op/s 2024-09-13T15:05:24.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:23 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4172506030' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:24.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:23 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:24.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:23 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:24.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:23 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:24.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:23 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:05:24.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:23 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:05:24.220 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:23 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:24.289 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:24.323 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:24.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:24.425 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:24.464 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:24.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:24.566 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:24.598 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:24.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:24.699 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:24.728 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:24.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:24.830 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:24.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.861 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:24.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:24.962 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:25.011 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:25.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:25.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:24 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:25.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:24 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:25.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:24 smithi096 ceph-mon[98901]: osdmap e380: 8 total, 8 up, 7 in 2024-09-13T15:05:25.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:24 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:25.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:24 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:25.074 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:24 smithi096 ceph-mon[102928]: osdmap e380: 8 total, 8 up, 7 in 2024-09-13T15:05:25.113 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:25.148 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:24 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:25.148 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:24 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:25.148 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:24 smithi204 ceph-mon[87071]: osdmap e380: 8 total, 8 up, 7 in 2024-09-13T15:05:25.154 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:25.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:25.256 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:25.284 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:25.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:25.386 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:25.419 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:25.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:25.521 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:25.558 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:25.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:25.659 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:25.687 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:25.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:25.789 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:25.821 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:25.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:25.923 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:25.957 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:25.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:25.970 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:25 smithi096 ceph-mon[102928]: pgmap v1285: 119 pgs: 119 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-13T15:05:25.970 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:25 smithi096 ceph-mon[98901]: pgmap v1285: 119 pgs: 119 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-13T15:05:25.979 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:26.060 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:26.097 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:26.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:26.199 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:26.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:25 smithi204 ceph-mon[87071]: pgmap v1285: 119 pgs: 119 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-13T15:05:26.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.246 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:26.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:26.347 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:26.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.383 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:26.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:26.484 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:26.519 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:26.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:26.621 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:26.658 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:26.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:26.760 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:26.794 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:26.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:26.896 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:26.927 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:26.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:27.029 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:27.062 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:27.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:27.163 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:27.193 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:27.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:27.295 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:27.325 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:27.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:27.427 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:27.464 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:27.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:27.566 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:27.594 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:27.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:27.696 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:27.729 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:27.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:27.760 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:05:27.760 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:05:27.789 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:05:27.789 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:05:27.821 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:05:27.821 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:05:27.832 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:27.851 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:05:27.851 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:05:27.871 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:27.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:27.877 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:05:27.878 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:05:27.912 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:05:27.912 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:05:27.936 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:05:27.937 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:05:27.973 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:27.979 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:05:27.979 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:05:27.979 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:27 smithi096 ceph-mon[98901]: pgmap v1286: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T15:05:27.980 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:27 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/37767985' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:27.980 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:27 smithi096 ceph-mon[102928]: pgmap v1286: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T15:05:27.980 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:27 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/37767985' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:27.980 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:05:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:05:27] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.51.0" 2024-09-13T15:05:28.009 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:28.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:28.111 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:28.159 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:28.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:28.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:27 smithi204 ceph-mon[87071]: pgmap v1286: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T15:05:28.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:27 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/37767985' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:28.260 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:28.292 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:28.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:28.332 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T15:05:28.334 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T15:05:28.394 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:28.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.422 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:28.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:28.523 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:28.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.554 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:28.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:28.655 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:28.690 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:28.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:28.792 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:28.822 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:28.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:28.925 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:28.956 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:28.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:29.018 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:05:29.058 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:29.094 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:29.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:29.196 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:29.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.235 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:29.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:29.336 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:29.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.375 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:29.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:29.476 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:29.517 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:29.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:29.619 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:29.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.649 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:29.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:29.717 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":1287,"stamp":"2024-09-13T15:05:28.216302+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4210366437,"num_objects":64201,"num_object_clones":0,"num_object_copies":192603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64201,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":208114,"num_write_kb":8696720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":109,"num_bytes_recovered":8404818,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96439,"ondisk_log_size":96439,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18808196,"kb_used_data":12564420,"kb_used_omap":29,"kb_used_meta":6243298,"kb_avail":637407868,"statfs":{"total":671965249536,"available":652705656832,"internally_reserved":0,"allocated":12865966080,"data_stored":12847508259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30654,"internal_metadata":6393137218},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-703987712,"num_objects":-10742,"num_object_clones":0,"num_object_copies":-32226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-10742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10742,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003044"},"pg_stats":[{"pgid":"1.11","version":"82'100","reported_seq":365,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.885693+0000","last_change":"2024-09-13T14:59:00.929470+0000","last_active":"2024-09-13T15:05:23.885693+0000","last_peered":"2024-09-13T15:05:23.885693+0000","last_clean":"2024-09-13T15:05:23.885693+0000","last_became_active":"2024-09-13T14:59:00.928726+0000","last_became_peered":"2024-09-13T14:59:00.928726+0000","last_unstale":"2024-09-13T15:05:23.885693+0000","last_undegraded":"2024-09-13T15:05:23.885693+0000","last_fullsized":"2024-09-13T15:05:23.885693+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.087994+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.087994+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:13.136818+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":206,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.899759+0000","last_change":"2024-09-13T14:59:00.935629+0000","last_active":"2024-09-13T15:05:23.899759+0000","last_peered":"2024-09-13T15:05:23.899759+0000","last_clean":"2024-09-13T15:05:23.899759+0000","last_became_active":"2024-09-13T14:59:00.935106+0000","last_became_peered":"2024-09-13T14:59:00.935106+0000","last_unstale":"2024-09-13T15:05:23.899759+0000","last_undegraded":"2024-09-13T15:05:23.899759+0000","last_fullsized":"2024-09-13T15:05:23.899759+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:12.481049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":346,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.878568+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T15:05:06.878568+0000","last_peered":"2024-09-13T15:05:06.878568+0000","last_clean":"2024-09-13T15:05:06.878568+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T15:05:06.878568+0000","last_undegraded":"2024-09-13T15:05:06.878568+0000","last_fullsized":"2024-09-13T15:05:06.878568+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":110,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.886576+0000","last_change":"2024-09-13T14:56:04.643672+0000","last_active":"2024-09-13T15:05:06.886576+0000","last_peered":"2024-09-13T15:05:06.886576+0000","last_clean":"2024-09-13T15:05:06.886576+0000","last_became_active":"2024-09-13T14:55:55.185995+0000","last_became_peered":"2024-09-13T14:55:55.185995+0000","last_unstale":"2024-09-13T15:05:06.886576+0000","last_undegraded":"2024-09-13T15:05:06.886576+0000","last_fullsized":"2024-09-13T15:05:06.886576+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.643614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.643614+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:36.495546+0000","scrub_duration":0.001441681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":353,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.887514+0000","last_change":"2024-09-13T14:59:00.926231+0000","last_active":"2024-09-13T15:05:06.887514+0000","last_peered":"2024-09-13T15:05:06.887514+0000","last_clean":"2024-09-13T15:05:06.887514+0000","last_became_active":"2024-09-13T14:59:00.925808+0000","last_became_peered":"2024-09-13T14:59:00.925808+0000","last_unstale":"2024-09-13T15:05:06.887514+0000","last_undegraded":"2024-09-13T15:05:06.887514+0000","last_fullsized":"2024-09-13T15:05:06.887514+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:39.672143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":655,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.887539+0000","last_change":"2024-09-13T14:59:00.923848+0000","last_active":"2024-09-13T15:05:23.887539+0000","last_peered":"2024-09-13T15:05:23.887539+0000","last_clean":"2024-09-13T15:05:23.887539+0000","last_became_active":"2024-09-13T14:59:00.923522+0000","last_became_peered":"2024-09-13T14:59:00.923522+0000","last_unstale":"2024-09-13T15:05:23.887539+0000","last_undegraded":"2024-09-13T15:05:23.887539+0000","last_fullsized":"2024-09-13T15:05:23.887539+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:35:35.267527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":796,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.897659+0000","last_change":"2024-09-13T14:59:00.930294+0000","last_active":"2024-09-13T15:05:23.897659+0000","last_peered":"2024-09-13T15:05:23.897659+0000","last_clean":"2024-09-13T15:05:23.897659+0000","last_became_active":"2024-09-13T14:59:00.929860+0000","last_became_peered":"2024-09-13T14:59:00.929860+0000","last_unstale":"2024-09-13T15:05:23.897659+0000","last_undegraded":"2024-09-13T15:05:23.897659+0000","last_fullsized":"2024-09-13T15:05:23.897659+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.240882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.240882+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:51.950106+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":649,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.888780+0000","last_change":"2024-09-13T14:59:00.925448+0000","last_active":"2024-09-13T15:05:23.888780+0000","last_peered":"2024-09-13T15:05:23.888780+0000","last_clean":"2024-09-13T15:05:23.888780+0000","last_became_active":"2024-09-13T14:59:00.925152+0000","last_became_peered":"2024-09-13T14:59:00.925152+0000","last_unstale":"2024-09-13T15:05:23.888780+0000","last_undegraded":"2024-09-13T15:05:23.888780+0000","last_fullsized":"2024-09-13T15:05:23.888780+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:06.703642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":347,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.878453+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T15:05:06.878453+0000","last_peered":"2024-09-13T15:05:06.878453+0000","last_clean":"2024-09-13T15:05:06.878453+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T15:05:06.878453+0000","last_undegraded":"2024-09-13T15:05:06.878453+0000","last_fullsized":"2024-09-13T15:05:06.878453+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":110,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.887050+0000","last_change":"2024-09-13T14:56:03.605447+0000","last_active":"2024-09-13T15:05:06.887050+0000","last_peered":"2024-09-13T15:05:06.887050+0000","last_clean":"2024-09-13T15:05:06.887050+0000","last_became_active":"2024-09-13T14:55:55.218939+0000","last_became_peered":"2024-09-13T14:55:55.218939+0000","last_unstale":"2024-09-13T15:05:06.887050+0000","last_undegraded":"2024-09-13T15:05:06.887050+0000","last_fullsized":"2024-09-13T15:05:06.887050+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.605356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.605356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:12.467371+0000","scrub_duration":0.0020269620000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":102,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.879177+0000","last_change":"2024-09-13T14:59:00.925287+0000","last_active":"2024-09-13T15:05:06.879177+0000","last_peered":"2024-09-13T15:05:06.879177+0000","last_clean":"2024-09-13T15:05:06.879177+0000","last_became_active":"2024-09-13T14:59:00.923868+0000","last_became_peered":"2024-09-13T14:59:00.923868+0000","last_unstale":"2024-09-13T15:05:06.879177+0000","last_undegraded":"2024-09-13T15:05:06.879177+0000","last_fullsized":"2024-09-13T15:05:06.879177+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:28.106847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":174,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.884168+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T15:05:06.884168+0000","last_peered":"2024-09-13T15:05:06.884168+0000","last_clean":"2024-09-13T15:05:06.884168+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T15:05:06.884168+0000","last_undegraded":"2024-09-13T15:05:06.884168+0000","last_fullsized":"2024-09-13T15:05:06.884168+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"380'12969","reported_seq":13002,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:26.669366+0000","last_change":"2024-09-13T15:02:39.884666+0000","last_active":"2024-09-13T15:05:26.669366+0000","last_peered":"2024-09-13T15:05:26.669366+0000","last_clean":"2024-09-13T15:05:26.669366+0000","last_became_active":"2024-09-13T15:02:39.884349+0000","last_became_peered":"2024-09-13T15:02:39.884349+0000","last_unstale":"2024-09-13T15:05:26.669366+0000","last_undegraded":"2024-09-13T15:05:26.669366+0000","last_fullsized":"2024-09-13T15:05:26.669366+0000","mapping_epoch":355,"log_start":"365'6300","ondisk_log_start":"365'6300","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6669,"log_dups_size":0,"ondisk_log_size":6669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259194880,"num_objects":3955,"num_object_clones":0,"num_object_copies":11865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12969,"num_write_kb":541568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":130,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.878825+0000","last_change":"2024-09-13T15:02:53.820519+0000","last_active":"2024-09-13T15:05:06.878825+0000","last_peered":"2024-09-13T15:05:06.878825+0000","last_clean":"2024-09-13T15:05:06.878825+0000","last_became_active":"2024-09-13T14:59:53.222306+0000","last_became_peered":"2024-09-13T14:59:53.222306+0000","last_unstale":"2024-09-13T15:05:06.878825+0000","last_undegraded":"2024-09-13T15:05:06.878825+0000","last_fullsized":"2024-09-13T15:05:06.878825+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:53.820435+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:53.820435+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:40.108691+0000","scrub_duration":0.0030873739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":619,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.883153+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T15:05:23.883153+0000","last_peered":"2024-09-13T15:05:23.883153+0000","last_clean":"2024-09-13T15:05:23.883153+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T15:05:23.883153+0000","last_undegraded":"2024-09-13T15:05:23.883153+0000","last_fullsized":"2024-09-13T15:05:23.883153+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":366,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.886775+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T15:05:06.886775+0000","last_peered":"2024-09-13T15:05:06.886775+0000","last_clean":"2024-09-13T15:05:06.886775+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T15:05:06.886775+0000","last_undegraded":"2024-09-13T15:05:06.886775+0000","last_fullsized":"2024-09-13T15:05:06.886775+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"380'13054","reported_seq":13087,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:26.770586+0000","last_change":"2024-09-13T15:02:39.921810+0000","last_active":"2024-09-13T15:05:26.770586+0000","last_peered":"2024-09-13T15:05:26.770586+0000","last_clean":"2024-09-13T15:05:26.770586+0000","last_became_active":"2024-09-13T15:02:39.921510+0000","last_became_peered":"2024-09-13T15:02:39.921510+0000","last_unstale":"2024-09-13T15:05:26.770586+0000","last_undegraded":"2024-09-13T15:05:26.770586+0000","last_fullsized":"2024-09-13T15:05:26.770586+0000","mapping_epoch":355,"log_start":"366'7600","ondisk_log_start":"366'7600","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5454,"log_dups_size":0,"ondisk_log_size":5454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:53:53.564891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264634368,"num_objects":4038,"num_object_clones":0,"num_object_copies":12114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13054,"num_write_kb":546944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":579,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.887540+0000","last_change":"2024-09-13T15:00:10.016332+0000","last_active":"2024-09-13T15:05:23.887540+0000","last_peered":"2024-09-13T15:05:23.887540+0000","last_clean":"2024-09-13T15:05:23.887540+0000","last_became_active":"2024-09-13T14:59:53.227343+0000","last_became_peered":"2024-09-13T14:59:53.227343+0000","last_unstale":"2024-09-13T15:05:23.887540+0000","last_undegraded":"2024-09-13T15:05:23.887540+0000","last_fullsized":"2024-09-13T15:05:23.887540+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.016260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.016260+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:49:12.716184+0000","scrub_duration":71,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":346,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.879455+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T15:05:06.879455+0000","last_peered":"2024-09-13T15:05:06.879455+0000","last_clean":"2024-09-13T15:05:06.879455+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T15:05:06.879455+0000","last_undegraded":"2024-09-13T15:05:06.879455+0000","last_fullsized":"2024-09-13T15:05:06.879455+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":411,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.882072+0000","last_change":"2024-09-13T14:59:00.929566+0000","last_active":"2024-09-13T15:05:06.882072+0000","last_peered":"2024-09-13T15:05:06.882072+0000","last_clean":"2024-09-13T15:05:06.882072+0000","last_became_active":"2024-09-13T14:59:00.929271+0000","last_became_peered":"2024-09-13T14:59:00.929271+0000","last_unstale":"2024-09-13T15:05:06.882072+0000","last_undegraded":"2024-09-13T15:05:06.882072+0000","last_fullsized":"2024-09-13T15:05:06.882072+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:25.655536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"380'12682","reported_seq":12744,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:24.549433+0000","last_change":"2024-09-13T15:02:39.880701+0000","last_active":"2024-09-13T15:05:24.549433+0000","last_peered":"2024-09-13T15:05:24.549433+0000","last_clean":"2024-09-13T15:05:24.549433+0000","last_became_active":"2024-09-13T15:02:39.880319+0000","last_became_peered":"2024-09-13T15:02:39.880319+0000","last_unstale":"2024-09-13T15:05:24.549433+0000","last_undegraded":"2024-09-13T15:05:24.549433+0000","last_fullsized":"2024-09-13T15:05:24.549433+0000","mapping_epoch":355,"log_start":"366'7000","ondisk_log_start":"366'7000","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5682,"log_dups_size":0,"ondisk_log_size":5682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:25:44.594262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266076160,"num_objects":4060,"num_object_clones":0,"num_object_copies":12180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12682,"num_write_kb":535744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":213,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.885068+0000","last_change":"2024-09-13T14:56:08.889974+0000","last_active":"2024-09-13T15:05:06.885068+0000","last_peered":"2024-09-13T15:05:06.885068+0000","last_clean":"2024-09-13T15:05:06.885068+0000","last_became_active":"2024-09-13T14:55:55.185755+0000","last_became_peered":"2024-09-13T14:55:55.185755+0000","last_unstale":"2024-09-13T15:05:06.885068+0000","last_undegraded":"2024-09-13T15:05:06.885068+0000","last_fullsized":"2024-09-13T15:05:06.885068+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:08.889814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:08.889814+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:02.911564+0000","scrub_duration":0.0053731569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":356,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.879129+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T15:05:06.879129+0000","last_peered":"2024-09-13T15:05:06.879129+0000","last_clean":"2024-09-13T15:05:06.879129+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T15:05:06.879129+0000","last_undegraded":"2024-09-13T15:05:06.879129+0000","last_fullsized":"2024-09-13T15:05:06.879129+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":184,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.883951+0000","last_change":"2024-09-13T15:02:38.875727+0000","last_active":"2024-09-13T15:05:06.883951+0000","last_peered":"2024-09-13T15:05:06.883951+0000","last_clean":"2024-09-13T15:05:06.883951+0000","last_became_active":"2024-09-13T15:02:38.875372+0000","last_became_peered":"2024-09-13T15:02:38.875372+0000","last_unstale":"2024-09-13T15:05:06.883951+0000","last_undegraded":"2024-09-13T15:05:06.883951+0000","last_fullsized":"2024-09-13T15:05:06.883951+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:08.504810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"380'12691","reported_seq":12753,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:24.463655+0000","last_change":"2024-09-13T15:02:39.921711+0000","last_active":"2024-09-13T15:05:24.463655+0000","last_peered":"2024-09-13T15:05:24.463655+0000","last_clean":"2024-09-13T15:05:24.463655+0000","last_became_active":"2024-09-13T15:02:39.921415+0000","last_became_peered":"2024-09-13T15:02:39.921415+0000","last_unstale":"2024-09-13T15:05:24.463655+0000","last_undegraded":"2024-09-13T15:05:24.463655+0000","last_fullsized":"2024-09-13T15:05:24.463655+0000","mapping_epoch":355,"log_start":"366'7000","ondisk_log_start":"366'7000","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5691,"log_dups_size":0,"ondisk_log_size":5691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:22.478391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265093120,"num_objects":4045,"num_object_clones":0,"num_object_copies":12135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12691,"num_write_kb":535552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":140,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.882994+0000","last_change":"2024-09-13T15:02:55.358839+0000","last_active":"2024-09-13T15:05:06.882994+0000","last_peered":"2024-09-13T15:05:06.882994+0000","last_clean":"2024-09-13T15:05:06.882994+0000","last_became_active":"2024-09-13T14:59:53.220175+0000","last_became_peered":"2024-09-13T14:59:53.220175+0000","last_unstale":"2024-09-13T15:05:06.882994+0000","last_undegraded":"2024-09-13T15:05:06.882994+0000","last_fullsized":"2024-09-13T15:05:06.882994+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.358699+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.358699+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:13:15.538248+0000","scrub_duration":0.0023994049999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":374,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.887313+0000","last_change":"2024-09-13T14:59:00.930126+0000","last_active":"2024-09-13T15:05:06.887313+0000","last_peered":"2024-09-13T15:05:06.887313+0000","last_clean":"2024-09-13T15:05:06.887313+0000","last_became_active":"2024-09-13T14:59:00.929803+0000","last_became_peered":"2024-09-13T14:59:00.929803+0000","last_unstale":"2024-09-13T15:05:06.887313+0000","last_undegraded":"2024-09-13T15:05:06.887313+0000","last_fullsized":"2024-09-13T15:05:06.887313+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:44.456770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":629,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.886275+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T15:05:23.886275+0000","last_peered":"2024-09-13T15:05:23.886275+0000","last_clean":"2024-09-13T15:05:23.886275+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T15:05:23.886275+0000","last_undegraded":"2024-09-13T15:05:23.886275+0000","last_fullsized":"2024-09-13T15:05:23.886275+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"380'12589","reported_seq":12651,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.975780+0000","last_change":"2024-09-13T15:02:39.881732+0000","last_active":"2024-09-13T15:05:23.975780+0000","last_peered":"2024-09-13T15:05:23.975780+0000","last_clean":"2024-09-13T15:05:23.975780+0000","last_became_active":"2024-09-13T15:02:39.880853+0000","last_became_peered":"2024-09-13T15:02:39.880853+0000","last_unstale":"2024-09-13T15:05:23.975780+0000","last_undegraded":"2024-09-13T15:05:23.975780+0000","last_fullsized":"2024-09-13T15:05:23.975780+0000","mapping_epoch":355,"log_start":"366'6900","ondisk_log_start":"366'6900","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5689,"log_dups_size":0,"ondisk_log_size":5689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:19.523510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269942784,"num_objects":4119,"num_object_clones":0,"num_object_copies":12357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12589,"num_write_kb":534656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":557,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.878226+0000","last_change":"2024-09-13T15:02:50.810133+0000","last_active":"2024-09-13T15:05:06.878226+0000","last_peered":"2024-09-13T15:05:06.878226+0000","last_clean":"2024-09-13T15:05:06.878226+0000","last_became_active":"2024-09-13T14:59:53.234208+0000","last_became_peered":"2024-09-13T14:59:53.234208+0000","last_unstale":"2024-09-13T15:05:06.878226+0000","last_undegraded":"2024-09-13T15:05:06.878226+0000","last_fullsized":"2024-09-13T15:05:06.878226+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:50.810045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:50.810045+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:37:54.116248+0000","scrub_duration":0.0042318529999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":168,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.885860+0000","last_change":"2024-09-13T14:59:00.936078+0000","last_active":"2024-09-13T15:05:23.885860+0000","last_peered":"2024-09-13T15:05:23.885860+0000","last_clean":"2024-09-13T15:05:23.885860+0000","last_became_active":"2024-09-13T14:59:00.925997+0000","last_became_peered":"2024-09-13T14:59:00.925997+0000","last_unstale":"2024-09-13T15:05:23.885860+0000","last_undegraded":"2024-09-13T15:05:23.885860+0000","last_fullsized":"2024-09-13T15:05:23.885860+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:43.716136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":374,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.882430+0000","last_change":"2024-09-13T14:59:00.934706+0000","last_active":"2024-09-13T15:05:06.882430+0000","last_peered":"2024-09-13T15:05:06.882430+0000","last_clean":"2024-09-13T15:05:06.882430+0000","last_became_active":"2024-09-13T14:59:00.934367+0000","last_became_peered":"2024-09-13T14:59:00.934367+0000","last_unstale":"2024-09-13T15:05:06.882430+0000","last_undegraded":"2024-09-13T15:05:06.882430+0000","last_fullsized":"2024-09-13T15:05:06.882430+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:37:52.860543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"380'13164","reported_seq":13196,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:26.683383+0000","last_change":"2024-09-13T15:02:39.923388+0000","last_active":"2024-09-13T15:05:26.683383+0000","last_peered":"2024-09-13T15:05:26.683383+0000","last_clean":"2024-09-13T15:05:26.683383+0000","last_became_active":"2024-09-13T15:02:39.921428+0000","last_became_peered":"2024-09-13T15:02:39.921428+0000","last_unstale":"2024-09-13T15:05:26.683383+0000","last_undegraded":"2024-09-13T15:05:26.683383+0000","last_fullsized":"2024-09-13T15:05:26.683383+0000","mapping_epoch":355,"log_start":"365'6400","ondisk_log_start":"365'6400","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6764,"log_dups_size":0,"ondisk_log_size":6764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:55.689072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268042240,"num_objects":4090,"num_object_clones":0,"num_object_copies":12270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13164,"num_write_kb":552128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":321,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.886631+0000","last_change":"2024-09-13T15:02:57.138892+0000","last_active":"2024-09-13T15:05:06.886631+0000","last_peered":"2024-09-13T15:05:06.886631+0000","last_clean":"2024-09-13T15:05:06.886631+0000","last_became_active":"2024-09-13T14:59:53.223473+0000","last_became_peered":"2024-09-13T14:59:53.223473+0000","last_unstale":"2024-09-13T15:05:06.886631+0000","last_undegraded":"2024-09-13T15:05:06.886631+0000","last_fullsized":"2024-09-13T15:05:06.886631+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:57.138789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:57.138789+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:17.296460+0000","scrub_duration":0.0036861670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":164,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.884301+0000","last_change":"2024-09-13T14:48:32.200521+0000","last_active":"2024-09-13T15:05:06.884301+0000","last_peered":"2024-09-13T15:05:06.884301+0000","last_clean":"2024-09-13T15:05:06.884301+0000","last_became_active":"2024-09-13T14:48:32.200256+0000","last_became_peered":"2024-09-13T14:48:32.200256+0000","last_unstale":"2024-09-13T15:05:06.884301+0000","last_undegraded":"2024-09-13T15:05:06.884301+0000","last_fullsized":"2024-09-13T15:05:06.884301+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:38.899629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":361,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.882487+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T15:05:06.882487+0000","last_peered":"2024-09-13T15:05:06.882487+0000","last_clean":"2024-09-13T15:05:06.882487+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T15:05:06.882487+0000","last_undegraded":"2024-09-13T15:05:06.882487+0000","last_fullsized":"2024-09-13T15:05:06.882487+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"380'13027","reported_seq":13060,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:26.793579+0000","last_change":"2024-09-13T15:02:39.881185+0000","last_active":"2024-09-13T15:05:26.793579+0000","last_peered":"2024-09-13T15:05:26.793579+0000","last_clean":"2024-09-13T15:05:26.793579+0000","last_became_active":"2024-09-13T15:02:39.880841+0000","last_became_peered":"2024-09-13T15:02:39.880841+0000","last_unstale":"2024-09-13T15:05:26.793579+0000","last_undegraded":"2024-09-13T15:05:26.793579+0000","last_fullsized":"2024-09-13T15:05:26.793579+0000","mapping_epoch":355,"log_start":"366'7600","ondisk_log_start":"366'7600","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5427,"log_dups_size":0,"ondisk_log_size":5427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:49:37.338743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258015232,"num_objects":3937,"num_object_clones":0,"num_object_copies":11811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13027,"num_write_kb":542848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":812,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.897730+0000","last_change":"2024-09-13T15:00:10.659910+0000","last_active":"2024-09-13T15:05:23.897730+0000","last_peered":"2024-09-13T15:05:23.897730+0000","last_clean":"2024-09-13T15:05:23.897730+0000","last_became_active":"2024-09-13T14:59:53.216739+0000","last_became_peered":"2024-09-13T14:59:53.216739+0000","last_unstale":"2024-09-13T15:05:23.897730+0000","last_undegraded":"2024-09-13T15:05:23.897730+0000","last_fullsized":"2024-09-13T15:05:23.897730+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.659838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.659838+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:23:09.068190+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":615,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.885569+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T15:05:23.885569+0000","last_peered":"2024-09-13T15:05:23.885569+0000","last_clean":"2024-09-13T15:05:23.885569+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T15:05:23.885569+0000","last_undegraded":"2024-09-13T15:05:23.885569+0000","last_fullsized":"2024-09-13T15:05:23.885569+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":364,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.883092+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T15:05:06.883092+0000","last_peered":"2024-09-13T15:05:06.883092+0000","last_clean":"2024-09-13T15:05:06.883092+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T15:05:06.883092+0000","last_undegraded":"2024-09-13T15:05:06.883092+0000","last_fullsized":"2024-09-13T15:05:06.883092+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"380'13089","reported_seq":13122,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:26.711333+0000","last_change":"2024-09-13T15:02:39.881232+0000","last_active":"2024-09-13T15:05:26.711333+0000","last_peered":"2024-09-13T15:05:26.711333+0000","last_clean":"2024-09-13T15:05:26.711333+0000","last_became_active":"2024-09-13T15:02:39.880899+0000","last_became_peered":"2024-09-13T15:02:39.880899+0000","last_unstale":"2024-09-13T15:05:26.711333+0000","last_undegraded":"2024-09-13T15:05:26.711333+0000","last_fullsized":"2024-09-13T15:05:26.711333+0000","mapping_epoch":355,"log_start":"366'7700","ondisk_log_start":"366'7700","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5389,"log_dups_size":0,"ondisk_log_size":5389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:53:52.712556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263389184,"num_objects":4019,"num_object_clones":0,"num_object_copies":12057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13089,"num_write_kb":547456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":310,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.886887+0000","last_change":"2024-09-13T15:02:58.197427+0000","last_active":"2024-09-13T15:05:06.886887+0000","last_peered":"2024-09-13T15:05:06.886887+0000","last_clean":"2024-09-13T15:05:06.886887+0000","last_became_active":"2024-09-13T14:59:53.232987+0000","last_became_peered":"2024-09-13T14:59:53.232987+0000","last_unstale":"2024-09-13T15:05:06.886887+0000","last_undegraded":"2024-09-13T15:05:06.886887+0000","last_fullsized":"2024-09-13T15:05:06.886887+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:58.195731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:58.195731+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:51.565489+0000","scrub_duration":0.011077536000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":369,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.897360+0000","last_change":"2024-09-13T14:59:00.931433+0000","last_active":"2024-09-13T15:05:23.897360+0000","last_peered":"2024-09-13T15:05:23.897360+0000","last_clean":"2024-09-13T15:05:23.897360+0000","last_became_active":"2024-09-13T14:59:00.928906+0000","last_became_peered":"2024-09-13T14:59:00.928906+0000","last_unstale":"2024-09-13T15:05:23.897360+0000","last_undegraded":"2024-09-13T15:05:23.897360+0000","last_fullsized":"2024-09-13T15:05:23.897360+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:12.786568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":623,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.898358+0000","last_change":"2024-09-13T14:59:00.925277+0000","last_active":"2024-09-13T15:05:23.898358+0000","last_peered":"2024-09-13T15:05:23.898358+0000","last_clean":"2024-09-13T15:05:23.898358+0000","last_became_active":"2024-09-13T14:59:00.924952+0000","last_became_peered":"2024-09-13T14:59:00.924952+0000","last_unstale":"2024-09-13T15:05:23.898358+0000","last_undegraded":"2024-09-13T15:05:23.898358+0000","last_fullsized":"2024-09-13T15:05:23.898358+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:08.888187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"380'12842","reported_seq":12875,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:26.839828+0000","last_change":"2024-09-13T15:02:39.880337+0000","last_active":"2024-09-13T15:05:26.839828+0000","last_peered":"2024-09-13T15:05:26.839828+0000","last_clean":"2024-09-13T15:05:26.839828+0000","last_became_active":"2024-09-13T15:02:39.880044+0000","last_became_peered":"2024-09-13T15:02:39.880044+0000","last_unstale":"2024-09-13T15:05:26.839828+0000","last_undegraded":"2024-09-13T15:05:26.839828+0000","last_fullsized":"2024-09-13T15:05:26.839828+0000","mapping_epoch":355,"log_start":"366'7400","ondisk_log_start":"366'7400","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5442,"log_dups_size":0,"ondisk_log_size":5442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:38.582990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258605056,"num_objects":3946,"num_object_clones":0,"num_object_copies":11838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12842,"num_write_kb":537216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":528,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.879813+0000","last_change":"2024-09-13T14:55:58.568852+0000","last_active":"2024-09-13T15:05:06.879813+0000","last_peered":"2024-09-13T15:05:06.879813+0000","last_clean":"2024-09-13T15:05:06.879813+0000","last_became_active":"2024-09-13T14:55:51.169987+0000","last_became_peered":"2024-09-13T14:55:51.169987+0000","last_unstale":"2024-09-13T15:05:06.879813+0000","last_undegraded":"2024-09-13T15:05:06.879813+0000","last_fullsized":"2024-09-13T15:05:06.879813+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.568750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.568750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:56.273429+0000","scrub_duration":0.0074886340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":906,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.887196+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T15:05:06.887196+0000","last_peered":"2024-09-13T15:05:06.887196+0000","last_clean":"2024-09-13T15:05:06.887196+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T15:05:06.887196+0000","last_undegraded":"2024-09-13T15:05:06.887196+0000","last_fullsized":"2024-09-13T15:05:06.887196+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":200,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.886977+0000","last_change":"2024-09-13T14:59:00.930790+0000","last_active":"2024-09-13T15:05:06.886977+0000","last_peered":"2024-09-13T15:05:06.886977+0000","last_clean":"2024-09-13T15:05:06.886977+0000","last_became_active":"2024-09-13T14:59:00.930439+0000","last_became_peered":"2024-09-13T14:59:00.930439+0000","last_unstale":"2024-09-13T15:05:06.886977+0000","last_undegraded":"2024-09-13T15:05:06.886977+0000","last_fullsized":"2024-09-13T15:05:06.886977+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:15.636095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":164,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.885230+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T15:05:06.885230+0000","last_peered":"2024-09-13T15:05:06.885230+0000","last_clean":"2024-09-13T15:05:06.885230+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T15:05:06.885230+0000","last_undegraded":"2024-09-13T15:05:06.885230+0000","last_fullsized":"2024-09-13T15:05:06.885230+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"380'12844","reported_seq":12906,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:24.542353+0000","last_change":"2024-09-13T15:02:39.921581+0000","last_active":"2024-09-13T15:05:24.542353+0000","last_peered":"2024-09-13T15:05:24.542353+0000","last_clean":"2024-09-13T15:05:24.542353+0000","last_became_active":"2024-09-13T15:02:39.921286+0000","last_became_peered":"2024-09-13T15:02:39.921286+0000","last_unstale":"2024-09-13T15:05:24.542353+0000","last_undegraded":"2024-09-13T15:05:24.542353+0000","last_fullsized":"2024-09-13T15:05:24.542353+0000","mapping_epoch":355,"log_start":"366'7100","ondisk_log_start":"366'7100","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5744,"log_dups_size":0,"ondisk_log_size":5744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:54.996704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265814016,"num_objects":4056,"num_object_clones":0,"num_object_copies":12168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12844,"num_write_kb":540800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":783,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.897562+0000","last_change":"2024-09-13T14:55:59.245841+0000","last_active":"2024-09-13T15:05:23.897562+0000","last_peered":"2024-09-13T15:05:23.897562+0000","last_clean":"2024-09-13T15:05:23.897562+0000","last_became_active":"2024-09-13T14:55:51.159487+0000","last_became_peered":"2024-09-13T14:55:51.159487+0000","last_unstale":"2024-09-13T15:05:23.897562+0000","last_undegraded":"2024-09-13T15:05:23.897562+0000","last_fullsized":"2024-09-13T15:05:23.897562+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.245213+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.245213+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:28.202646+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":591,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.879732+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T15:05:06.879732+0000","last_peered":"2024-09-13T15:05:06.879732+0000","last_clean":"2024-09-13T15:05:06.879732+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T15:05:06.879732+0000","last_undegraded":"2024-09-13T15:05:06.879732+0000","last_fullsized":"2024-09-13T15:05:06.879732+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":658,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.885667+0000","last_change":"2024-09-13T14:59:00.928965+0000","last_active":"2024-09-13T15:05:23.885667+0000","last_peered":"2024-09-13T15:05:23.885667+0000","last_clean":"2024-09-13T15:05:23.885667+0000","last_became_active":"2024-09-13T14:59:00.928166+0000","last_became_peered":"2024-09-13T14:59:00.928166+0000","last_unstale":"2024-09-13T15:05:23.885667+0000","last_undegraded":"2024-09-13T15:05:23.885667+0000","last_fullsized":"2024-09-13T15:05:23.885667+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:34.878172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":654,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.888862+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T15:05:23.888862+0000","last_peered":"2024-09-13T15:05:23.888862+0000","last_clean":"2024-09-13T15:05:23.888862+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T15:05:23.888862+0000","last_undegraded":"2024-09-13T15:05:23.888862+0000","last_fullsized":"2024-09-13T15:05:23.888862+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"380'12809","reported_seq":12842,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:26.694176+0000","last_change":"2024-09-13T15:02:39.921711+0000","last_active":"2024-09-13T15:05:26.694176+0000","last_peered":"2024-09-13T15:05:26.694176+0000","last_clean":"2024-09-13T15:05:26.694176+0000","last_became_active":"2024-09-13T15:02:39.921395+0000","last_became_peered":"2024-09-13T15:02:39.921395+0000","last_unstale":"2024-09-13T15:05:26.694176+0000","last_undegraded":"2024-09-13T15:05:26.694176+0000","last_fullsized":"2024-09-13T15:05:26.694176+0000","mapping_epoch":355,"log_start":"365'6100","ondisk_log_start":"365'6100","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6709,"log_dups_size":0,"ondisk_log_size":6709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:12.076600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259063808,"num_objects":3953,"num_object_clones":0,"num_object_copies":11859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12809,"num_write_kb":536384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":157,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.878387+0000","last_change":"2024-09-13T14:55:59.574827+0000","last_active":"2024-09-13T15:05:06.878387+0000","last_peered":"2024-09-13T15:05:06.878387+0000","last_clean":"2024-09-13T15:05:06.878387+0000","last_became_active":"2024-09-13T14:55:51.162903+0000","last_became_peered":"2024-09-13T14:55:51.162903+0000","last_unstale":"2024-09-13T15:05:06.878387+0000","last_undegraded":"2024-09-13T15:05:06.878387+0000","last_fullsized":"2024-09-13T15:05:06.878387+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.574773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.574773+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:39.074072+0000","scrub_duration":0.001553708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":580,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.885791+0000","last_change":"2024-09-13T14:59:00.950349+0000","last_active":"2024-09-13T15:05:23.885791+0000","last_peered":"2024-09-13T15:05:23.885791+0000","last_clean":"2024-09-13T15:05:23.885791+0000","last_became_active":"2024-09-13T14:59:00.925703+0000","last_became_peered":"2024-09-13T14:59:00.925703+0000","last_unstale":"2024-09-13T15:05:23.885791+0000","last_undegraded":"2024-09-13T15:05:23.885791+0000","last_fullsized":"2024-09-13T15:05:23.885791+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:43.851359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":632,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.897419+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T15:05:23.897419+0000","last_peered":"2024-09-13T15:05:23.897419+0000","last_clean":"2024-09-13T15:05:23.897419+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T15:05:23.897419+0000","last_undegraded":"2024-09-13T15:05:23.897419+0000","last_fullsized":"2024-09-13T15:05:23.897419+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":650,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.898288+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T15:05:23.898288+0000","last_peered":"2024-09-13T15:05:23.898288+0000","last_clean":"2024-09-13T15:05:23.898288+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T15:05:23.898288+0000","last_undegraded":"2024-09-13T15:05:23.898288+0000","last_fullsized":"2024-09-13T15:05:23.898288+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"380'13078","reported_seq":13111,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:26.834461+0000","last_change":"2024-09-13T15:02:39.882940+0000","last_active":"2024-09-13T15:05:26.834461+0000","last_peered":"2024-09-13T15:05:26.834461+0000","last_clean":"2024-09-13T15:05:26.834461+0000","last_became_active":"2024-09-13T15:02:39.882641+0000","last_became_peered":"2024-09-13T15:02:39.882641+0000","last_unstale":"2024-09-13T15:05:26.834461+0000","last_undegraded":"2024-09-13T15:05:26.834461+0000","last_fullsized":"2024-09-13T15:05:26.834461+0000","mapping_epoch":355,"log_start":"366'7700","ondisk_log_start":"366'7700","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5378,"log_dups_size":0,"ondisk_log_size":5378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:10.481971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258473984,"num_objects":3944,"num_object_clones":0,"num_object_copies":11832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13078,"num_write_kb":544704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":300,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.886908+0000","last_change":"2024-09-13T14:56:01.658240+0000","last_active":"2024-09-13T15:05:06.886908+0000","last_peered":"2024-09-13T15:05:06.886908+0000","last_clean":"2024-09-13T15:05:06.886908+0000","last_became_active":"2024-09-13T14:55:51.160480+0000","last_became_peered":"2024-09-13T14:55:51.160480+0000","last_unstale":"2024-09-13T15:05:06.886908+0000","last_undegraded":"2024-09-13T15:05:06.886908+0000","last_fullsized":"2024-09-13T15:05:06.886908+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.658143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.658143+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:19.262955+0000","scrub_duration":0.0038903079999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":1006,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.887230+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T15:05:06.887230+0000","last_peered":"2024-09-13T15:05:06.887230+0000","last_clean":"2024-09-13T15:05:06.887230+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T15:05:06.887230+0000","last_undegraded":"2024-09-13T15:05:06.887230+0000","last_fullsized":"2024-09-13T15:05:06.887230+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":365,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.882810+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T15:05:06.882810+0000","last_peered":"2024-09-13T15:05:06.882810+0000","last_clean":"2024-09-13T15:05:06.882810+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T15:05:06.882810+0000","last_undegraded":"2024-09-13T15:05:06.882810+0000","last_fullsized":"2024-09-13T15:05:06.882810+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":656,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.899615+0000","last_change":"2024-09-13T14:59:00.930713+0000","last_active":"2024-09-13T15:05:23.899615+0000","last_peered":"2024-09-13T15:05:23.899615+0000","last_clean":"2024-09-13T15:05:23.899615+0000","last_became_active":"2024-09-13T14:59:00.928260+0000","last_became_peered":"2024-09-13T14:59:00.928260+0000","last_unstale":"2024-09-13T15:05:23.899615+0000","last_undegraded":"2024-09-13T15:05:23.899615+0000","last_fullsized":"2024-09-13T15:05:23.899615+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:07.048853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"380'13002","reported_seq":13035,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:26.607770+0000","last_change":"2024-09-13T15:02:39.882990+0000","last_active":"2024-09-13T15:05:26.607770+0000","last_peered":"2024-09-13T15:05:26.607770+0000","last_clean":"2024-09-13T15:05:26.607770+0000","last_became_active":"2024-09-13T15:02:39.882663+0000","last_became_peered":"2024-09-13T15:02:39.882663+0000","last_unstale":"2024-09-13T15:05:26.607770+0000","last_undegraded":"2024-09-13T15:05:26.607770+0000","last_fullsized":"2024-09-13T15:05:26.607770+0000","mapping_epoch":355,"log_start":"366'7000","ondisk_log_start":"366'7000","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6002,"log_dups_size":0,"ondisk_log_size":6002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:20.368697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260308992,"num_objects":3972,"num_object_clones":0,"num_object_copies":11916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13002,"num_write_kb":543168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":243,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.884989+0000","last_change":"2024-09-13T15:02:49.771643+0000","last_active":"2024-09-13T15:05:06.884989+0000","last_peered":"2024-09-13T15:05:06.884989+0000","last_clean":"2024-09-13T15:05:06.884989+0000","last_became_active":"2024-09-13T14:59:53.249243+0000","last_became_peered":"2024-09-13T14:59:53.249243+0000","last_unstale":"2024-09-13T15:05:06.884989+0000","last_undegraded":"2024-09-13T15:05:06.884989+0000","last_fullsized":"2024-09-13T15:05:06.884989+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:49.771572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:49.771572+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:11:27.741449+0000","scrub_duration":0.0035237329999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":371,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.887668+0000","last_change":"2024-09-13T14:59:00.931326+0000","last_active":"2024-09-13T15:05:06.887668+0000","last_peered":"2024-09-13T15:05:06.887668+0000","last_clean":"2024-09-13T15:05:06.887668+0000","last_became_active":"2024-09-13T14:59:00.931011+0000","last_became_peered":"2024-09-13T14:59:00.931011+0000","last_unstale":"2024-09-13T15:05:06.887668+0000","last_undegraded":"2024-09-13T15:05:06.887668+0000","last_fullsized":"2024-09-13T15:05:06.887668+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:24:41.833263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":1134,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.899676+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T15:05:23.899676+0000","last_peered":"2024-09-13T15:05:23.899676+0000","last_clean":"2024-09-13T15:05:23.899676+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T15:05:23.899676+0000","last_undegraded":"2024-09-13T15:05:23.899676+0000","last_fullsized":"2024-09-13T15:05:23.899676+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":606,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.897861+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T15:05:23.897861+0000","last_peered":"2024-09-13T15:05:23.897861+0000","last_clean":"2024-09-13T15:05:23.897861+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T15:05:23.897861+0000","last_undegraded":"2024-09-13T15:05:23.897861+0000","last_fullsized":"2024-09-13T15:05:23.897861+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"380'13109","reported_seq":13143,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:26.811610+0000","last_change":"2024-09-13T15:02:39.881285+0000","last_active":"2024-09-13T15:05:26.811610+0000","last_peered":"2024-09-13T15:05:26.811610+0000","last_clean":"2024-09-13T15:05:26.811610+0000","last_became_active":"2024-09-13T15:02:39.880984+0000","last_became_peered":"2024-09-13T15:02:39.880984+0000","last_unstale":"2024-09-13T15:05:26.811610+0000","last_undegraded":"2024-09-13T15:05:26.811610+0000","last_fullsized":"2024-09-13T15:05:26.811610+0000","mapping_epoch":355,"log_start":"366'7700","ondisk_log_start":"366'7700","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5409,"log_dups_size":0,"ondisk_log_size":5409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:31.193802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264503320,"num_objects":4037,"num_object_clones":0,"num_object_copies":12111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4037,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13109,"num_write_kb":548609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":233,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.884670+0000","last_change":"2024-09-13T15:02:51.748425+0000","last_active":"2024-09-13T15:05:06.884670+0000","last_peered":"2024-09-13T15:05:06.884670+0000","last_clean":"2024-09-13T15:05:06.884670+0000","last_became_active":"2024-09-13T14:59:53.226288+0000","last_became_peered":"2024-09-13T14:59:53.226288+0000","last_unstale":"2024-09-13T15:05:06.884670+0000","last_undegraded":"2024-09-13T15:05:06.884670+0000","last_fullsized":"2024-09-13T15:05:06.884670+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:51.748121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:51.748121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:17:47.226084+0000","scrub_duration":0.0026525849999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":631,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.899678+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T15:05:23.899678+0000","last_peered":"2024-09-13T15:05:23.899678+0000","last_clean":"2024-09-13T15:05:23.899678+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T15:05:23.899678+0000","last_undegraded":"2024-09-13T15:05:23.899678+0000","last_fullsized":"2024-09-13T15:05:23.899678+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":593,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.879274+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T15:05:06.879274+0000","last_peered":"2024-09-13T15:05:06.879274+0000","last_clean":"2024-09-13T15:05:06.879274+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T15:05:06.879274+0000","last_undegraded":"2024-09-13T15:05:06.879274+0000","last_fullsized":"2024-09-13T15:05:06.879274+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":619,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.886195+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T15:05:23.886195+0000","last_peered":"2024-09-13T15:05:23.886195+0000","last_clean":"2024-09-13T15:05:23.886195+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T15:05:23.886195+0000","last_undegraded":"2024-09-13T15:05:23.886195+0000","last_fullsized":"2024-09-13T15:05:23.886195+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"380'13216","reported_seq":13249,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:26.720508+0000","last_change":"2024-09-13T15:02:39.921655+0000","last_active":"2024-09-13T15:05:26.720508+0000","last_peered":"2024-09-13T15:05:26.720508+0000","last_clean":"2024-09-13T15:05:26.720508+0000","last_became_active":"2024-09-13T15:02:39.921331+0000","last_became_peered":"2024-09-13T15:02:39.921331+0000","last_unstale":"2024-09-13T15:05:26.720508+0000","last_undegraded":"2024-09-13T15:05:26.720508+0000","last_fullsized":"2024-09-13T15:05:26.720508+0000","mapping_epoch":355,"log_start":"367'7800","ondisk_log_start":"367'7800","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5416,"log_dups_size":0,"ondisk_log_size":5416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:48.004179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261357568,"num_objects":3988,"num_object_clones":0,"num_object_copies":11964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13216,"num_write_kb":550528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":351,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.882180+0000","last_change":"2024-09-13T15:02:52.361444+0000","last_active":"2024-09-13T15:05:06.882180+0000","last_peered":"2024-09-13T15:05:06.882180+0000","last_clean":"2024-09-13T15:05:06.882180+0000","last_became_active":"2024-09-13T14:59:53.220135+0000","last_became_peered":"2024-09-13T14:59:53.220135+0000","last_unstale":"2024-09-13T15:05:06.882180+0000","last_undegraded":"2024-09-13T15:05:06.882180+0000","last_fullsized":"2024-09-13T15:05:06.882180+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.361359+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.361359+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:35.003140+0000","scrub_duration":0.005845154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":101,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.879309+0000","last_change":"2024-09-13T14:59:00.936064+0000","last_active":"2024-09-13T15:05:06.879309+0000","last_peered":"2024-09-13T15:05:06.879309+0000","last_clean":"2024-09-13T15:05:06.879309+0000","last_became_active":"2024-09-13T14:59:00.932311+0000","last_became_peered":"2024-09-13T14:59:00.932311+0000","last_unstale":"2024-09-13T15:05:06.879309+0000","last_undegraded":"2024-09-13T15:05:06.879309+0000","last_fullsized":"2024-09-13T15:05:06.879309+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:16.456004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":989,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.898099+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T15:05:23.898099+0000","last_peered":"2024-09-13T15:05:23.898099+0000","last_clean":"2024-09-13T15:05:23.898099+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T15:05:23.898099+0000","last_undegraded":"2024-09-13T15:05:23.898099+0000","last_fullsized":"2024-09-13T15:05:23.898099+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":102,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.878068+0000","last_change":"2024-09-13T14:59:00.924324+0000","last_active":"2024-09-13T15:05:06.878068+0000","last_peered":"2024-09-13T15:05:06.878068+0000","last_clean":"2024-09-13T15:05:06.878068+0000","last_became_active":"2024-09-13T14:59:00.923798+0000","last_became_peered":"2024-09-13T14:59:00.923798+0000","last_unstale":"2024-09-13T15:05:06.878068+0000","last_undegraded":"2024-09-13T15:05:06.878068+0000","last_fullsized":"2024-09-13T15:05:06.878068+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:10.994877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"380'13042","reported_seq":13075,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:26.798988+0000","last_change":"2024-09-13T15:02:39.922368+0000","last_active":"2024-09-13T15:05:26.798988+0000","last_peered":"2024-09-13T15:05:26.798988+0000","last_clean":"2024-09-13T15:05:26.798988+0000","last_became_active":"2024-09-13T15:02:39.922106+0000","last_became_peered":"2024-09-13T15:02:39.922106+0000","last_unstale":"2024-09-13T15:05:26.798988+0000","last_undegraded":"2024-09-13T15:05:26.798988+0000","last_fullsized":"2024-09-13T15:05:26.798988+0000","mapping_epoch":355,"log_start":"366'7600","ondisk_log_start":"366'7600","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":5442,"log_dups_size":0,"ondisk_log_size":5442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:13:52.416513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263061504,"num_objects":4014,"num_object_clones":0,"num_object_copies":12042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13042,"num_write_kb":545792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":88,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.887155+0000","last_change":"2024-09-13T15:02:54.193233+0000","last_active":"2024-09-13T15:05:06.887155+0000","last_peered":"2024-09-13T15:05:06.887155+0000","last_clean":"2024-09-13T15:05:06.887155+0000","last_became_active":"2024-09-13T14:59:00.925167+0000","last_became_peered":"2024-09-13T14:59:00.925167+0000","last_unstale":"2024-09-13T15:05:06.887155+0000","last_undegraded":"2024-09-13T15:05:06.887155+0000","last_fullsized":"2024-09-13T15:05:06.887155+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:54.193156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:54.193156+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:16.851666+0000","scrub_duration":0.0040069900000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":640,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.888663+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T15:05:23.888663+0000","last_peered":"2024-09-13T15:05:23.888663+0000","last_clean":"2024-09-13T15:05:23.888663+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T15:05:23.888663+0000","last_undegraded":"2024-09-13T15:05:23.888663+0000","last_fullsized":"2024-09-13T15:05:23.888663+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1193,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.885303+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T15:05:06.885303+0000","last_peered":"2024-09-13T15:05:06.885303+0000","last_clean":"2024-09-13T15:05:06.885303+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T15:05:06.885303+0000","last_undegraded":"2024-09-13T15:05:06.885303+0000","last_fullsized":"2024-09-13T15:05:06.885303+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":347,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.881945+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T15:05:06.881945+0000","last_peered":"2024-09-13T15:05:06.881945+0000","last_clean":"2024-09-13T15:05:06.881945+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T15:05:06.881945+0000","last_undegraded":"2024-09-13T15:05:06.881945+0000","last_fullsized":"2024-09-13T15:05:06.881945+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"82'100","reported_seq":93,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.897972+0000","last_change":"2024-09-13T15:02:51.192587+0000","last_active":"2024-09-13T15:05:23.897972+0000","last_peered":"2024-09-13T15:05:23.897972+0000","last_clean":"2024-09-13T15:05:23.897972+0000","last_became_active":"2024-09-13T15:02:23.878033+0000","last_became_peered":"2024-09-13T15:02:23.878033+0000","last_unstale":"2024-09-13T15:05:23.897972+0000","last_undegraded":"2024-09-13T15:05:23.897972+0000","last_fullsized":"2024-09-13T15:05:23.897972+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:51.192503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:51.192503+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:38.276910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":364,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.882628+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T15:05:06.882628+0000","last_peered":"2024-09-13T15:05:06.882628+0000","last_clean":"2024-09-13T15:05:06.882628+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T15:05:06.882628+0000","last_undegraded":"2024-09-13T15:05:06.882628+0000","last_fullsized":"2024-09-13T15:05:06.882628+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":647,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.887654+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T15:05:23.887654+0000","last_peered":"2024-09-13T15:05:23.887654+0000","last_clean":"2024-09-13T15:05:23.887654+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T15:05:23.887654+0000","last_undegraded":"2024-09-13T15:05:23.887654+0000","last_fullsized":"2024-09-13T15:05:23.887654+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.17","version":"82'100","reported_seq":192,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.888042+0000","last_change":"2024-09-13T15:03:03.108031+0000","last_active":"2024-09-13T15:05:06.888042+0000","last_peered":"2024-09-13T15:05:06.888042+0000","last_clean":"2024-09-13T15:05:06.888042+0000","last_became_active":"2024-09-13T15:02:23.883373+0000","last_became_peered":"2024-09-13T15:02:23.883373+0000","last_unstale":"2024-09-13T15:05:06.888042+0000","last_undegraded":"2024-09-13T15:05:06.888042+0000","last_fullsized":"2024-09-13T15:05:06.888042+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:03.107786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:03.107786+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:17:10.041468+0000","scrub_duration":0.0049024139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":630,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.897721+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T15:05:23.897721+0000","last_peered":"2024-09-13T15:05:23.897721+0000","last_clean":"2024-09-13T15:05:23.897721+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T15:05:23.897721+0000","last_undegraded":"2024-09-13T15:05:23.897721+0000","last_fullsized":"2024-09-13T15:05:23.897721+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":370,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.879384+0000","last_change":"2024-09-13T14:59:00.929406+0000","last_active":"2024-09-13T15:05:06.879384+0000","last_peered":"2024-09-13T15:05:06.879384+0000","last_clean":"2024-09-13T15:05:06.879384+0000","last_became_active":"2024-09-13T14:59:00.928524+0000","last_became_peered":"2024-09-13T14:59:00.928524+0000","last_unstale":"2024-09-13T15:05:06.879384+0000","last_undegraded":"2024-09-13T15:05:06.879384+0000","last_fullsized":"2024-09-13T15:05:06.879384+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:10.609448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":212,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.882779+0000","last_change":"2024-09-13T14:56:01.235655+0000","last_active":"2024-09-13T15:05:23.882779+0000","last_peered":"2024-09-13T15:05:23.882779+0000","last_clean":"2024-09-13T15:05:23.882779+0000","last_became_active":"2024-09-13T14:55:55.195703+0000","last_became_peered":"2024-09-13T14:55:55.195703+0000","last_unstale":"2024-09-13T15:05:23.882779+0000","last_undegraded":"2024-09-13T15:05:23.882779+0000","last_fullsized":"2024-09-13T15:05:23.882779+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.235497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.235497+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:36:33.861613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":344,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.887391+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T15:05:06.887391+0000","last_peered":"2024-09-13T15:05:06.887391+0000","last_clean":"2024-09-13T15:05:06.887391+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T15:05:06.887391+0000","last_undegraded":"2024-09-13T15:05:06.887391+0000","last_fullsized":"2024-09-13T15:05:06.887391+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":351,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.878304+0000","last_change":"2024-09-13T14:59:00.926586+0000","last_active":"2024-09-13T15:05:06.878304+0000","last_peered":"2024-09-13T15:05:06.878304+0000","last_clean":"2024-09-13T15:05:06.878304+0000","last_became_active":"2024-09-13T14:59:00.926206+0000","last_became_peered":"2024-09-13T14:59:00.926206+0000","last_unstale":"2024-09-13T15:05:06.878304+0000","last_undegraded":"2024-09-13T15:05:06.878304+0000","last_fullsized":"2024-09-13T15:05:06.878304+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:17:18.805856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.15","version":"82'100","reported_seq":48,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.879200+0000","last_change":"2024-09-13T15:02:55.870615+0000","last_active":"2024-09-13T15:05:06.879200+0000","last_peered":"2024-09-13T15:05:06.879200+0000","last_clean":"2024-09-13T15:05:06.879200+0000","last_became_active":"2024-09-13T15:02:23.883217+0000","last_became_peered":"2024-09-13T15:02:23.883217+0000","last_unstale":"2024-09-13T15:05:06.879200+0000","last_undegraded":"2024-09-13T15:05:06.879200+0000","last_fullsized":"2024-09-13T15:05:06.879200+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.870528+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.870528+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:10.023787+0000","scrub_duration":0.0047551850000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":634,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.886035+0000","last_change":"2024-09-13T14:59:00.932503+0000","last_active":"2024-09-13T15:05:23.886035+0000","last_peered":"2024-09-13T15:05:23.886035+0000","last_clean":"2024-09-13T15:05:23.886035+0000","last_became_active":"2024-09-13T14:59:00.931358+0000","last_became_peered":"2024-09-13T14:59:00.931358+0000","last_unstale":"2024-09-13T15:05:23.886035+0000","last_undegraded":"2024-09-13T15:05:23.886035+0000","last_fullsized":"2024-09-13T15:05:23.886035+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:19.226182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":466,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.897265+0000","last_change":"2024-09-13T14:59:00.927060+0000","last_active":"2024-09-13T15:05:23.897265+0000","last_peered":"2024-09-13T15:05:23.897265+0000","last_clean":"2024-09-13T15:05:23.897265+0000","last_became_active":"2024-09-13T14:59:00.925402+0000","last_became_peered":"2024-09-13T14:59:00.925402+0000","last_unstale":"2024-09-13T15:05:23.897265+0000","last_undegraded":"2024-09-13T15:05:23.897265+0000","last_fullsized":"2024-09-13T15:05:23.897265+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:31:08.336446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"82'100","reported_seq":48,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.883830+0000","last_change":"2024-09-13T15:02:56.319389+0000","last_active":"2024-09-13T15:05:06.883830+0000","last_peered":"2024-09-13T15:05:06.883830+0000","last_clean":"2024-09-13T15:05:06.883830+0000","last_became_active":"2024-09-13T15:02:23.876739+0000","last_became_peered":"2024-09-13T15:02:23.876739+0000","last_unstale":"2024-09-13T15:05:06.883830+0000","last_undegraded":"2024-09-13T15:05:06.883830+0000","last_fullsized":"2024-09-13T15:05:06.883830+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:56.319302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:56.319302+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:58.230158+0000","scrub_duration":0.0031182190000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":164,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.885141+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T15:05:06.885141+0000","last_peered":"2024-09-13T15:05:06.885141+0000","last_clean":"2024-09-13T15:05:06.885141+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T15:05:06.885141+0000","last_undegraded":"2024-09-13T15:05:06.885141+0000","last_fullsized":"2024-09-13T15:05:06.885141+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":224,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.886927+0000","last_change":"2024-09-13T14:59:00.924196+0000","last_active":"2024-09-13T15:05:23.886927+0000","last_peered":"2024-09-13T15:05:23.886927+0000","last_clean":"2024-09-13T15:05:23.886927+0000","last_became_active":"2024-09-13T14:59:00.923901+0000","last_became_peered":"2024-09-13T14:59:00.923901+0000","last_unstale":"2024-09-13T15:05:23.886927+0000","last_undegraded":"2024-09-13T15:05:23.886927+0000","last_fullsized":"2024-09-13T15:05:23.886927+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:01.913804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"82'100","reported_seq":630,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.887039+0000","last_change":"2024-09-13T15:03:07.166808+0000","last_active":"2024-09-13T15:05:06.887039+0000","last_peered":"2024-09-13T15:05:06.887039+0000","last_clean":"2024-09-13T15:05:06.887039+0000","last_became_active":"2024-09-13T15:02:23.883387+0000","last_became_peered":"2024-09-13T15:02:23.883387+0000","last_unstale":"2024-09-13T15:05:06.887039+0000","last_undegraded":"2024-09-13T15:05:06.887039+0000","last_fullsized":"2024-09-13T15:05:06.887039+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:07.166034+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:07.166034+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:32:42.605595+0000","scrub_duration":0.01094734,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":658,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.888813+0000","last_change":"2024-09-13T14:59:00.924814+0000","last_active":"2024-09-13T15:05:23.888813+0000","last_peered":"2024-09-13T15:05:23.888813+0000","last_clean":"2024-09-13T15:05:23.888813+0000","last_became_active":"2024-09-13T14:59:00.924501+0000","last_became_peered":"2024-09-13T14:59:00.924501+0000","last_unstale":"2024-09-13T15:05:23.888813+0000","last_undegraded":"2024-09-13T15:05:23.888813+0000","last_fullsized":"2024-09-13T15:05:23.888813+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:11:15.382207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":622,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.897359+0000","last_change":"2024-09-13T14:59:00.925431+0000","last_active":"2024-09-13T15:05:23.897359+0000","last_peered":"2024-09-13T15:05:23.897359+0000","last_clean":"2024-09-13T15:05:23.897359+0000","last_became_active":"2024-09-13T14:59:00.925023+0000","last_became_peered":"2024-09-13T14:59:00.925023+0000","last_unstale":"2024-09-13T15:05:23.897359+0000","last_undegraded":"2024-09-13T15:05:23.897359+0000","last_fullsized":"2024-09-13T15:05:23.897359+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:38:18.686253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"82'100","reported_seq":335,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.886732+0000","last_change":"2024-09-13T15:03:04.101898+0000","last_active":"2024-09-13T15:05:06.886732+0000","last_peered":"2024-09-13T15:05:06.886732+0000","last_clean":"2024-09-13T15:05:06.886732+0000","last_became_active":"2024-09-13T15:02:23.884406+0000","last_became_peered":"2024-09-13T15:02:23.884406+0000","last_unstale":"2024-09-13T15:05:06.886732+0000","last_undegraded":"2024-09-13T15:05:06.886732+0000","last_fullsized":"2024-09-13T15:05:06.886732+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:04.101802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:04.101802+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:34.459623+0000","scrub_duration":0.003875819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":154,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.882314+0000","last_change":"2024-09-13T14:59:00.927250+0000","last_active":"2024-09-13T15:05:06.882314+0000","last_peered":"2024-09-13T15:05:06.882314+0000","last_clean":"2024-09-13T15:05:06.882314+0000","last_became_active":"2024-09-13T14:59:00.926953+0000","last_became_peered":"2024-09-13T14:59:00.926953+0000","last_unstale":"2024-09-13T15:05:06.882314+0000","last_undegraded":"2024-09-13T15:05:06.882314+0000","last_fullsized":"2024-09-13T15:05:06.882314+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:58.306547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":363,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.884232+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T15:05:06.884232+0000","last_peered":"2024-09-13T15:05:06.884232+0000","last_clean":"2024-09-13T15:05:06.884232+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T15:05:06.884232+0000","last_undegraded":"2024-09-13T15:05:06.884232+0000","last_fullsized":"2024-09-13T15:05:06.884232+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"82'100","reported_seq":93,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.885439+0000","last_change":"2024-09-13T15:02:52.429623+0000","last_active":"2024-09-13T15:05:23.885439+0000","last_peered":"2024-09-13T15:05:23.885439+0000","last_clean":"2024-09-13T15:05:23.885439+0000","last_became_active":"2024-09-13T15:02:23.880201+0000","last_became_peered":"2024-09-13T15:02:23.880201+0000","last_unstale":"2024-09-13T15:05:23.885439+0000","last_undegraded":"2024-09-13T15:05:23.885439+0000","last_fullsized":"2024-09-13T15:05:23.885439+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.429543+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.429543+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:32.778195+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":456,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.899818+0000","last_change":"2024-09-13T14:59:00.934412+0000","last_active":"2024-09-13T15:05:23.899818+0000","last_peered":"2024-09-13T15:05:23.899818+0000","last_clean":"2024-09-13T15:05:23.899818+0000","last_became_active":"2024-09-13T14:59:00.932986+0000","last_became_peered":"2024-09-13T14:59:00.932986+0000","last_unstale":"2024-09-13T15:05:23.899818+0000","last_undegraded":"2024-09-13T15:05:23.899818+0000","last_fullsized":"2024-09-13T15:05:23.899818+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:21:44.285324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":619,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.886102+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T15:05:23.886102+0000","last_peered":"2024-09-13T15:05:23.886102+0000","last_clean":"2024-09-13T15:05:23.886102+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T15:05:23.886102+0000","last_undegraded":"2024-09-13T15:05:23.886102+0000","last_fullsized":"2024-09-13T15:05:23.886102+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"82'100","reported_seq":48,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.884338+0000","last_change":"2024-09-13T15:02:55.679993+0000","last_active":"2024-09-13T15:05:06.884338+0000","last_peered":"2024-09-13T15:05:06.884338+0000","last_clean":"2024-09-13T15:05:06.884338+0000","last_became_active":"2024-09-13T15:02:23.878770+0000","last_became_peered":"2024-09-13T15:02:23.878770+0000","last_unstale":"2024-09-13T15:05:06.884338+0000","last_undegraded":"2024-09-13T15:05:06.884338+0000","last_fullsized":"2024-09-13T15:05:06.884338+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.679893+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.679893+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:25.717099+0000","scrub_duration":0.0050514899999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":371,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.882560+0000","last_change":"2024-09-13T14:59:00.929751+0000","last_active":"2024-09-13T15:05:06.882560+0000","last_peered":"2024-09-13T15:05:06.882560+0000","last_clean":"2024-09-13T15:05:06.882560+0000","last_became_active":"2024-09-13T14:59:00.929456+0000","last_became_peered":"2024-09-13T14:59:00.929456+0000","last_unstale":"2024-09-13T15:05:06.882560+0000","last_undegraded":"2024-09-13T15:05:06.882560+0000","last_fullsized":"2024-09-13T15:05:06.882560+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:11.462853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":346,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.887234+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T15:05:06.887234+0000","last_peered":"2024-09-13T15:05:06.887234+0000","last_clean":"2024-09-13T15:05:06.887234+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T15:05:06.887234+0000","last_undegraded":"2024-09-13T15:05:06.887234+0000","last_fullsized":"2024-09-13T15:05:06.887234+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":371,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.884111+0000","last_change":"2024-09-13T14:59:00.928898+0000","last_active":"2024-09-13T15:05:06.884111+0000","last_peered":"2024-09-13T15:05:06.884111+0000","last_clean":"2024-09-13T15:05:06.884111+0000","last_became_active":"2024-09-13T14:59:00.928585+0000","last_became_peered":"2024-09-13T14:59:00.928585+0000","last_unstale":"2024-09-13T15:05:06.884111+0000","last_undegraded":"2024-09-13T15:05:06.884111+0000","last_fullsized":"2024-09-13T15:05:06.884111+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:26:57.155149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":385,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.887596+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T15:05:06.887596+0000","last_peered":"2024-09-13T15:05:06.887596+0000","last_clean":"2024-09-13T15:05:06.887596+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T15:05:06.887596+0000","last_undegraded":"2024-09-13T15:05:06.887596+0000","last_fullsized":"2024-09-13T15:05:06.887596+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"82'100","reported_seq":38,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.878440+0000","last_change":"2024-09-13T15:02:23.879862+0000","last_active":"2024-09-13T15:05:06.878440+0000","last_peered":"2024-09-13T15:05:06.878440+0000","last_clean":"2024-09-13T15:05:06.878440+0000","last_became_active":"2024-09-13T15:02:23.879170+0000","last_became_peered":"2024-09-13T15:02:23.879170+0000","last_unstale":"2024-09-13T15:05:06.878440+0000","last_undegraded":"2024-09-13T15:05:06.878440+0000","last_fullsized":"2024-09-13T15:05:06.878440+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:06.945525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:06.945525+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:48.496118+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":190,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.886340+0000","last_change":"2024-09-13T14:59:00.929835+0000","last_active":"2024-09-13T15:05:23.886340+0000","last_peered":"2024-09-13T15:05:23.886340+0000","last_clean":"2024-09-13T15:05:23.886340+0000","last_became_active":"2024-09-13T14:59:00.929514+0000","last_became_peered":"2024-09-13T14:59:00.929514+0000","last_unstale":"2024-09-13T15:05:23.886340+0000","last_undegraded":"2024-09-13T15:05:23.886340+0000","last_fullsized":"2024-09-13T15:05:23.886340+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:48.981240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":407,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.884548+0000","last_change":"2024-09-13T14:59:00.925895+0000","last_active":"2024-09-13T15:05:06.884548+0000","last_peered":"2024-09-13T15:05:06.884548+0000","last_clean":"2024-09-13T15:05:06.884548+0000","last_became_active":"2024-09-13T14:59:00.925564+0000","last_became_peered":"2024-09-13T14:59:00.925564+0000","last_unstale":"2024-09-13T15:05:06.884548+0000","last_undegraded":"2024-09-13T15:05:06.884548+0000","last_fullsized":"2024-09-13T15:05:06.884548+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:23.283927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1d","version":"82'100","reported_seq":95,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.898453+0000","last_change":"2024-09-13T15:02:52.149390+0000","last_active":"2024-09-13T15:05:23.898453+0000","last_peered":"2024-09-13T15:05:23.898453+0000","last_clean":"2024-09-13T15:05:23.898453+0000","last_became_active":"2024-09-13T15:02:23.914809+0000","last_became_peered":"2024-09-13T15:02:23.914809+0000","last_unstale":"2024-09-13T15:05:23.898453+0000","last_undegraded":"2024-09-13T15:05:23.898453+0000","last_fullsized":"2024-09-13T15:05:23.898453+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.149290+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.149290+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:50:54.160344+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":367,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T15:05:06.882779+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T15:05:06.882779+0000","last_peered":"2024-09-13T15:05:06.882779+0000","last_clean":"2024-09-13T15:05:06.882779+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T15:05:06.882779+0000","last_undegraded":"2024-09-13T15:05:06.882779+0000","last_fullsized":"2024-09-13T15:05:06.882779+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":672,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.885944+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T15:05:23.885944+0000","last_peered":"2024-09-13T15:05:23.885944+0000","last_clean":"2024-09-13T15:05:23.885944+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T15:05:23.885944+0000","last_undegraded":"2024-09-13T15:05:23.885944+0000","last_fullsized":"2024-09-13T15:05:23.885944+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4205576216,"num_objects":64173,"num_object_clones":0,"num_object_copies":192519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64173,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":207207,"num_write_kb":8684097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12731625472,"data_stored":12731613256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92307,"ondisk_log_size":92307,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":16172,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20401,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397978,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2180776,"kb_used_data":1287200,"kb_used_omap":1,"kb_used_meta":893566,"kb_avail":91564376,"statfs":{"total":95995035648,"available":93761921024,"internally_reserved":0,"allocated":1318092800,"data_stored":1315446708,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915011928},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594207,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2114720,"kb_used_data":1804888,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91630432,"statfs":{"total":95995035648,"available":93829562368,"internally_reserved":0,"allocated":1848205312,"data_stored":1845566464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790436,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2467800,"kb_used_data":1290576,"kb_used_omap":1,"kb_used_meta":1177086,"kb_avail":91277352,"statfs":{"total":95995035648,"available":93468008448,"internally_reserved":0,"allocated":1321549824,"data_stored":1318915585,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1205336408},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986665,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3473372,"kb_used_data":2571796,"kb_used_omap":1,"kb_used_meta":901374,"kb_avail":90271780,"statfs":{"total":95995035648,"available":92438302720,"internally_reserved":0,"allocated":2633519104,"data_stored":2630879980,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923007320},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":0,"seq":1438814044212,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762639,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2912476,"kb_used_data":1872912,"kb_used_omap":8,"kb_used_meta":1039543,"kb_avail":90832676,"statfs":{"total":95995035648,"available":93012660224,"internally_reserved":0,"allocated":1917861888,"data_stored":1915227644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":1064492490},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926165,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2054976,"kb_used_data":1076280,"kb_used_omap":7,"kb_used_meta":978680,"kb_avail":91690176,"statfs":{"total":95995035648,"available":93890740224,"internally_reserved":0,"allocated":1102110720,"data_stored":1099491550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":1002169033},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122405,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3604076,"kb_used_data":2660768,"kb_used_omap":7,"kb_used_meta":943224,"kb_avail":90141076,"statfs":{"total":95995035648,"available":92304461824,"internally_reserved":0,"allocated":2724626432,"data_stored":2721980328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":965861967},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3169,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2693140480,"data_stored":2693136408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1083703296,"data_stored":1083703296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1906573312,"data_stored":1906573312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2612465664,"data_stored":2612461592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1310982144,"data_stored":1310982144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1827737600,"data_stored":1827733528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1297022976,"data_stored":1297022976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:05:29.722 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T15:05:29.722 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:05:29.722 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-09-13T15:05:29.722 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T15:05:29.722 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T15:05:29.723 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-09-13T15:05:29.751 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:29.778 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:29.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:29.881 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:29.929 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:29.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:30.032 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:29 smithi096 ceph-mon[98901]: pgmap v1287: 119 pgs: 119 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T15:05:30.032 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:29 smithi096 ceph-mon[98901]: from='client.36723 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:05:30.032 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:29 smithi096 ceph-mon[102928]: pgmap v1287: 119 pgs: 119 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T15:05:30.032 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:29 smithi096 ceph-mon[102928]: from='client.36723 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:05:30.036 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:30.093 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:30.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:30.187 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:05:30.195 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:30.215 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:29 smithi204 ceph-mon[87071]: pgmap v1287: 119 pgs: 119 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T15:05:30.215 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:29 smithi204 ceph-mon[87071]: from='client.36723 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:05:30.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.264 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:30.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:30.365 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:30.411 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:30.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:05:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:05:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:05:30.513 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:30.544 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:30.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:30.646 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:30.686 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:30.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:30.788 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:30.820 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:30.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:30.921 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:30.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.956 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:30.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:31.057 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:31.099 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:31.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:31.201 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:31.239 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:31.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:31.341 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:31.379 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:31.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:31.481 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:31.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.521 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:31.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:31.603 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T15:05:31.604 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T15:05:31.604 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T15:05:31.604 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T15:05:31.604 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T15:05:31.622 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:31.656 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:31.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:31.758 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:31.786 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:31.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:31.888 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:31.937 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:31.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:32.038 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:32.081 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:32.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:32.112 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:31 smithi204 ceph-mon[87071]: pgmap v1288: 119 pgs: 119 active+clean; 3.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T15:05:32.183 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:32.210 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:32.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:32.235 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:31 smithi096 ceph-mon[98901]: pgmap v1288: 119 pgs: 119 active+clean; 3.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T15:05:32.235 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:31 smithi096 ceph-mon[102928]: pgmap v1288: 119 pgs: 119 active+clean; 3.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T15:05:32.274 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:05:32.275 INFO:teuthology.orchestra.run.smithi204.stdout: "success": "" 2024-09-13T15:05:32.275 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:05:32.312 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:32.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.338 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:32.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:32.396 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:32.430 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T15:05:32.440 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:32.503 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:05:32.510 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:32.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:32.612 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:32.647 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:32.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:32.749 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:32.784 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:32.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:32.886 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:32.908 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:32.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:33.010 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:33.032 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:33.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:33.134 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:33.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.163 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:33.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:33.264 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:33.295 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:33.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:33.397 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:33.432 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:33.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:33.534 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:33.560 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:33.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:33.662 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:33.693 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:33.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:33.795 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:33.835 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:33.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:33.937 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:33.967 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:33.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:34.069 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:34.099 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:34.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:34.170 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:33 smithi096 ceph-mon[98901]: pgmap v1289: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T15:05:34.170 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:33 smithi096 ceph-mon[102928]: pgmap v1289: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T15:05:34.201 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:34.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:33 smithi204 ceph-mon[87071]: pgmap v1289: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T15:05:34.241 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:34.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:34.343 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:34.371 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:34.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:34.473 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:34.502 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:34.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:34.604 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:34.641 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:34.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:34.654 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:34.743 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:34.772 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:34.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:34.874 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:34.911 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:34.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:35.013 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:35.045 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:35.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:35.147 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:35.184 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:35.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:35.286 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:35.317 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:35.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:35.419 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:35.456 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:35.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:35.558 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:35.590 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:35.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:35.692 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:35.723 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:35.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:35.825 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:35.856 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:35.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:35.960 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:35.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.995 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:35.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:36.096 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:36.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.129 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:36.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:35 smithi204 ceph-mon[87071]: pgmap v1290: 119 pgs: 119 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T15:05:36.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:35 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:36.229 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:36.262 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:36.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:35 smithi096 ceph-mon[98901]: pgmap v1290: 119 pgs: 119 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T15:05:36.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:35 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:36.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:35 smithi096 ceph-mon[102928]: pgmap v1290: 119 pgs: 119 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T15:05:36.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:35 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:36.364 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:36.403 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:36.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:36.505 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:36.535 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:36.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:36.637 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:36.671 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:36.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:36.773 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:36.815 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:36.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:36.914 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T15:05:36.917 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:36.956 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:36.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:37.057 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:37.095 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:37.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:37.112 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:36 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1188108274' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:05:37.113 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:36 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1188108274' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:05:37.197 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:37.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:36 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1188108274' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:05:37.234 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:37.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:37.336 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:37.369 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:37.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:37.471 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:37.512 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:37.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:37.614 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:37.659 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:37.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:37.761 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:37.798 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:37.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:37.866 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:05:37.866 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:05:37.896 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:05:37.897 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:05:37.900 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:37.924 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:05:37.924 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:05:37.944 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:37.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:37.953 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:05:37.953 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:05:37.968 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:37 smithi096 ceph-mon[98901]: pgmap v1291: 119 pgs: 119 active+clean; 3.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T15:05:37.968 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:37 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:37.968 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:37 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1188108274' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:05:37.969 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:37 smithi096 ceph-mon[98901]: osdmap e381: 8 total, 8 up, 7 in 2024-09-13T15:05:37.969 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:37 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/32697970' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:37.969 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:37 smithi096 ceph-mon[102928]: pgmap v1291: 119 pgs: 119 active+clean; 3.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T15:05:37.969 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:37 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:37.969 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:37 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1188108274' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:05:37.969 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:37 smithi096 ceph-mon[102928]: osdmap e381: 8 total, 8 up, 7 in 2024-09-13T15:05:37.969 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:37 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/32697970' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:37.984 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:05:37.985 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:05:38.017 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:05:38.017 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:05:38.046 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:38.047 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:05:38.047 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:05:38.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.084 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:38.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:38.088 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:05:38.088 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:05:38.185 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:37 smithi204 ceph-mon[87071]: pgmap v1291: 119 pgs: 119 active+clean; 3.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T15:05:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:37 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:37 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1188108274' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:05:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:37 smithi204 ceph-mon[87071]: osdmap e381: 8 total, 8 up, 7 in 2024-09-13T15:05:38.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:37 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/32697970' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:38.223 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:38.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:38.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:05:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:05:37] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-09-13T15:05:38.325 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:38.355 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:38.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:38.457 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:38.558 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:38.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:38.660 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:38.689 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:38.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:38.737 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:38.790 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:38.819 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:38.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:38.921 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:38.923 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 6] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:05:38.924 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:05:38.924 INFO:tasks.thrashosds.thrasher:Adding osd 3 2024-09-13T15:05:38.924 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd in 3 2024-09-13T15:05:38.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.955 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:38.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:39.056 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:39.094 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:39.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:39.195 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:39.253 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:39.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:39.354 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:39.395 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:39.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:39.497 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:39.522 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:39.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:39.624 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:39.656 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:39.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:39.757 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:39.785 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:39.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:39.887 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:39.913 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:39.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:40.015 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:40.054 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:40.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:40.156 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:40.184 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:40.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:40.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:39 smithi204 ceph-mon[87071]: pgmap v1293: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-13T15:05:40.286 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:40.315 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:40.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:40.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:39 smithi096 ceph-mon[98901]: pgmap v1293: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-13T15:05:40.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:39 smithi096 ceph-mon[102928]: pgmap v1293: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-13T15:05:40.417 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:40.453 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:40.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:40.522 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:05:40.555 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:40.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.603 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:40.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:40.704 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:40.719 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:05:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:05:40.229Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:05:40.737 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:40.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:40.839 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:40.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.872 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:40.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:40.973 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:41.001 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:41.008 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:41.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:41.110 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:41.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.146 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:41.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:41.170 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:05:41.248 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:41.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.285 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:41.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:41.386 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:41.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.422 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:41.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:41.523 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:41.553 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:41.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:41.655 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:41.682 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:41.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:41.784 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:41.825 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:41.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:41.926 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:41.968 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:41.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:42.070 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:42.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:41 smithi096 ceph-mon[98901]: pgmap v1294: 119 pgs: 119 active+clean; 3.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-13T15:05:42.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:41 smithi096 ceph-mon[102928]: pgmap v1294: 119 pgs: 119 active+clean; 3.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-13T15:05:42.110 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:42.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:42.190 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:41 smithi204 ceph-mon[87071]: pgmap v1294: 119 pgs: 119 active+clean; 3.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-13T15:05:42.212 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:42.243 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:42.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:42.345 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:42.352 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:05:42.352 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:05:42.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.387 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:42.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:42.488 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:42.527 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:42.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:42.629 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:42.663 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:42.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:42.765 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:42.794 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:42.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:42.896 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:42.933 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:42.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:43.035 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:43.064 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:43.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:43.166 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:43.204 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:43.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:43.306 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:43.342 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:43.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:43.444 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:43.476 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:43.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:43.578 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:43.615 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:43.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:43.717 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:43.753 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:43.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:43.855 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:43.890 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:43.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:43.991 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:44.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.027 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:44.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:44.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:43 smithi096 ceph-mon[98901]: pgmap v1295: 119 pgs: 119 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T15:05:44.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:43 smithi096 ceph-mon[102928]: pgmap v1295: 119 pgs: 119 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T15:05:44.128 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:44.159 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:44.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:44.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:43 smithi204 ceph-mon[87071]: pgmap v1295: 119 pgs: 119 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T15:05:44.261 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:44.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.288 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:44.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:44.389 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:44.410 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T15:05:44.423 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:44.525 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:44.563 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:44.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:44.665 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:44.701 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:44.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:44.802 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:44.840 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:44.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:44.942 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:44.980 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:44.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:45.032 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:45.035 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:45.082 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:45.113 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:45.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:45.215 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:45.252 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:45.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:45.351 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:45.353 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:45.386 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:45.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:45.488 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:45.522 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:45.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:45.624 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:45.664 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:45.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:45.766 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:45.795 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:45.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:45.897 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:45.932 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:45.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:45.938 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T15:05:46.034 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:46.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.069 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:46.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:46.170 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:46.198 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:46.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:46.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:45 smithi204 ceph-mon[87071]: pgmap v1296: 119 pgs: 119 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T15:05:46.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:45 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2560592759' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:46.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:45 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:46.300 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:46.319 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:45 smithi096 ceph-mon[98901]: pgmap v1296: 119 pgs: 119 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T15:05:46.319 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:45 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2560592759' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:46.319 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:45 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:46.319 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:45 smithi096 ceph-mon[102928]: pgmap v1296: 119 pgs: 119 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T15:05:46.319 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:45 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2560592759' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:46.319 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:45 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:46.341 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:46.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:46.443 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:46.474 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:46.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:46.576 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:46.612 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:46.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:46.714 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:46.747 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:46.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:46.849 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:46.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.889 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:46.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:46.990 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:47.017 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:47.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:47.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:46 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:05:47.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:46 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T15:05:47.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:46 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:47.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:46 smithi096 ceph-mon[98901]: osdmap e382: 8 total, 8 up, 7 in 2024-09-13T15:05:47.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:46 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:05:47.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:46 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T15:05:47.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:46 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:47.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:46 smithi096 ceph-mon[102928]: osdmap e382: 8 total, 8 up, 7 in 2024-09-13T15:05:47.119 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:47.146 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:47.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:47.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:46 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:05:47.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:46 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T15:05:47.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:46 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:47.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:46 smithi204 ceph-mon[87071]: osdmap e382: 8 total, 8 up, 7 in 2024-09-13T15:05:47.249 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:47.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.284 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:47.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:47.385 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:47.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.425 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:47.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:47.526 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:47.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.565 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:47.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:47.666 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:47.694 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:47.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:47.796 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:47.826 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:47.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:47.928 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:47.957 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:47.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:48.059 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:48.092 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:48.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:48.194 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:48.221 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:47 smithi204 ceph-mon[87071]: pgmap v1298: 119 pgs: 119 active+clean; 2.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 803 op/s 2024-09-13T15:05:48.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.230 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:48.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:48.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:47 smithi096 ceph-mon[98901]: pgmap v1298: 119 pgs: 119 active+clean; 2.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 803 op/s 2024-09-13T15:05:48.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:47 smithi096 ceph-mon[102928]: pgmap v1298: 119 pgs: 119 active+clean; 2.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 803 op/s 2024-09-13T15:05:48.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:05:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:05:47] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-09-13T15:05:48.331 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:48.365 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:48.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:48.467 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:48.506 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:48.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:48.607 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:48.642 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:48.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:48.744 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:48.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.779 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:48.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:48.880 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:48.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.914 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:48.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:49.015 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:49.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.056 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:49.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:49.157 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:49.187 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:49.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:49.247 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:05:49.290 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:49.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.342 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:49.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:49.444 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:49.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.477 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:49.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:49.578 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:49.616 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:49.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:49.718 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:49.756 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:49.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:49.858 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:49.893 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:49.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:49.975 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:05:49.976 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:05:49.976 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:05:49.976 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:05:49.976 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:05:49.976 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:05:49.976 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:05:49.976 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:05:49.985 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:05:49.985 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:05:49.995 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:50.027 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:05:50.027 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:05:50.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.040 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:50.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:50.075 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:05:50.075 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:05:50.107 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:05:50.107 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:05:50.141 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:50.190 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:50.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:50.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:49 smithi204 ceph-mon[87071]: pgmap v1299: 119 pgs: 119 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-13T15:05:50.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:49 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1267062933' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:50.284 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:49 smithi096 ceph-mon[98901]: pgmap v1299: 119 pgs: 119 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-13T15:05:50.284 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:49 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1267062933' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:50.285 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:49 smithi096 ceph-mon[102928]: pgmap v1299: 119 pgs: 119 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-13T15:05:50.285 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:49 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1267062933' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:50.292 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:50.326 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:50.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:50.428 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:50.461 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:50.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:05:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:05:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:05:50.562 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:50.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:50 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a[98876]: 2024-09-13T15:05:50.282+0000 7f16aeda4640 -1 mon.a@0(leader).osd e382 definitely_dead 0 2024-09-13T15:05:50.594 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:50.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:50.696 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:50.727 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:50.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:50.829 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:50.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.860 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:50.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:50.962 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:50.982 INFO:teuthology.orchestra.run.smithi096.stderr:marked in osd.3. 2024-09-13T15:05:51.001 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:51.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:51.103 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:51.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.124 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:51.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:51.225 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:51.247 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:51.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:50 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2968608818' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-09-13T15:05:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:50 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-09-13T15:05:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:50 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:51.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:50 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:51.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:50 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2968608818' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-09-13T15:05:51.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:50 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-09-13T15:05:51.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:50 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:51.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:50 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:51.349 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:51.370 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:50 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2968608818' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-09-13T15:05:51.370 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:50 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-09-13T15:05:51.370 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:50 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:05:51.371 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:50 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:51.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.372 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:51.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:51.473 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:51.496 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:51.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:51.598 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:51.619 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:51.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:51.634 INFO:tasks.thrashosds.thrasher:Added osd 3 2024-09-13T15:05:51.721 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:51.742 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:51.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:51.844 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:51.867 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:51.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:51.969 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:52.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.001 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:52.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:52.102 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:52.125 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:52.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:52.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:51 smithi204 ceph-mon[87071]: pgmap v1300: 119 pgs: 119 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-13T15:05:52.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:51 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-09-13T15:05:52.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:51 smithi204 ceph-mon[87071]: osdmap e383: 8 total, 8 up, 8 in 2024-09-13T15:05:52.227 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:52.248 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:52.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:52.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:51 smithi096 ceph-mon[98901]: pgmap v1300: 119 pgs: 119 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-13T15:05:52.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:51 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-09-13T15:05:52.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:51 smithi096 ceph-mon[98901]: osdmap e383: 8 total, 8 up, 8 in 2024-09-13T15:05:52.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:51 smithi096 ceph-mon[102928]: pgmap v1300: 119 pgs: 119 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-13T15:05:52.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:51 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-09-13T15:05:52.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:51 smithi096 ceph-mon[102928]: osdmap e383: 8 total, 8 up, 8 in 2024-09-13T15:05:52.351 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:52.371 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:52.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:52.474 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:52.494 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:52.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:52.596 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:52.617 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:52.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:52.719 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:52.741 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:52.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:52.842 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:52.864 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:52.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:52.966 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:52.990 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:53.092 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:53.126 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:53.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:53.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:52 smithi204 ceph-mon[87071]: osdmap e384: 8 total, 8 up, 8 in 2024-09-13T15:05:53.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:52 smithi204 ceph-mon[87071]: pgmap v1303: 119 pgs: 34 peering, 85 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.54k op/s 2024-09-13T15:05:53.228 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:53.242 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:52 smithi096 ceph-mon[98901]: osdmap e384: 8 total, 8 up, 8 in 2024-09-13T15:05:53.242 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:52 smithi096 ceph-mon[98901]: pgmap v1303: 119 pgs: 34 peering, 85 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.54k op/s 2024-09-13T15:05:53.242 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:52 smithi096 ceph-mon[102928]: osdmap e384: 8 total, 8 up, 8 in 2024-09-13T15:05:53.242 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:52 smithi096 ceph-mon[102928]: pgmap v1303: 119 pgs: 34 peering, 85 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.54k op/s 2024-09-13T15:05:53.278 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T15:05:53.278 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T15:05:53.279 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.279 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.36552.0:236467 10.8 10:1c7baef6:::benchmark_data_smithi096_288650_object100775:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2024-09-13T15:05:53.279 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:05:53.260355+0000", 2024-09-13T15:05:53.279 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.012998466, 2024-09-13T15:05:53.279 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.013028194999999999, 2024-09-13T15:05:53.279 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:05:53.279 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:05:53.279 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:05:53.279 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T15:05:53.279 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.36552", 2024-09-13T15:05:53.279 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:05:53.279 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 236467 2024-09-13T15:05:53.279 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.280 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:05:53.280 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.280 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:05:53.280 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.260355+0000", 2024-09-13T15:05:53.280 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.280 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.280 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.280 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:05:53.280 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.260355+0000", 2024-09-13T15:05:53.280 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.280 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.280 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.280 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:05:53.281 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.260356+0000", 2024-09-13T15:05:53.281 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.911e-06 2024-09-13T15:05:53.281 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.281 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.281 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:05:53.281 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.260362+0000", 2024-09-13T15:05:53.281 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.0629999999999996e-06 2024-09-13T15:05:53.281 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.281 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.281 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:05:53.281 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.260380+0000", 2024-09-13T15:05:53.281 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8369999999999999e-05 2024-09-13T15:05:53.282 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.282 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.282 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:05:53.282 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.260389+0000", 2024-09-13T15:05:53.282 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.1859999999999998e-06 2024-09-13T15:05:53.282 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.282 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.282 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:05:53.282 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.260446+0000", 2024-09-13T15:05:53.282 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6527000000000003e-05 2024-09-13T15:05:53.282 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.282 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.282 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:05:53.283 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.260594+0000", 2024-09-13T15:05:53.283 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000148517 2024-09-13T15:05:53.283 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.283 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.283 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:05:53.283 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.260710+0000", 2024-09-13T15:05:53.283 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000116171 2024-09-13T15:05:53.283 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.283 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.283 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:53.283 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.271417+0000", 2024-09-13T15:05:53.283 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.010706833000000001 2024-09-13T15:05:53.283 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.283 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.284 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:53.284 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.272895+0000", 2024-09-13T15:05:53.284 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001477372 2024-09-13T15:05:53.284 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.284 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:05:53.284 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.284 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.284 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.284 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.36552.0:236471 10.d e385/355 10:b7d8208a:::benchmark_data_smithi096_288650_object100779:head v 385'14918, mlcod=385'14916)", 2024-09-13T15:05:53.284 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:05:53.261533+0000", 2024-09-13T15:05:53.284 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.01181988, 2024-09-13T15:05:53.284 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011944361000000001, 2024-09-13T15:05:53.284 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:05:53.284 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:05:53.285 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T15:05:53.285 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:05:53.285 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.285 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:05:53.285 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.261533+0000", 2024-09-13T15:05:53.285 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.285 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.285 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.285 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:05:53.285 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.261533+0000", 2024-09-13T15:05:53.285 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.285 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.285 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.286 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:05:53.286 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.261534+0000", 2024-09-13T15:05:53.286 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1299999999999999e-07 2024-09-13T15:05:53.286 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.286 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.286 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:05:53.286 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.261539+0000", 2024-09-13T15:05:53.286 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.558e-06 2024-09-13T15:05:53.286 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.286 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.286 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:05:53.286 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.261557+0000", 2024-09-13T15:05:53.286 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7935999999999999e-05 2024-09-13T15:05:53.287 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.287 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.287 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:05:53.287 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.261564+0000", 2024-09-13T15:05:53.287 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.0960000000000003e-06 2024-09-13T15:05:53.287 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.287 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.287 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:05:53.287 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.261702+0000", 2024-09-13T15:05:53.287 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013731400000000001 2024-09-13T15:05:53.287 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.287 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.287 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:05:53.288 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.261818+0000", 2024-09-13T15:05:53.288 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00011602 2024-09-13T15:05:53.288 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.288 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:05:53.288 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.288 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.288 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.288 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.36552.0:236469 10.6 e385/355 10:637daa2c:::benchmark_data_smithi096_288650_object100777:head v 385'14742, mlcod=385'14740)", 2024-09-13T15:05:53.288 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:05:53.261619+0000", 2024-09-13T15:05:53.288 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.011733748, 2024-09-13T15:05:53.288 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.011943855999999999, 2024-09-13T15:05:53.288 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:05:53.288 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:05:53.289 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T15:05:53.289 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:05:53.289 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.289 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:05:53.289 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.261619+0000", 2024-09-13T15:05:53.289 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.289 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.289 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.289 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:05:53.289 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.261619+0000", 2024-09-13T15:05:53.289 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.289 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.289 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.289 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:05:53.290 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.261620+0000", 2024-09-13T15:05:53.290 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.6799999999999999e-07 2024-09-13T15:05:53.290 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.290 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.290 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:05:53.290 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.261625+0000", 2024-09-13T15:05:53.290 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.4920000000000004e-06 2024-09-13T15:05:53.290 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.290 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.290 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:05:53.290 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.261633+0000", 2024-09-13T15:05:53.290 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.1859999999999992e-06 2024-09-13T15:05:53.290 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.291 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.291 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:05:53.291 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.261638+0000", 2024-09-13T15:05:53.291 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.3170000000000001e-06 2024-09-13T15:05:53.291 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.291 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.291 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:05:53.291 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.262205+0000", 2024-09-13T15:05:53.291 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000566724 2024-09-13T15:05:53.291 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.291 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.291 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:05:53.291 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.262227+0000", 2024-09-13T15:05:53.292 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.2044000000000001e-05 2024-09-13T15:05:53.292 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.292 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:05:53.292 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.292 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.292 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.292 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.36552.0:236474 10.8 10:17826558:::benchmark_data_smithi096_288650_object100782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2024-09-13T15:05:53.292 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:05:53.265623+0000", 2024-09-13T15:05:53.292 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0077296939999999996, 2024-09-13T15:05:53.292 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0079906809999999995, 2024-09-13T15:05:53.292 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:05:53.292 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:05:53.292 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:05:53.293 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T15:05:53.293 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.36552", 2024-09-13T15:05:53.293 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:05:53.293 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 236474 2024-09-13T15:05:53.293 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.293 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:05:53.293 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.293 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:05:53.293 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265623+0000", 2024-09-13T15:05:53.293 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.293 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.293 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.293 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:05:53.294 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265623+0000", 2024-09-13T15:05:53.294 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.294 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.294 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.294 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:05:53.294 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265625+0000", 2024-09-13T15:05:53.294 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.73e-06 2024-09-13T15:05:53.294 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.294 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.294 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:05:53.294 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265631+0000", 2024-09-13T15:05:53.294 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1299999999999998e-06 2024-09-13T15:05:53.294 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.294 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.294 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:05:53.295 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265645+0000", 2024-09-13T15:05:53.295 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3886999999999999e-05 2024-09-13T15:05:53.295 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.295 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.295 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:05:53.295 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265654+0000", 2024-09-13T15:05:53.295 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.4619999999999996e-06 2024-09-13T15:05:53.295 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.295 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.295 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:05:53.295 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.267923+0000", 2024-09-13T15:05:53.295 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0022690399999999999 2024-09-13T15:05:53.295 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.295 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.296 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:05:53.296 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.268052+0000", 2024-09-13T15:05:53.296 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00012900800000000001 2024-09-13T15:05:53.296 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.296 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.296 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:05:53.296 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.268168+0000", 2024-09-13T15:05:53.296 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.000116619 2024-09-13T15:05:53.296 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.296 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.296 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:53.296 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.272562+0000", 2024-09-13T15:05:53.296 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0043937100000000003 2024-09-13T15:05:53.296 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.297 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.297 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:05:53.297 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.272930+0000", 2024-09-13T15:05:53.297 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00036808099999999997 2024-09-13T15:05:53.297 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.297 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:05:53.297 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.297 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.297 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.297 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.36552.0:236475 10.4 10:2433a967:::benchmark_data_smithi096_288650_object100783:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2024-09-13T15:05:53.297 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:05:53.265673+0000", 2024-09-13T15:05:53.297 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0076797929999999999, 2024-09-13T15:05:53.297 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0080246840000000007, 2024-09-13T15:05:53.297 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:05:53.297 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:05:53.298 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:05:53.298 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T15:05:53.298 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.36552", 2024-09-13T15:05:53.298 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:05:53.298 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 236475 2024-09-13T15:05:53.298 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.298 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:05:53.298 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.298 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:05:53.298 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265673+0000", 2024-09-13T15:05:53.298 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.298 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.298 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.299 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:05:53.299 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265673+0000", 2024-09-13T15:05:53.299 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.299 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.299 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.299 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:05:53.299 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265674+0000", 2024-09-13T15:05:53.299 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2799999999999996e-07 2024-09-13T15:05:53.299 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.300 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.300 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:05:53.300 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265676+0000", 2024-09-13T15:05:53.300 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.446e-06 2024-09-13T15:05:53.300 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.300 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.300 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:05:53.300 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265682+0000", 2024-09-13T15:05:53.300 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.9619999999999998e-06 2024-09-13T15:05:53.300 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.300 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.300 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:05:53.300 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265687+0000", 2024-09-13T15:05:53.300 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.1070000000000004e-06 2024-09-13T15:05:53.301 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.301 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.301 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:05:53.301 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265773+0000", 2024-09-13T15:05:53.301 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.5480000000000002e-05 2024-09-13T15:05:53.301 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.301 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.301 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:05:53.301 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265903+0000", 2024-09-13T15:05:53.301 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013017000000000001 2024-09-13T15:05:53.301 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.301 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.301 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "waiting for subops from 1,3,4", 2024-09-13T15:05:53.302 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.266034+0000", 2024-09-13T15:05:53.302 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00013092400000000001 2024-09-13T15:05:53.302 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.302 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:05:53.302 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.302 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.302 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.302 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_op(client.36552.0:236476 10.5 10.63282975 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e385)", 2024-09-13T15:05:53.302 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:05:53.265698+0000", 2024-09-13T15:05:53.302 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.007654709, 2024-09-13T15:05:53.302 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0080588659999999996, 2024-09-13T15:05:53.302 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:05:53.302 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:05:53.303 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "queued for pg", 2024-09-13T15:05:53.303 INFO:teuthology.orchestra.run.smithi096.stdout: "client_info": { 2024-09-13T15:05:53.303 INFO:teuthology.orchestra.run.smithi096.stdout: "client": "client.36552", 2024-09-13T15:05:53.303 INFO:teuthology.orchestra.run.smithi096.stdout: "client_addr": "172.21.15.96:0/1508787212", 2024-09-13T15:05:53.303 INFO:teuthology.orchestra.run.smithi096.stdout: "tid": 236476 2024-09-13T15:05:53.303 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.303 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:05:53.303 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.303 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:05:53.303 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265698+0000", 2024-09-13T15:05:53.303 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.303 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.303 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.304 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:05:53.304 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265698+0000", 2024-09-13T15:05:53.304 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.304 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.304 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.304 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:05:53.304 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265699+0000", 2024-09-13T15:05:53.304 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.6499999999999999e-07 2024-09-13T15:05:53.304 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.304 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.304 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:05:53.304 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265701+0000", 2024-09-13T15:05:53.304 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.858e-06 2024-09-13T15:05:53.304 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.305 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.305 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:05:53.305 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265706+0000", 2024-09-13T15:05:53.305 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.4979999999999997e-06 2024-09-13T15:05:53.305 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.305 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.305 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:05:53.305 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.265711+0000", 2024-09-13T15:05:53.305 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.544e-06 2024-09-13T15:05:53.305 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.305 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:05:53.305 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.305 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.305 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.306 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.36552.0:236472 10.c e385/355 10:34f51c14:::benchmark_data_smithi096_288650_object100780:head v 385'14823, mlcod=385'14821)", 2024-09-13T15:05:53.306 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:05:53.268216+0000", 2024-09-13T15:05:53.306 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0051371560000000004, 2024-09-13T15:05:53.306 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0055969269999999998, 2024-09-13T15:05:53.306 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:05:53.306 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:05:53.306 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T15:05:53.306 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:05:53.306 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.306 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:05:53.306 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.268216+0000", 2024-09-13T15:05:53.306 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.306 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.306 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.307 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:05:53.307 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.268216+0000", 2024-09-13T15:05:53.307 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.307 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.307 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.307 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:05:53.307 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.268217+0000", 2024-09-13T15:05:53.307 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.1399999999999995e-07 2024-09-13T15:05:53.307 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.307 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.307 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:05:53.307 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.268223+0000", 2024-09-13T15:05:53.307 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.8889999999999999e-06 2024-09-13T15:05:53.308 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.308 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.308 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:05:53.308 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.268235+0000", 2024-09-13T15:05:53.308 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.2194e-05 2024-09-13T15:05:53.308 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.308 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.308 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:05:53.308 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.268243+0000", 2024-09-13T15:05:53.308 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.4980000000000006e-06 2024-09-13T15:05:53.308 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.308 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.308 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:05:53.309 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.269864+0000", 2024-09-13T15:05:53.309 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.001620565 2024-09-13T15:05:53.309 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.309 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.309 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:05:53.309 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.269884+0000", 2024-09-13T15:05:53.309 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.9630999999999999e-05 2024-09-13T15:05:53.309 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.309 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:05:53.309 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.309 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.309 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.309 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.36552.0:236473 10.d e385/355)", 2024-09-13T15:05:53.310 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:05:53.269773+0000", 2024-09-13T15:05:53.310 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0035798710000000001, 2024-09-13T15:05:53.310 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0040871240000000001, 2024-09-13T15:05:53.310 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:05:53.310 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:05:53.310 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "queued for pg", 2024-09-13T15:05:53.310 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:05:53.310 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.310 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:05:53.310 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.269773+0000", 2024-09-13T15:05:53.310 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.310 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.310 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.310 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:05:53.311 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.269773+0000", 2024-09-13T15:05:53.311 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.311 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.311 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.311 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:05:53.311 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.269774+0000", 2024-09-13T15:05:53.311 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.7699999999999992e-07 2024-09-13T15:05:53.311 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.311 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.311 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:05:53.311 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.269781+0000", 2024-09-13T15:05:53.311 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.3100000000000003e-06 2024-09-13T15:05:53.311 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.312 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.312 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:05:53.312 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.269793+0000", 2024-09-13T15:05:53.312 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.1548e-05 2024-09-13T15:05:53.312 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.312 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.312 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:05:53.312 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.269801+0000", 2024-09-13T15:05:53.312 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.5689999999999999e-06 2024-09-13T15:05:53.312 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.312 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:05:53.312 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.312 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.312 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.313 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.36552.0:236479 10.c e385/355)", 2024-09-13T15:05:53.313 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:05:53.270818+0000", 2024-09-13T15:05:53.313 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0025348559999999998, 2024-09-13T15:05:53.313 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.003081089, 2024-09-13T15:05:53.313 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:05:53.313 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:05:53.313 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "queued for pg", 2024-09-13T15:05:53.313 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:05:53.313 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.313 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:05:53.313 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.270818+0000", 2024-09-13T15:05:53.313 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.313 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.313 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.314 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:05:53.314 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.270818+0000", 2024-09-13T15:05:53.314 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.314 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.314 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.314 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:05:53.314 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.270819+0000", 2024-09-13T15:05:53.314 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 3.4900000000000001e-07 2024-09-13T15:05:53.314 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.314 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.314 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:05:53.314 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.270823+0000", 2024-09-13T15:05:53.314 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.1189999999999999e-06 2024-09-13T15:05:53.315 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.315 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.315 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:05:53.315 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.270829+0000", 2024-09-13T15:05:53.315 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5560000000000004e-06 2024-09-13T15:05:53.315 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.315 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.315 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:05:53.315 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.270834+0000", 2024-09-13T15:05:53.315 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 4.6949999999999999e-06 2024-09-13T15:05:53.315 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.315 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:05:53.315 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.315 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.316 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.316 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop_reply(client.36552.0:236475 10.4 e385/384)", 2024-09-13T15:05:53.316 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:05:53.273060+0000", 2024-09-13T15:05:53.316 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.00029293800000000002, 2024-09-13T15:05:53.316 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00087564999999999995, 2024-09-13T15:05:53.316 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:05:53.316 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:05:53.316 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "queued for pg", 2024-09-13T15:05:53.316 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:05:53.316 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.316 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:05:53.316 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.273060+0000", 2024-09-13T15:05:53.316 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.317 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.317 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.317 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:05:53.317 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.273060+0000", 2024-09-13T15:05:53.317 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:05:53.317 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.317 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.317 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:05:53.317 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.273061+0000", 2024-09-13T15:05:53.317 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.1699999999999998e-07 2024-09-13T15:05:53.317 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.317 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.317 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:05:53.318 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.273063+0000", 2024-09-13T15:05:53.318 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 2.6189999999999998e-06 2024-09-13T15:05:53.318 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.318 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.318 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:05:53.318 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.273072+0000", 2024-09-13T15:05:53.318 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.6999999999999997e-06 2024-09-13T15:05:53.318 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:05:53.318 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:05:53.318 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:05:53.318 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:05:53.273078+0000", 2024-09-13T15:05:53.318 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.7490000000000004e-06 2024-09-13T15:05:53.318 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.318 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:05:53.318 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.319 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:05:53.319 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2024-09-13T15:05:53.319 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 10 2024-09-13T15:05:53.319 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T15:05:53.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.320 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:53.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:53.421 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:53.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.457 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:53.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:53.558 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:53.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.604 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:53.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:53.705 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:53.743 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:53.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:53.845 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:53.883 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:53.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:53.984 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:54.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.025 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:54.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:54.126 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:54.163 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:54.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:54.230 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[102928]: 1.1e scrub starts 2024-09-13T15:05:54.230 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[102928]: 1.1e scrub ok 2024-09-13T15:05:54.230 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[102928]: Health check failed: Reduced data availability: 6 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-13T15:05:54.230 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[102928]: 10.4 starting backfill to osd.3 from (0'0,0'0] MAX to 382'14562 2024-09-13T15:05:54.230 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[102928]: 10.3 starting backfill to osd.3 from (0'0,0'0] MAX to 382'14933 2024-09-13T15:05:54.230 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[102928]: osdmap e385: 8 total, 8 up, 8 in 2024-09-13T15:05:54.230 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[102928]: 10.5 starting backfill to osd.3 from (0'0,0'0] MAX to 382'14603 2024-09-13T15:05:54.230 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[102928]: 10.f starting backfill to osd.3 from (0'0,0'0] MAX to 382'14835 2024-09-13T15:05:54.230 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[102928]: 10.7 starting backfill to osd.3 from (0'0,0'0] MAX to 382'14833 2024-09-13T15:05:54.237 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[98901]: 1.1e scrub starts 2024-09-13T15:05:54.237 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[98901]: 1.1e scrub ok 2024-09-13T15:05:54.237 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[98901]: Health check failed: Reduced data availability: 6 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-13T15:05:54.237 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[98901]: 10.4 starting backfill to osd.3 from (0'0,0'0] MAX to 382'14562 2024-09-13T15:05:54.237 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[98901]: 10.3 starting backfill to osd.3 from (0'0,0'0] MAX to 382'14933 2024-09-13T15:05:54.237 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[98901]: osdmap e385: 8 total, 8 up, 8 in 2024-09-13T15:05:54.237 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[98901]: 10.5 starting backfill to osd.3 from (0'0,0'0] MAX to 382'14603 2024-09-13T15:05:54.237 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[98901]: 10.f starting backfill to osd.3 from (0'0,0'0] MAX to 382'14835 2024-09-13T15:05:54.237 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:53 smithi096 ceph-mon[98901]: 10.7 starting backfill to osd.3 from (0'0,0'0] MAX to 382'14833 2024-09-13T15:05:54.265 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:54.269 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T15:05:54.283 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:54.308 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:54.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:54.410 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:54.437 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:54.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:53 smithi204 ceph-mon[87071]: 1.1e scrub starts 2024-09-13T15:05:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:53 smithi204 ceph-mon[87071]: 1.1e scrub ok 2024-09-13T15:05:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:53 smithi204 ceph-mon[87071]: Health check failed: Reduced data availability: 6 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-13T15:05:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:53 smithi204 ceph-mon[87071]: 10.4 starting backfill to osd.3 from (0'0,0'0] MAX to 382'14562 2024-09-13T15:05:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:53 smithi204 ceph-mon[87071]: 10.3 starting backfill to osd.3 from (0'0,0'0] MAX to 382'14933 2024-09-13T15:05:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:53 smithi204 ceph-mon[87071]: osdmap e385: 8 total, 8 up, 8 in 2024-09-13T15:05:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:53 smithi204 ceph-mon[87071]: 10.5 starting backfill to osd.3 from (0'0,0'0] MAX to 382'14603 2024-09-13T15:05:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:53 smithi204 ceph-mon[87071]: 10.f starting backfill to osd.3 from (0'0,0'0] MAX to 382'14835 2024-09-13T15:05:54.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:53 smithi204 ceph-mon[87071]: 10.7 starting backfill to osd.3 from (0'0,0'0] MAX to 382'14833 2024-09-13T15:05:54.539 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:54.576 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:54.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:54.678 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:54.720 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:54.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:54.756 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:05:54.821 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:54.852 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:54.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:54.954 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:54.993 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:54.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:55.095 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:55.135 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:55.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:55.237 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:55.263 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:54 smithi096 ceph-mon[98901]: pgmap v1305: 119 pgs: 34 peering, 85 active+clean; 2.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2024-09-13T15:05:55.264 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:54 smithi096 ceph-mon[102928]: pgmap v1305: 119 pgs: 34 peering, 85 active+clean; 2.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2024-09-13T15:05:55.279 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:55.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:55.380 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:55.415 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:55.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:55.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:54 smithi204 ceph-mon[87071]: pgmap v1305: 119 pgs: 34 peering, 85 active+clean; 2.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2024-09-13T15:05:55.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:55.554 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:55.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:55.656 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:55.693 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:55.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:55.701 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:55.763 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:55.794 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:55.833 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:55.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:55.934 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:55.968 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:55.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:56.070 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:56.110 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:56.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:56.212 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:56.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.254 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:56.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:56.355 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:56.397 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:56.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:56.499 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:56.530 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:56.631 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:56.634 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 6, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:05:56.634 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:05:56.634 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-13T15:05:56.635 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.647148 2024-09-13T15:05:56.635 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 6 0.647148427300576 2024-09-13T15:05:56.661 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:56.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:56.764 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:56.785 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:56.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:56.887 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:56.920 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:56.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:57.022 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:57.059 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:57.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:57.161 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:57.195 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:57.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:57.297 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:57.331 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:57.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:57.433 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:57.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.468 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:57.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:57.569 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:57.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:57 smithi096 ceph-mon[98901]: pgmap v1306: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 34 peering, 83 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 3941/104055 objects misplaced (3.787%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T15:05:57.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:57 smithi096 ceph-mon[102928]: pgmap v1306: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 34 peering, 83 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 3941/104055 objects misplaced (3.787%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T15:05:57.606 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:57.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:57.708 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:57.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:57 smithi204 ceph-mon[87071]: pgmap v1306: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 34 peering, 83 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 3941/104055 objects misplaced (3.787%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T15:05:57.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.743 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:57.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:57.844 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:57.878 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:57.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:57.980 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:58.013 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:58.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:58.114 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:58.143 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:58.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:58.245 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:58.291 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:58.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:58.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:05:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:05:57] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.51.0" 2024-09-13T15:05:58.392 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:58.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.430 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:58.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:58.531 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:58.564 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:58.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:58.666 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:58.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.697 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:58.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:58.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:58 smithi204 ceph-mon[87071]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 8 pgs peering) 2024-09-13T15:05:58.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:58 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T15:05:58.797 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:58.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:58 smithi096 ceph-mon[98901]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 8 pgs peering) 2024-09-13T15:05:58.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:58 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T15:05:58.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:58 smithi096 ceph-mon[102928]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 8 pgs peering) 2024-09-13T15:05:58.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:58 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T15:05:58.832 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:58.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:58.934 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:58.975 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:58.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:59.077 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:59.112 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:59.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:59.214 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:59.251 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:59.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:59.346 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T15:05:59.353 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:59.392 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:59.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:59.493 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:59.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.524 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:59.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:59.625 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:59.659 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:05:59.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:59 smithi204 ceph-mon[87071]: pgmap v1307: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 10024/95421 objects misplaced (10.505%); 1.6 MiB/s, 27 objects/s recovering 2024-09-13T15:05:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:59 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2024244355' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:05:59.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:05:59 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:05:59.761 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:59.794 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:59.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:59.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:59 smithi096 ceph-mon[98901]: pgmap v1307: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 10024/95421 objects misplaced (10.505%); 1.6 MiB/s, 27 objects/s recovering 2024-09-13T15:05:59.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:59 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2024244355' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:05:59.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:05:59 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:05:59.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:59 smithi096 ceph-mon[102928]: pgmap v1307: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 10024/95421 objects misplaced (10.505%); 1.6 MiB/s, 27 objects/s recovering 2024-09-13T15:05:59.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:59 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2024244355' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:05:59.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:05:59 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:05:59.896 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:59.932 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:05:59.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:00.034 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:00.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.076 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:00.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:00.176 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:00.207 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:00.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:00.309 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:00.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.348 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:00.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:00.449 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:00 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:00 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:06:00.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:00 smithi204 ceph-mon[87071]: osdmap e386: 8 total, 8 up, 8 in 2024-09-13T15:06:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:06:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:06:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:06:00.478 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:00.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:00.579 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:00.615 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:00.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:00.717 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:00.753 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:00.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:00.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:00 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:00.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:00 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:06:00.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:00 smithi096 ceph-mon[98901]: osdmap e386: 8 total, 8 up, 8 in 2024-09-13T15:06:00.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:00 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:00.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:00 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:06:00.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:00 smithi096 ceph-mon[102928]: osdmap e386: 8 total, 8 up, 8 in 2024-09-13T15:06:00.856 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:00.891 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:00.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:00.992 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:01.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.025 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:01.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:01.126 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:01.162 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:01.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:01.264 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:01.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.305 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:01.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:01.406 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:01.431 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:01.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:01.533 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:01.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.565 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:01.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:01.666 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:01.709 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:01.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:01.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:01 smithi204 ceph-mon[87071]: pgmap v1309: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 8666/91488 objects misplaced (9.472%); 7.7 MiB/s, 125 objects/s recovering 2024-09-13T15:06:01.811 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:01.816 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:01 smithi096 ceph-mon[98901]: pgmap v1309: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 8666/91488 objects misplaced (9.472%); 7.7 MiB/s, 125 objects/s recovering 2024-09-13T15:06:01.816 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:01 smithi096 ceph-mon[102928]: pgmap v1309: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 8666/91488 objects misplaced (9.472%); 7.7 MiB/s, 125 objects/s recovering 2024-09-13T15:06:01.843 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:01.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:01.944 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:01.977 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:01.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:02.079 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:02.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.116 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:02.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:02.217 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:02.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.250 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:02.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:02.352 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:02.390 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:02.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:02.400 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:02.492 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:02.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.534 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:02.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:02.635 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:02.679 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:02.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:02.780 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:02.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.815 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:02.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:02.916 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:02.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.964 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:02.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:03.065 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:03.103 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:03.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:03.205 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:03.241 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:03.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:03.343 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:03.373 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:03.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:03.402 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:03 smithi096 ceph-mon[98901]: pgmap v1310: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 7681/79629 objects misplaced (9.646%); 6.7 MiB/s, 108 objects/s recovering 2024-09-13T15:06:03.403 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:03 smithi096 ceph-mon[102928]: pgmap v1310: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 7681/79629 objects misplaced (9.646%); 6.7 MiB/s, 108 objects/s recovering 2024-09-13T15:06:03.475 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:03.504 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:03.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:03.606 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:03.642 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:03.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:03.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:03 smithi204 ceph-mon[87071]: pgmap v1310: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 7681/79629 objects misplaced (9.646%); 6.7 MiB/s, 108 objects/s recovering 2024-09-13T15:06:03.744 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:03.780 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:03.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:03.882 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:03.923 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:03.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:04.025 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:04.066 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:04.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:04.151 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T15:06:04.152 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T15:06:04.152 INFO:teuthology.orchestra.run.smithi096.stdout: "complaint_time": 30, 2024-09-13T15:06:04.152 INFO:teuthology.orchestra.run.smithi096.stdout: "num_blocked_ops": 0 2024-09-13T15:06:04.152 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T15:06:04.168 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:04.217 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:04.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:04.318 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:04.354 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:04.456 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:04.495 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:04.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:04.597 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:04.629 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:04.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:04.731 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:04.765 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:04.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:04.866 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:04.901 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:04.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:04.928 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:06:05.004 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:05.006 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T15:06:05.032 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:05.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:05.134 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:05.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.225 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:05.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:05.326 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:05.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.381 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:05.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:05.483 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:05.516 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:05.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:05.618 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:05.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.652 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:05.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:05.662 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:05 smithi096 ceph-mon[98901]: pgmap v1311: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 7681/78663 objects misplaced (9.764%); 6.2 MiB/s, 100 objects/s recovering 2024-09-13T15:06:05.663 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:05 smithi096 ceph-mon[102928]: pgmap v1311: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 7681/78663 objects misplaced (9.764%); 6.2 MiB/s, 100 objects/s recovering 2024-09-13T15:06:05.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:05 smithi204 ceph-mon[87071]: pgmap v1311: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 7681/78663 objects misplaced (9.764%); 6.2 MiB/s, 100 objects/s recovering 2024-09-13T15:06:05.753 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:05.775 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:05.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:05.877 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:05.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.899 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:05.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:06.000 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:06.022 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:06.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:06.073 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:06:06.073 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:06:06.074 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:06:06.074 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:06:06.074 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:06:06.074 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:06:06.074 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:06:06.074 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:06:06.074 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:06:06.074 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:06:06.074 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:06:06.075 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:06:06.075 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:06:06.075 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:06:06.075 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:06:06.075 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:06:06.124 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:06.145 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:06.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:06.247 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:06.268 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:06.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:06.370 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:06.392 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:06.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:06.494 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:06.527 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:06.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:06.629 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:06.660 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:06.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:06 smithi204 ceph-mon[87071]: osdmap e387: 8 total, 8 up, 8 in 2024-09-13T15:06:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:06 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1124347903' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:06.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:06.762 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:06.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.798 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:06.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:06.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:06 smithi096 ceph-mon[98901]: osdmap e387: 8 total, 8 up, 8 in 2024-09-13T15:06:06.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:06 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1124347903' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:06.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:06.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:06.824 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:06.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:06 smithi096 ceph-mon[102928]: osdmap e387: 8 total, 8 up, 8 in 2024-09-13T15:06:06.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:06 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1124347903' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:06.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:06.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:06.825 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:06.899 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:06.933 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:06.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:07.035 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:07.072 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:07.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:07.174 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:07.209 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:07.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:07.311 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:07.345 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:07.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:07.448 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:07.481 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:07.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:07.560 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:07.583 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:07.620 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:07.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:07.688 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:07 smithi096 ceph-mon[98901]: pgmap v1313: 119 pgs: 1 active+clean+remapped, 4 active+remapped+backfill_wait, 114 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 6502/75696 objects misplaced (8.590%); 12 MiB/s, 194 objects/s recovering 2024-09-13T15:06:07.689 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:07 smithi096 ceph-mon[98901]: osdmap e388: 8 total, 8 up, 8 in 2024-09-13T15:06:07.689 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:07 smithi096 ceph-mon[102928]: pgmap v1313: 119 pgs: 1 active+clean+remapped, 4 active+remapped+backfill_wait, 114 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 6502/75696 objects misplaced (8.590%); 12 MiB/s, 194 objects/s recovering 2024-09-13T15:06:07.689 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:07 smithi096 ceph-mon[102928]: osdmap e388: 8 total, 8 up, 8 in 2024-09-13T15:06:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:07 smithi204 ceph-mon[87071]: pgmap v1313: 119 pgs: 1 active+clean+remapped, 4 active+remapped+backfill_wait, 114 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 6502/75696 objects misplaced (8.590%); 12 MiB/s, 194 objects/s recovering 2024-09-13T15:06:07.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:07 smithi204 ceph-mon[87071]: osdmap e388: 8 total, 8 up, 8 in 2024-09-13T15:06:07.723 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:07.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.759 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:07.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:07.860 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:07.894 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:07.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:07.996 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:08.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.035 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:08.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:08.136 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:08.170 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:08.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:08.254 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:06:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:06:07] "GET /metrics HTTP/1.1" 200 36609 "" "Prometheus/2.51.0" 2024-09-13T15:06:08.272 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:08.313 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:08.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:08.415 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:08.453 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:08.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:08.555 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:08.594 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:08.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:08.696 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:08.731 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:08.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:08.833 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:08.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.874 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:08.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:08.975 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:09.010 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:09.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:09.112 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:09.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.173 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:09.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:09.274 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:09.306 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:09.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:09.404 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:09.409 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:09.451 INFO:teuthology.orchestra.run.smithi096.stderr:set osd.6 primary-affinity to 0.647148 (8424112) 2024-09-13T15:06:09.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.483 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:09.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:09.520 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:06:09.584 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:09.605 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:09.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:09.693 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:09 smithi096 ceph-mon[98901]: pgmap v1315: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 3 active+remapped+backfill_wait, 114 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 4932/66111 objects misplaced (7.460%); 12 MiB/s, 184 objects/s recovering 2024-09-13T15:06:09.693 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:09 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3688879343' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.647148427300576} : dispatch 2024-09-13T15:06:09.693 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:09 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.647148427300576} : dispatch 2024-09-13T15:06:09.693 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:09 smithi096 ceph-mon[102928]: pgmap v1315: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 3 active+remapped+backfill_wait, 114 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 4932/66111 objects misplaced (7.460%); 12 MiB/s, 184 objects/s recovering 2024-09-13T15:06:09.693 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:09 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3688879343' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.647148427300576} : dispatch 2024-09-13T15:06:09.694 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:09 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.647148427300576} : dispatch 2024-09-13T15:06:09.707 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:09 smithi204 ceph-mon[87071]: pgmap v1315: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 3 active+remapped+backfill_wait, 114 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 4932/66111 objects misplaced (7.460%); 12 MiB/s, 184 objects/s recovering 2024-09-13T15:06:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:09 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3688879343' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.647148427300576} : dispatch 2024-09-13T15:06:09.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:09 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.647148427300576} : dispatch 2024-09-13T15:06:09.730 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:09.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:09.831 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:09.853 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:09.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:09.955 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:09.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.978 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:09.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:10.079 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:10.100 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:10.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:10.202 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:10.224 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:10.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:10.326 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:10.347 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:10.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:10.449 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:10.466 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:06:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:06:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:06:10.472 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:10.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:10.574 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:10.604 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:10.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:10.706 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:10.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:10 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.647148427300576}]': finished 2024-09-13T15:06:10.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:10 smithi204 ceph-mon[87071]: osdmap e389: 8 total, 8 up, 8 in 2024-09-13T15:06:10.721 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:10 smithi204 ceph-mon[87071]: osdmap e390: 8 total, 8 up, 8 in 2024-09-13T15:06:10.731 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:10.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:10.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:10 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.647148427300576}]': finished 2024-09-13T15:06:10.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:10 smithi096 ceph-mon[98901]: osdmap e389: 8 total, 8 up, 8 in 2024-09-13T15:06:10.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:10 smithi096 ceph-mon[98901]: osdmap e390: 8 total, 8 up, 8 in 2024-09-13T15:06:10.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:10 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.647148427300576}]': finished 2024-09-13T15:06:10.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:10 smithi096 ceph-mon[102928]: osdmap e389: 8 total, 8 up, 8 in 2024-09-13T15:06:10.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:10 smithi096 ceph-mon[102928]: osdmap e390: 8 total, 8 up, 8 in 2024-09-13T15:06:10.833 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:10.855 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:10.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:10.957 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:10.980 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:10.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:11.082 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:11.104 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:11.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:11.206 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:11.228 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:11.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:11.330 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:11.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.357 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:11.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:11.458 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:11.486 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:11.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:11.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:11 smithi096 ceph-mon[98901]: pgmap v1317: 119 pgs: 1 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 4677/63144 objects misplaced (7.407%); 15 MiB/s, 245 objects/s recovering 2024-09-13T15:06:11.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:11 smithi096 ceph-mon[98901]: 10.7 continuing backfill to osd.3 from (371'9400,388'15814] MIN to 388'15814 2024-09-13T15:06:11.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:11 smithi096 ceph-mon[98901]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T15:06:11.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:11 smithi096 ceph-mon[98901]: 1.6 scrub starts 2024-09-13T15:06:11.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:11 smithi096 ceph-mon[98901]: osdmap e391: 8 total, 8 up, 8 in 2024-09-13T15:06:11.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:11 smithi096 ceph-mon[102928]: pgmap v1317: 119 pgs: 1 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 4677/63144 objects misplaced (7.407%); 15 MiB/s, 245 objects/s recovering 2024-09-13T15:06:11.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:11 smithi096 ceph-mon[102928]: 10.7 continuing backfill to osd.3 from (371'9400,388'15814] MIN to 388'15814 2024-09-13T15:06:11.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:11 smithi096 ceph-mon[102928]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T15:06:11.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:11 smithi096 ceph-mon[102928]: 1.6 scrub starts 2024-09-13T15:06:11.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:11 smithi096 ceph-mon[102928]: osdmap e391: 8 total, 8 up, 8 in 2024-09-13T15:06:11.588 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:11.626 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:11.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:11 smithi204 ceph-mon[87071]: pgmap v1317: 119 pgs: 1 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 4677/63144 objects misplaced (7.407%); 15 MiB/s, 245 objects/s recovering 2024-09-13T15:06:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:11 smithi204 ceph-mon[87071]: 10.7 continuing backfill to osd.3 from (371'9400,388'15814] MIN to 388'15814 2024-09-13T15:06:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:11 smithi204 ceph-mon[87071]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T15:06:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:11 smithi204 ceph-mon[87071]: 1.6 scrub starts 2024-09-13T15:06:11.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:11 smithi204 ceph-mon[87071]: osdmap e391: 8 total, 8 up, 8 in 2024-09-13T15:06:11.728 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:11.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.773 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:11.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:11.874 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:11.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.912 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:11.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:11.949 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:12.013 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:12.046 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:12.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:12.148 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:12.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.189 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:12.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:12.290 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:12.326 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:12.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:12.428 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:12.461 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:12.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:12.478 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T15:06:12.563 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:12.603 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:12.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:12.705 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:12.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:12 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4035988003' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:12.726 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:12 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4035988003' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:12.726 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:12 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4035988003' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:12.750 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:12.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:12.852 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:12.889 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:12.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:12.991 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:13.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.024 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:13.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:13.125 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:13.160 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:13.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:13.262 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:13.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.300 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:13.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:13.401 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:13.434 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:13.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:13.536 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:13.576 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:13.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:13.678 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:13.714 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:13.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:13.768 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:13 smithi096 ceph-mon[98901]: pgmap v1320: 119 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 2724/55530 objects misplaced (4.905%); 24 MiB/s, 382 objects/s recovering 2024-09-13T15:06:13.768 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:13 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:13.769 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:13 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4035988003' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:06:13.769 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:13 smithi096 ceph-mon[98901]: osdmap e392: 8 total, 8 up, 8 in 2024-09-13T15:06:13.769 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:13 smithi096 ceph-mon[102928]: pgmap v1320: 119 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 2724/55530 objects misplaced (4.905%); 24 MiB/s, 382 objects/s recovering 2024-09-13T15:06:13.769 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:13 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:13.769 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:13 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4035988003' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:06:13.769 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:13 smithi096 ceph-mon[102928]: osdmap e392: 8 total, 8 up, 8 in 2024-09-13T15:06:13.816 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:13.847 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:13.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:13.949 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:13.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:13 smithi204 ceph-mon[87071]: pgmap v1320: 119 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 2724/55530 objects misplaced (4.905%); 24 MiB/s, 382 objects/s recovering 2024-09-13T15:06:13.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:13 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:13.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:13 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4035988003' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:06:13.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:13 smithi204 ceph-mon[87071]: osdmap e392: 8 total, 8 up, 8 in 2024-09-13T15:06:13.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.984 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:13.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:14.085 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:14.126 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:14.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:14.228 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:14.270 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:14.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:14.371 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:14.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.406 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:14.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:14.507 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:14.534 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:14.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:14.636 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:14.658 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:14.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:14.761 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:14.782 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:14.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:14.884 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:14.908 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:14.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:15.010 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:15.032 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:15.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:15.134 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:15.155 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:15.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:15.257 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:15.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.283 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:15.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:15.348 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:06:15.348 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:06:15.385 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:15.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.410 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:15.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:15.511 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:15.549 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:15.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:15.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:15 smithi096 ceph-mon[98901]: pgmap v1322: 119 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 2724/54963 objects misplaced (4.956%); 19 MiB/s, 311 objects/s recovering 2024-09-13T15:06:15.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:15 smithi096 ceph-mon[98901]: osdmap e393: 8 total, 8 up, 8 in 2024-09-13T15:06:15.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:15 smithi096 ceph-mon[102928]: pgmap v1322: 119 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 2724/54963 objects misplaced (4.956%); 19 MiB/s, 311 objects/s recovering 2024-09-13T15:06:15.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:15 smithi096 ceph-mon[102928]: osdmap e393: 8 total, 8 up, 8 in 2024-09-13T15:06:15.651 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:15.691 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:15.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:15.699 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:06:15.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:15 smithi204 ceph-mon[87071]: pgmap v1322: 119 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 2724/54963 objects misplaced (4.956%); 19 MiB/s, 311 objects/s recovering 2024-09-13T15:06:15.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:15 smithi204 ceph-mon[87071]: osdmap e393: 8 total, 8 up, 8 in 2024-09-13T15:06:15.793 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:15.823 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:15.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:15.925 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:15.960 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:15.994 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 6, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:06:15.995 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:06:15.995 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2024-09-13T15:06:15.995 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:06:16.062 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:16.094 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:16.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:16.196 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:16.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.243 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:16.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:16.345 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:16.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.381 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:16.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:16.482 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:16.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.518 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:16.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:16.620 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:16.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.658 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:16.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:16.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:16 smithi204 ceph-mon[87071]: osdmap e394: 8 total, 8 up, 8 in 2024-09-13T15:06:16.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:16 smithi204 ceph-mon[87071]: 1.6 scrub starts 2024-09-13T15:06:16.759 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:16.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.802 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:16.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:16 smithi096 ceph-mon[98901]: osdmap e394: 8 total, 8 up, 8 in 2024-09-13T15:06:16.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:16 smithi096 ceph-mon[98901]: 1.6 scrub starts 2024-09-13T15:06:16.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:16 smithi096 ceph-mon[102928]: osdmap e394: 8 total, 8 up, 8 in 2024-09-13T15:06:16.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:16 smithi096 ceph-mon[102928]: 1.6 scrub starts 2024-09-13T15:06:16.904 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:16.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.947 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:16.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:17.048 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:17.089 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:17.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:17.191 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:17.223 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:17.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:17.325 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:17.361 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T15:06:17.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.377 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:17.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:17.478 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:17.510 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:17.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:17.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:17 smithi096 ceph-mon[98901]: pgmap v1325: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 1.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 2450/53001 objects misplaced (4.623%); 20 MiB/s, 324 objects/s recovering 2024-09-13T15:06:17.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:17 smithi096 ceph-mon[98901]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T15:06:17.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:17 smithi096 ceph-mon[102928]: pgmap v1325: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 1.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 2450/53001 objects misplaced (4.623%); 20 MiB/s, 324 objects/s recovering 2024-09-13T15:06:17.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:17 smithi096 ceph-mon[102928]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T15:06:17.612 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:17.661 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:17.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:17.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:17 smithi204 ceph-mon[87071]: pgmap v1325: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 1.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 2450/53001 objects misplaced (4.623%); 20 MiB/s, 324 objects/s recovering 2024-09-13T15:06:17.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:17 smithi204 ceph-mon[87071]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T15:06:17.764 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:17.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.793 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:17.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:17.894 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:17.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.924 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:17.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:18.025 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:18.061 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:18.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:18.163 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:18.189 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:18.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:18.292 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:18.317 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:18.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:18.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:06:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:06:17] "GET /metrics HTTP/1.1" 200 36609 "" "Prometheus/2.51.0" 2024-09-13T15:06:18.419 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:18.457 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:18.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:18.559 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:18.593 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:18.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:18.611 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:18 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2297225940' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:18.611 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:18 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2297225940' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:18.695 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:18.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:18 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2297225940' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:18.721 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:18.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:18.790 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:06:18.791 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:06:18.791 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:06:18.791 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:06:18.791 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:06:18.791 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:06:18.791 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:06:18.791 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:06:18.792 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:06:18.792 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:06:18.792 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:06:18.792 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:06:18.792 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:06:18.792 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:06:18.792 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:06:18.792 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:06:18.823 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:18.846 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:18.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:18.948 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:18.970 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:18.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:19.072 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:19.094 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:19.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:19.196 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:19.218 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:19.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:19.320 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:19.342 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:19.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:19.444 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:19.469 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:19.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:19.571 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:19.595 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:19.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:19.601 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:19.661 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:19.697 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:19.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:19 smithi204 ceph-mon[87071]: pgmap v1326: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 906 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 1818/43362 objects misplaced (4.193%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T15:06:19.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:19 smithi204 ceph-mon[87071]: osdmap e395: 8 total, 8 up, 8 in 2024-09-13T15:06:19.733 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:19.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:19.752 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:19 smithi096 ceph-mon[102928]: pgmap v1326: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 906 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 1818/43362 objects misplaced (4.193%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T15:06:19.753 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:19 smithi096 ceph-mon[102928]: osdmap e395: 8 total, 8 up, 8 in 2024-09-13T15:06:19.753 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:19 smithi096 ceph-mon[98901]: pgmap v1326: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 906 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 1818/43362 objects misplaced (4.193%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T15:06:19.753 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:19 smithi096 ceph-mon[98901]: osdmap e395: 8 total, 8 up, 8 in 2024-09-13T15:06:19.835 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:19.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.872 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:19.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:19.974 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:20.002 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:20.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:20.104 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:20.146 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:20.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:20.248 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:20.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.278 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:20.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:20.380 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:20.414 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:20.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:20.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:20 smithi204 ceph-mon[87071]: osdmap e396: 8 total, 8 up, 8 in 2024-09-13T15:06:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:06:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:06:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:06:20.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:20.547 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:20.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:20.649 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:20.657 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:20.676 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:20.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:20.778 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:20.782 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:20 smithi096 ceph-mon[98901]: osdmap e396: 8 total, 8 up, 8 in 2024-09-13T15:06:20.782 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:20 smithi096 ceph-mon[102928]: osdmap e396: 8 total, 8 up, 8 in 2024-09-13T15:06:20.811 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:20.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:20.913 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:20.949 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:20.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:21.051 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:21.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.089 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:21.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:21.190 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:21.223 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:21.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:21.325 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:21.356 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:21.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:21.458 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:21.491 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:21.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:21.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:21 smithi096 ceph-mon[98901]: pgmap v1329: 119 pgs: 1 active+remapped+backfill_wait, 118 active+clean; 854 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 943/40836 objects misplaced (2.309%); 13 MiB/s, 212 objects/s recovering 2024-09-13T15:06:21.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:21.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:21 smithi096 ceph-mon[98901]: 1.6 scrub starts 2024-09-13T15:06:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:21 smithi096 ceph-mon[102928]: pgmap v1329: 119 pgs: 1 active+remapped+backfill_wait, 118 active+clean; 854 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 943/40836 objects misplaced (2.309%); 13 MiB/s, 212 objects/s recovering 2024-09-13T15:06:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:21.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:21 smithi096 ceph-mon[102928]: 1.6 scrub starts 2024-09-13T15:06:21.593 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:21.608 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:06:21.624 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:21.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:21.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:21 smithi204 ceph-mon[87071]: pgmap v1329: 119 pgs: 1 active+remapped+backfill_wait, 118 active+clean; 854 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 943/40836 objects misplaced (2.309%); 13 MiB/s, 212 objects/s recovering 2024-09-13T15:06:21.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:21.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:21 smithi204 ceph-mon[87071]: 1.6 scrub starts 2024-09-13T15:06:21.726 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:21.747 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:21.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:21.849 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:21.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.871 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:21.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:21.972 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:21.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.994 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:21.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:22.096 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:22.117 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:22.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:22.219 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:22.240 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:22.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:22.342 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:22.365 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:22.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:22.467 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:22.492 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:22.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:22.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:22.632 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:22.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:22.694 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:22 smithi096 ceph-mon[98901]: osdmap e397: 8 total, 8 up, 8 in 2024-09-13T15:06:22.694 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:22 smithi096 ceph-mon[102928]: osdmap e397: 8 total, 8 up, 8 in 2024-09-13T15:06:22.737 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:22.777 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:22.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:22.879 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:22.911 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:22.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:22.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:22 smithi204 ceph-mon[87071]: osdmap e397: 8 total, 8 up, 8 in 2024-09-13T15:06:23.013 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:23.057 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:23.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:23.159 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:23.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.198 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:23.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:23.299 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:23.338 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:23.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:23.440 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:23.475 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:23.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:23.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:23 smithi096 ceph-mon[98901]: pgmap v1331: 119 pgs: 1 peering, 118 active+clean; 659 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s; 21 MiB/s, 329 objects/s recovering 2024-09-13T15:06:23.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:23 smithi096 ceph-mon[98901]: osdmap e398: 8 total, 8 up, 8 in 2024-09-13T15:06:23.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:23 smithi096 ceph-mon[102928]: pgmap v1331: 119 pgs: 1 peering, 118 active+clean; 659 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s; 21 MiB/s, 329 objects/s recovering 2024-09-13T15:06:23.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:23 smithi096 ceph-mon[102928]: osdmap e398: 8 total, 8 up, 8 in 2024-09-13T15:06:23.576 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:23.609 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:23.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:23.711 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:23.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.745 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:23.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:23.846 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:23.883 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:23.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:23.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:23 smithi204 ceph-mon[87071]: pgmap v1331: 119 pgs: 1 peering, 118 active+clean; 659 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s; 21 MiB/s, 329 objects/s recovering 2024-09-13T15:06:23.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:23 smithi204 ceph-mon[87071]: osdmap e398: 8 total, 8 up, 8 in 2024-09-13T15:06:23.985 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:24.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.027 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:24.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:24.128 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:24.163 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:24.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:24.265 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:24.307 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:24.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:24.409 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:24.456 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:24.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:24.481 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:24 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:06:24.559 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:24.596 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:24.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:24.694 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T15:06:24.696 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T15:06:24.698 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:24.742 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:24.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:24.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:24 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:06:24.844 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:24.877 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:24.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:24.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:24 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:06:24.980 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:25.007 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:25.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:25.109 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:25.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.147 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:25.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:25.248 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:25.282 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:25.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:25.384 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:25.411 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:25.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:25.513 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:25.548 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:25.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:25.650 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:25.688 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:25.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:25.790 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:25.821 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:25.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:25.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:25 smithi096 ceph-mon[98901]: pgmap v1333: 119 pgs: 1 peering, 118 active+clean; 645 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 18 MiB/s, 283 objects/s recovering 2024-09-13T15:06:25.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:25 smithi096 ceph-mon[98901]: from='client.46634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:06:25.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:25 smithi096 ceph-mon[102928]: pgmap v1333: 119 pgs: 1 peering, 118 active+clean; 645 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 18 MiB/s, 283 objects/s recovering 2024-09-13T15:06:25.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:25 smithi096 ceph-mon[102928]: from='client.46634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:06:25.923 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:25.954 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:25.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:25.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:25 smithi204 ceph-mon[87071]: pgmap v1333: 119 pgs: 1 peering, 118 active+clean; 645 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 18 MiB/s, 283 objects/s recovering 2024-09-13T15:06:25.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:25 smithi204 ceph-mon[87071]: from='client.46634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:06:26.056 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:26.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.095 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:26.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:26.196 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:26.241 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:26.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:26.343 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:26.377 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:26.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:26.479 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:26.503 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T15:06:26.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.528 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:26.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:26.629 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:26.667 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:26.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:26.694 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":1333,"stamp":"2024-09-13T15:06:24.234756+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":676403173,"num_objects":10277,"num_object_clones":0,"num_object_copies":30831,"num_objects_missing_on_primary":0,"num_objects_missing":1,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10277,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":262038,"num_write_kb":8696720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6118,"num_bytes_recovered":401358764,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109771,"ondisk_log_size":109771,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8832780,"kb_used_data":2363636,"kb_used_omap":39,"kb_used_meta":6468952,"kb_avail":741128436,"statfs":{"total":767960285184,"available":758915518464,"internally_reserved":0,"allocated":2420363264,"data_stored":2398289251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40927,"internal_metadata":6624206881},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,4,1,4,9,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-273743872,"num_objects":-4177,"num_object_clones":0,"num_object_copies":-12531,"num_objects_missing_on_primary":0,"num_objects_missing":1,"num_objects_degraded":0,"num_objects_misplaced":-1818,"num_objects_unfound":0,"num_objects_dirty":-4177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4177,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1647,"num_bytes_recovered":107937792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.811859"},"pg_stats":[{"pgid":"1.11","version":"82'100","reported_seq":404,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.444615+0000","last_change":"2024-09-13T15:05:52.008756+0000","last_active":"2024-09-13T15:06:19.444615+0000","last_peered":"2024-09-13T15:06:19.444615+0000","last_clean":"2024-09-13T15:06:19.444615+0000","last_became_active":"2024-09-13T15:05:52.007609+0000","last_became_peered":"2024-09-13T15:05:52.007609+0000","last_unstale":"2024-09-13T15:06:19.444615+0000","last_undegraded":"2024-09-13T15:06:19.444615+0000","last_fullsized":"2024-09-13T15:06:19.444615+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.087994+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.087994+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:33.077541+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":38,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.477529+0000","last_change":"2024-09-13T15:05:52.015281+0000","last_active":"2024-09-13T15:06:21.477529+0000","last_peered":"2024-09-13T15:06:21.477529+0000","last_clean":"2024-09-13T15:06:21.477529+0000","last_became_active":"2024-09-13T15:05:52.014517+0000","last_became_peered":"2024-09-13T15:05:52.014517+0000","last_unstale":"2024-09-13T15:06:21.477529+0000","last_undegraded":"2024-09-13T15:06:21.477529+0000","last_fullsized":"2024-09-13T15:06:21.477529+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:10:37.226490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":363,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.442575+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T15:06:19.442575+0000","last_peered":"2024-09-13T15:06:19.442575+0000","last_clean":"2024-09-13T15:06:19.442575+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T15:06:19.442575+0000","last_undegraded":"2024-09-13T15:06:19.442575+0000","last_fullsized":"2024-09-13T15:06:19.442575+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":127,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.494522+0000","last_change":"2024-09-13T14:56:04.643672+0000","last_active":"2024-09-13T15:06:19.494522+0000","last_peered":"2024-09-13T15:06:19.494522+0000","last_clean":"2024-09-13T15:06:19.494522+0000","last_became_active":"2024-09-13T14:55:55.185995+0000","last_became_peered":"2024-09-13T14:55:55.185995+0000","last_unstale":"2024-09-13T15:06:19.494522+0000","last_undegraded":"2024-09-13T15:06:19.494522+0000","last_fullsized":"2024-09-13T15:06:19.494522+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.643614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.643614+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:36.495546+0000","scrub_duration":0.001441681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":380,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.494826+0000","last_change":"2024-09-13T15:05:52.014781+0000","last_active":"2024-09-13T15:06:19.494826+0000","last_peered":"2024-09-13T15:06:19.494826+0000","last_clean":"2024-09-13T15:06:19.494826+0000","last_became_active":"2024-09-13T15:05:52.014049+0000","last_became_peered":"2024-09-13T15:05:52.014049+0000","last_unstale":"2024-09-13T15:06:19.494826+0000","last_undegraded":"2024-09-13T15:06:19.494826+0000","last_fullsized":"2024-09-13T15:06:19.494826+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:33.528483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":695,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.444725+0000","last_change":"2024-09-13T15:05:52.011548+0000","last_active":"2024-09-13T15:06:19.444725+0000","last_peered":"2024-09-13T15:06:19.444725+0000","last_clean":"2024-09-13T15:06:19.444725+0000","last_became_active":"2024-09-13T15:05:52.009924+0000","last_became_peered":"2024-09-13T15:05:52.009924+0000","last_unstale":"2024-09-13T15:06:19.444725+0000","last_undegraded":"2024-09-13T15:06:19.444725+0000","last_fullsized":"2024-09-13T15:06:19.444725+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:49.485860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":839,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.534846+0000","last_change":"2024-09-13T15:05:52.009899+0000","last_active":"2024-09-13T15:06:22.534846+0000","last_peered":"2024-09-13T15:06:22.534846+0000","last_clean":"2024-09-13T15:06:22.534846+0000","last_became_active":"2024-09-13T15:05:52.009365+0000","last_became_peered":"2024-09-13T15:05:52.009365+0000","last_unstale":"2024-09-13T15:06:22.534846+0000","last_undegraded":"2024-09-13T15:06:22.534846+0000","last_fullsized":"2024-09-13T15:06:22.534846+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.240882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.240882+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:02:12.573230+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":688,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.445081+0000","last_change":"2024-09-13T15:05:52.008360+0000","last_active":"2024-09-13T15:06:19.445081+0000","last_peered":"2024-09-13T15:06:19.445081+0000","last_clean":"2024-09-13T15:06:19.445081+0000","last_became_active":"2024-09-13T15:05:52.007789+0000","last_became_peered":"2024-09-13T15:05:52.007789+0000","last_unstale":"2024-09-13T15:06:19.445081+0000","last_undegraded":"2024-09-13T15:06:19.445081+0000","last_fullsized":"2024-09-13T15:06:19.445081+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:12:26.443562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":364,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.442722+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T15:06:19.442722+0000","last_peered":"2024-09-13T15:06:19.442722+0000","last_clean":"2024-09-13T15:06:19.442722+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T15:06:19.442722+0000","last_undegraded":"2024-09-13T15:06:19.442722+0000","last_fullsized":"2024-09-13T15:06:19.442722+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":127,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.495205+0000","last_change":"2024-09-13T14:56:03.605447+0000","last_active":"2024-09-13T15:06:19.495205+0000","last_peered":"2024-09-13T15:06:19.495205+0000","last_clean":"2024-09-13T15:06:19.495205+0000","last_became_active":"2024-09-13T14:55:55.218939+0000","last_became_peered":"2024-09-13T14:55:55.218939+0000","last_unstale":"2024-09-13T15:06:19.495205+0000","last_undegraded":"2024-09-13T15:06:19.495205+0000","last_fullsized":"2024-09-13T15:06:19.495205+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.605356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.605356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:12.467371+0000","scrub_duration":0.0020269620000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":38,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.478754+0000","last_change":"2024-09-13T15:05:51.999191+0000","last_active":"2024-09-13T15:06:21.478754+0000","last_peered":"2024-09-13T15:06:21.478754+0000","last_clean":"2024-09-13T15:06:21.478754+0000","last_became_active":"2024-09-13T15:05:51.998885+0000","last_became_peered":"2024-09-13T15:05:51.998885+0000","last_unstale":"2024-09-13T15:06:21.478754+0000","last_undegraded":"2024-09-13T15:06:21.478754+0000","last_fullsized":"2024-09-13T15:06:21.478754+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:52.548839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":191,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.448449+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T15:06:19.448449+0000","last_peered":"2024-09-13T15:06:19.448449+0000","last_clean":"2024-09-13T15:06:19.448449+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T15:06:19.448449+0000","last_undegraded":"2024-09-13T15:06:19.448449+0000","last_fullsized":"2024-09-13T15:06:19.448449+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"397'16286","reported_seq":16336,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.805143+0000","last_change":"2024-09-13T15:02:39.884666+0000","last_active":"2024-09-13T15:06:21.805143+0000","last_peered":"2024-09-13T15:06:21.805143+0000","last_clean":"2024-09-13T15:06:21.805143+0000","last_became_active":"2024-09-13T15:02:39.884349+0000","last_became_peered":"2024-09-13T15:02:39.884349+0000","last_unstale":"2024-09-13T15:06:21.805143+0000","last_undegraded":"2024-09-13T15:06:21.805143+0000","last_fullsized":"2024-09-13T15:06:21.805143+0000","mapping_epoch":355,"log_start":"369'8500","ondisk_log_start":"369'8500","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":7786,"log_dups_size":0,"ondisk_log_size":7786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41811968,"num_objects":638,"num_object_clones":0,"num_object_copies":1914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16286,"num_write_kb":541568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":147,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443635+0000","last_change":"2024-09-13T15:02:53.820519+0000","last_active":"2024-09-13T15:06:19.443635+0000","last_peered":"2024-09-13T15:06:19.443635+0000","last_clean":"2024-09-13T15:06:19.443635+0000","last_became_active":"2024-09-13T14:59:53.222306+0000","last_became_peered":"2024-09-13T14:59:53.222306+0000","last_unstale":"2024-09-13T15:06:19.443635+0000","last_undegraded":"2024-09-13T15:06:19.443635+0000","last_fullsized":"2024-09-13T15:06:19.443635+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:53.820435+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:53.820435+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:40.108691+0000","scrub_duration":0.0030873739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":649,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-09-13T15:06:18.440576+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T15:06:18.440576+0000","last_peered":"2024-09-13T15:06:18.440576+0000","last_clean":"2024-09-13T15:06:18.440576+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T15:06:18.440576+0000","last_undegraded":"2024-09-13T15:06:18.440576+0000","last_fullsized":"2024-09-13T15:06:18.440576+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":383,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.494698+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T15:06:19.494698+0000","last_peered":"2024-09-13T15:06:19.494698+0000","last_clean":"2024-09-13T15:06:19.494698+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T15:06:19.494698+0000","last_undegraded":"2024-09-13T15:06:19.494698+0000","last_fullsized":"2024-09-13T15:06:19.494698+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"396'16419","reported_seq":18101,"reported_epoch":397,"state":"peering","last_fresh":"2024-09-13T15:06:21.467361+0000","last_change":"2024-09-13T15:06:21.987194+0000","last_active":"2024-09-13T15:06:21.987194+0000","last_peered":"2024-09-13T15:06:21.467361+0000","last_clean":"2024-09-13T15:06:21.467361+0000","last_became_active":"2024-09-13T15:06:10.445385+0000","last_became_peered":"2024-09-13T15:06:10.445385+0000","last_unstale":"2024-09-13T15:06:21.467361+0000","last_undegraded":"2024-09-13T15:06:21.467361+0000","last_fullsized":"2024-09-13T15:06:21.467361+0000","mapping_epoch":397,"log_start":"373'10000","ondisk_log_start":"373'10000","created":355,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6419,"log_dups_size":0,"ondisk_log_size":6419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44105728,"num_objects":673,"num_object_clones":0,"num_object_copies":2019,"num_objects_missing_on_primary":0,"num_objects_missing":1,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16419,"num_write_kb":546944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":802,"num_bytes_recovered":52494336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":42,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.477438+0000","last_change":"2024-09-13T15:05:52.036891+0000","last_active":"2024-09-13T15:06:21.477438+0000","last_peered":"2024-09-13T15:06:21.477438+0000","last_clean":"2024-09-13T15:06:21.477438+0000","last_became_active":"2024-09-13T15:05:52.003251+0000","last_became_peered":"2024-09-13T15:05:52.003251+0000","last_unstale":"2024-09-13T15:06:21.477438+0000","last_undegraded":"2024-09-13T15:06:21.477438+0000","last_fullsized":"2024-09-13T15:06:21.477438+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.016260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.016260+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:22.300773+0000","scrub_duration":71,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":363,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443255+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T15:06:19.443255+0000","last_peered":"2024-09-13T15:06:19.443255+0000","last_clean":"2024-09-13T15:06:19.443255+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T15:06:19.443255+0000","last_undegraded":"2024-09-13T15:06:19.443255+0000","last_fullsized":"2024-09-13T15:06:19.443255+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":438,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.442965+0000","last_change":"2024-09-13T15:05:52.000473+0000","last_active":"2024-09-13T15:06:19.442965+0000","last_peered":"2024-09-13T15:06:19.442965+0000","last_clean":"2024-09-13T15:06:19.442965+0000","last_became_active":"2024-09-13T15:05:52.000152+0000","last_became_peered":"2024-09-13T15:05:52.000152+0000","last_unstale":"2024-09-13T15:06:19.442965+0000","last_undegraded":"2024-09-13T15:06:19.442965+0000","last_fullsized":"2024-09-13T15:06:19.442965+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:31.530855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"396'15991","reported_seq":17923,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.601274+0000","last_change":"2024-09-13T15:06:19.482802+0000","last_active":"2024-09-13T15:06:19.601274+0000","last_peered":"2024-09-13T15:06:19.601274+0000","last_clean":"2024-09-13T15:06:19.601274+0000","last_became_active":"2024-09-13T15:06:19.447969+0000","last_became_peered":"2024-09-13T15:06:19.447969+0000","last_unstale":"2024-09-13T15:06:19.601274+0000","last_undegraded":"2024-09-13T15:06:19.601274+0000","last_fullsized":"2024-09-13T15:06:19.601274+0000","mapping_epoch":395,"log_start":"373'9900","ondisk_log_start":"373'9900","created":355,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6091,"log_dups_size":0,"ondisk_log_size":6091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:24.747914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49217536,"num_objects":751,"num_object_clones":0,"num_object_copies":2253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15991,"num_write_kb":535744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":909,"num_bytes_recovered":59572224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":230,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.449150+0000","last_change":"2024-09-13T14:56:08.889974+0000","last_active":"2024-09-13T15:06:19.449150+0000","last_peered":"2024-09-13T15:06:19.449150+0000","last_clean":"2024-09-13T15:06:19.449150+0000","last_became_active":"2024-09-13T14:55:55.185755+0000","last_became_peered":"2024-09-13T14:55:55.185755+0000","last_unstale":"2024-09-13T15:06:19.449150+0000","last_undegraded":"2024-09-13T15:06:19.449150+0000","last_fullsized":"2024-09-13T15:06:19.449150+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:08.889814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:08.889814+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:02.911564+0000","scrub_duration":0.0053731569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":373,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443893+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T15:06:19.443893+0000","last_peered":"2024-09-13T15:06:19.443893+0000","last_clean":"2024-09-13T15:06:19.443893+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T15:06:19.443893+0000","last_undegraded":"2024-09-13T15:06:19.443893+0000","last_fullsized":"2024-09-13T15:06:19.443893+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":201,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.448663+0000","last_change":"2024-09-13T15:02:38.875727+0000","last_active":"2024-09-13T15:06:19.448663+0000","last_peered":"2024-09-13T15:06:19.448663+0000","last_clean":"2024-09-13T15:06:19.448663+0000","last_became_active":"2024-09-13T15:02:38.875372+0000","last_became_peered":"2024-09-13T15:02:38.875372+0000","last_unstale":"2024-09-13T15:06:19.448663+0000","last_undegraded":"2024-09-13T15:06:19.448663+0000","last_fullsized":"2024-09-13T15:06:19.448663+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:08.504810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"396'15952","reported_seq":19528,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.621251+0000","last_change":"2024-09-13T15:06:06.458626+0000","last_active":"2024-09-13T15:06:19.621251+0000","last_peered":"2024-09-13T15:06:19.621251+0000","last_clean":"2024-09-13T15:06:19.621251+0000","last_became_active":"2024-09-13T15:06:06.428952+0000","last_became_peered":"2024-09-13T15:06:06.428952+0000","last_unstale":"2024-09-13T15:06:19.621251+0000","last_undegraded":"2024-09-13T15:06:19.621251+0000","last_fullsized":"2024-09-13T15:06:19.621251+0000","mapping_epoch":387,"log_start":"373'9900","ondisk_log_start":"373'9900","created":355,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6052,"log_dups_size":0,"ondisk_log_size":6052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:23.903284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51380224,"num_objects":784,"num_object_clones":0,"num_object_copies":2352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15952,"num_write_kb":535552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1732,"num_bytes_recovered":113508352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":167,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443832+0000","last_change":"2024-09-13T15:05:51.996175+0000","last_active":"2024-09-13T15:06:19.443832+0000","last_peered":"2024-09-13T15:06:19.443832+0000","last_clean":"2024-09-13T15:06:19.443832+0000","last_became_active":"2024-09-13T15:05:51.995536+0000","last_became_peered":"2024-09-13T15:05:51.995536+0000","last_unstale":"2024-09-13T15:06:19.443832+0000","last_undegraded":"2024-09-13T15:06:19.443832+0000","last_fullsized":"2024-09-13T15:06:19.443832+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.358699+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.358699+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:58.331262+0000","scrub_duration":0.0023994049999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":401,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.494863+0000","last_change":"2024-09-13T15:05:52.015239+0000","last_active":"2024-09-13T15:06:19.494863+0000","last_peered":"2024-09-13T15:06:19.494863+0000","last_clean":"2024-09-13T15:06:19.494863+0000","last_became_active":"2024-09-13T15:05:52.014825+0000","last_became_peered":"2024-09-13T15:05:52.014825+0000","last_unstale":"2024-09-13T15:06:19.494863+0000","last_undegraded":"2024-09-13T15:06:19.494863+0000","last_fullsized":"2024-09-13T15:06:19.494863+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:47.548807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":661,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.445169+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T15:06:19.445169+0000","last_peered":"2024-09-13T15:06:19.445169+0000","last_clean":"2024-09-13T15:06:19.445169+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T15:06:19.445169+0000","last_undegraded":"2024-09-13T15:06:19.445169+0000","last_fullsized":"2024-09-13T15:06:19.445169+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"398'16210","reported_seq":16308,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:23.980946+0000","last_change":"2024-09-13T15:02:39.881732+0000","last_active":"2024-09-13T15:06:23.980946+0000","last_peered":"2024-09-13T15:06:23.980946+0000","last_clean":"2024-09-13T15:06:23.980946+0000","last_became_active":"2024-09-13T15:02:39.880853+0000","last_became_peered":"2024-09-13T15:02:39.880853+0000","last_unstale":"2024-09-13T15:06:23.980946+0000","last_undegraded":"2024-09-13T15:06:23.980946+0000","last_fullsized":"2024-09-13T15:06:23.980946+0000","mapping_epoch":355,"log_start":"371'9000","ondisk_log_start":"371'9000","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":7210,"log_dups_size":0,"ondisk_log_size":7210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:19.523510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32636928,"num_objects":498,"num_object_clones":0,"num_object_copies":1494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16210,"num_write_kb":534656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":584,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.442592+0000","last_change":"2024-09-13T15:05:52.006880+0000","last_active":"2024-09-13T15:06:19.442592+0000","last_peered":"2024-09-13T15:06:19.442592+0000","last_clean":"2024-09-13T15:06:19.442592+0000","last_became_active":"2024-09-13T15:05:52.005704+0000","last_became_peered":"2024-09-13T15:05:52.005704+0000","last_unstale":"2024-09-13T15:06:19.442592+0000","last_undegraded":"2024-09-13T15:06:19.442592+0000","last_fullsized":"2024-09-13T15:06:19.442592+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:50.810045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:50.810045+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:39:30.729375+0000","scrub_duration":0.0042318529999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":42,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.477775+0000","last_change":"2024-09-13T15:05:52.020378+0000","last_active":"2024-09-13T15:06:21.477775+0000","last_peered":"2024-09-13T15:06:21.477775+0000","last_clean":"2024-09-13T15:06:21.477775+0000","last_became_active":"2024-09-13T15:05:52.002686+0000","last_became_peered":"2024-09-13T15:05:52.002686+0000","last_unstale":"2024-09-13T15:06:21.477775+0000","last_undegraded":"2024-09-13T15:06:21.477775+0000","last_fullsized":"2024-09-13T15:06:21.477775+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.430732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":401,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443720+0000","last_change":"2024-09-13T15:05:51.998495+0000","last_active":"2024-09-13T15:06:19.443720+0000","last_peered":"2024-09-13T15:06:19.443720+0000","last_clean":"2024-09-13T15:06:19.443720+0000","last_became_active":"2024-09-13T15:05:51.998159+0000","last_became_peered":"2024-09-13T15:05:51.998159+0000","last_unstale":"2024-09-13T15:06:19.443720+0000","last_undegraded":"2024-09-13T15:06:19.443720+0000","last_fullsized":"2024-09-13T15:06:19.443720+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:23:05.948148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"397'16595","reported_seq":19106,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.752444+0000","last_change":"2024-09-13T15:06:15.455276+0000","last_active":"2024-09-13T15:06:21.752444+0000","last_peered":"2024-09-13T15:06:21.752444+0000","last_clean":"2024-09-13T15:06:21.752444+0000","last_became_active":"2024-09-13T15:06:15.410981+0000","last_became_peered":"2024-09-13T15:06:15.410981+0000","last_unstale":"2024-09-13T15:06:21.752444+0000","last_undegraded":"2024-09-13T15:06:21.752444+0000","last_fullsized":"2024-09-13T15:06:21.752444+0000","mapping_epoch":393,"log_start":"370'8900","ondisk_log_start":"370'8900","created":355,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":7695,"log_dups_size":0,"ondisk_log_size":7695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:44:22.392748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43188224,"num_objects":659,"num_object_clones":0,"num_object_copies":1977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16595,"num_write_kb":552128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1218,"num_bytes_recovered":79822848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":348,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.494523+0000","last_change":"2024-09-13T15:05:52.007408+0000","last_active":"2024-09-13T15:06:19.494523+0000","last_peered":"2024-09-13T15:06:19.494523+0000","last_clean":"2024-09-13T15:06:19.494523+0000","last_became_active":"2024-09-13T15:05:52.007032+0000","last_became_peered":"2024-09-13T15:05:52.007032+0000","last_unstale":"2024-09-13T15:06:19.494523+0000","last_undegraded":"2024-09-13T15:06:19.494523+0000","last_fullsized":"2024-09-13T15:06:19.494523+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:57.138789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:57.138789+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:28:20.106072+0000","scrub_duration":0.0036861670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":199,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443123+0000","last_change":"2024-09-13T15:06:10.447862+0000","last_active":"2024-09-13T15:06:19.443123+0000","last_peered":"2024-09-13T15:06:19.443123+0000","last_clean":"2024-09-13T15:06:19.443123+0000","last_became_active":"2024-09-13T15:06:10.447585+0000","last_became_peered":"2024-09-13T15:06:10.447585+0000","last_unstale":"2024-09-13T15:06:19.443123+0000","last_undegraded":"2024-09-13T15:06:19.443123+0000","last_fullsized":"2024-09-13T15:06:19.443123+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:02:14.708389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":378,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443258+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T15:06:19.443258+0000","last_peered":"2024-09-13T15:06:19.443258+0000","last_clean":"2024-09-13T15:06:19.443258+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T15:06:19.443258+0000","last_undegraded":"2024-09-13T15:06:19.443258+0000","last_fullsized":"2024-09-13T15:06:19.443258+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"397'16364","reported_seq":16414,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.763915+0000","last_change":"2024-09-13T15:02:39.881185+0000","last_active":"2024-09-13T15:06:21.763915+0000","last_peered":"2024-09-13T15:06:21.763915+0000","last_clean":"2024-09-13T15:06:21.763915+0000","last_became_active":"2024-09-13T15:02:39.880841+0000","last_became_peered":"2024-09-13T15:02:39.880841+0000","last_unstale":"2024-09-13T15:06:21.763915+0000","last_undegraded":"2024-09-13T15:06:21.763915+0000","last_fullsized":"2024-09-13T15:06:21.763915+0000","mapping_epoch":355,"log_start":"373'9900","ondisk_log_start":"373'9900","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6464,"log_dups_size":0,"ondisk_log_size":6464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:49:37.338743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39321600,"num_objects":600,"num_object_clones":0,"num_object_copies":1800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16364,"num_write_kb":542848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":848,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.534952+0000","last_change":"2024-09-13T15:00:10.659910+0000","last_active":"2024-09-13T15:06:22.534952+0000","last_peered":"2024-09-13T15:06:22.534952+0000","last_clean":"2024-09-13T15:06:22.534952+0000","last_became_active":"2024-09-13T14:59:53.216739+0000","last_became_peered":"2024-09-13T14:59:53.216739+0000","last_unstale":"2024-09-13T15:06:22.534952+0000","last_undegraded":"2024-09-13T15:06:22.534952+0000","last_fullsized":"2024-09-13T15:06:22.534952+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.659838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.659838+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:23:09.068190+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":648,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.444449+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T15:06:19.444449+0000","last_peered":"2024-09-13T15:06:19.444449+0000","last_clean":"2024-09-13T15:06:19.444449+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T15:06:19.444449+0000","last_undegraded":"2024-09-13T15:06:19.444449+0000","last_fullsized":"2024-09-13T15:06:19.444449+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":381,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443881+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T15:06:19.443881+0000","last_peered":"2024-09-13T15:06:19.443881+0000","last_clean":"2024-09-13T15:06:19.443881+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T15:06:19.443881+0000","last_undegraded":"2024-09-13T15:06:19.443881+0000","last_fullsized":"2024-09-13T15:06:19.443881+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"397'16480","reported_seq":16530,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.708715+0000","last_change":"2024-09-13T15:02:39.881232+0000","last_active":"2024-09-13T15:06:21.708715+0000","last_peered":"2024-09-13T15:06:21.708715+0000","last_clean":"2024-09-13T15:06:21.708715+0000","last_became_active":"2024-09-13T15:02:39.880899+0000","last_became_peered":"2024-09-13T15:02:39.880899+0000","last_unstale":"2024-09-13T15:06:21.708715+0000","last_undegraded":"2024-09-13T15:06:21.708715+0000","last_fullsized":"2024-09-13T15:06:21.708715+0000","mapping_epoch":355,"log_start":"373'10300","ondisk_log_start":"373'10300","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6180,"log_dups_size":0,"ondisk_log_size":6180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:53:52.712556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41156608,"num_objects":628,"num_object_clones":0,"num_object_copies":1884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16480,"num_write_kb":547456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":327,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.495073+0000","last_change":"2024-09-13T15:02:58.197427+0000","last_active":"2024-09-13T15:06:19.495073+0000","last_peered":"2024-09-13T15:06:19.495073+0000","last_clean":"2024-09-13T15:06:19.495073+0000","last_became_active":"2024-09-13T14:59:53.232987+0000","last_became_peered":"2024-09-13T14:59:53.232987+0000","last_unstale":"2024-09-13T15:06:19.495073+0000","last_undegraded":"2024-09-13T15:06:19.495073+0000","last_fullsized":"2024-09-13T15:06:19.495073+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:58.195731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:58.195731+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:51.565489+0000","scrub_duration":0.011077536000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":38,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.476535+0000","last_change":"2024-09-13T15:05:51.991616+0000","last_active":"2024-09-13T15:06:21.476535+0000","last_peered":"2024-09-13T15:06:21.476535+0000","last_clean":"2024-09-13T15:06:21.476535+0000","last_became_active":"2024-09-13T15:05:51.991294+0000","last_became_peered":"2024-09-13T15:05:51.991294+0000","last_unstale":"2024-09-13T15:06:21.476535+0000","last_undegraded":"2024-09-13T15:06:21.476535+0000","last_fullsized":"2024-09-13T15:06:21.476535+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:05.060508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":667,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.533497+0000","last_change":"2024-09-13T15:05:52.007594+0000","last_active":"2024-09-13T15:06:22.533497+0000","last_peered":"2024-09-13T15:06:22.533497+0000","last_clean":"2024-09-13T15:06:22.533497+0000","last_became_active":"2024-09-13T15:05:52.007146+0000","last_became_peered":"2024-09-13T15:05:52.007146+0000","last_unstale":"2024-09-13T15:06:22.533497+0000","last_undegraded":"2024-09-13T15:06:22.533497+0000","last_fullsized":"2024-09-13T15:06:22.533497+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:45.283696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"397'16204","reported_seq":16254,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.742595+0000","last_change":"2024-09-13T15:02:39.880337+0000","last_active":"2024-09-13T15:06:21.742595+0000","last_peered":"2024-09-13T15:06:21.742595+0000","last_clean":"2024-09-13T15:06:21.742595+0000","last_became_active":"2024-09-13T15:02:39.880044+0000","last_became_peered":"2024-09-13T15:02:39.880044+0000","last_unstale":"2024-09-13T15:06:21.742595+0000","last_undegraded":"2024-09-13T15:06:21.742595+0000","last_fullsized":"2024-09-13T15:06:21.742595+0000","mapping_epoch":355,"log_start":"373'9800","ondisk_log_start":"373'9800","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6404,"log_dups_size":0,"ondisk_log_size":6404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:38.582990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38273024,"num_objects":584,"num_object_clones":0,"num_object_copies":1752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16204,"num_write_kb":537216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":545,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443621+0000","last_change":"2024-09-13T14:55:58.568852+0000","last_active":"2024-09-13T15:06:19.443621+0000","last_peered":"2024-09-13T15:06:19.443621+0000","last_clean":"2024-09-13T15:06:19.443621+0000","last_became_active":"2024-09-13T14:55:51.169987+0000","last_became_peered":"2024-09-13T14:55:51.169987+0000","last_unstale":"2024-09-13T15:06:19.443621+0000","last_undegraded":"2024-09-13T15:06:19.443621+0000","last_fullsized":"2024-09-13T15:06:19.443621+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.568750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.568750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:56.273429+0000","scrub_duration":0.0074886340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":923,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.494895+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T15:06:19.494895+0000","last_peered":"2024-09-13T15:06:19.494895+0000","last_clean":"2024-09-13T15:06:19.494895+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T15:06:19.494895+0000","last_undegraded":"2024-09-13T15:06:19.494895+0000","last_fullsized":"2024-09-13T15:06:19.494895+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":38,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.476744+0000","last_change":"2024-09-13T15:05:52.011318+0000","last_active":"2024-09-13T15:06:21.476744+0000","last_peered":"2024-09-13T15:06:21.476744+0000","last_clean":"2024-09-13T15:06:21.476744+0000","last_became_active":"2024-09-13T15:05:52.011038+0000","last_became_peered":"2024-09-13T15:05:52.011038+0000","last_unstale":"2024-09-13T15:06:21.476744+0000","last_undegraded":"2024-09-13T15:06:21.476744+0000","last_fullsized":"2024-09-13T15:06:21.476744+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:35.033102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":181,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.448811+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T15:06:19.448811+0000","last_peered":"2024-09-13T15:06:19.448811+0000","last_clean":"2024-09-13T15:06:19.448811+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T15:06:19.448811+0000","last_undegraded":"2024-09-13T15:06:19.448811+0000","last_fullsized":"2024-09-13T15:06:19.448811+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"396'16168","reported_seq":16262,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.582386+0000","last_change":"2024-09-13T15:02:39.921581+0000","last_active":"2024-09-13T15:06:19.582386+0000","last_peered":"2024-09-13T15:06:19.582386+0000","last_clean":"2024-09-13T15:06:19.582386+0000","last_became_active":"2024-09-13T15:02:39.921286+0000","last_became_peered":"2024-09-13T15:02:39.921286+0000","last_unstale":"2024-09-13T15:06:19.582386+0000","last_undegraded":"2024-09-13T15:06:19.582386+0000","last_fullsized":"2024-09-13T15:06:19.582386+0000","mapping_epoch":355,"log_start":"373'10100","ondisk_log_start":"373'10100","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6068,"log_dups_size":0,"ondisk_log_size":6068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:54.996704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47972352,"num_objects":732,"num_object_clones":0,"num_object_copies":2196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16168,"num_write_kb":540800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":820,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.534655+0000","last_change":"2024-09-13T14:55:59.245841+0000","last_active":"2024-09-13T15:06:22.534655+0000","last_peered":"2024-09-13T15:06:22.534655+0000","last_clean":"2024-09-13T15:06:22.534655+0000","last_became_active":"2024-09-13T14:55:51.159487+0000","last_became_peered":"2024-09-13T14:55:51.159487+0000","last_unstale":"2024-09-13T15:06:22.534655+0000","last_undegraded":"2024-09-13T15:06:22.534655+0000","last_fullsized":"2024-09-13T15:06:22.534655+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.245213+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.245213+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:28.202646+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":608,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.442829+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T15:06:19.442829+0000","last_peered":"2024-09-13T15:06:19.442829+0000","last_clean":"2024-09-13T15:06:19.442829+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T15:06:19.442829+0000","last_undegraded":"2024-09-13T15:06:19.442829+0000","last_fullsized":"2024-09-13T15:06:19.442829+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":697,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.444453+0000","last_change":"2024-09-13T15:05:52.010114+0000","last_active":"2024-09-13T15:06:19.444453+0000","last_peered":"2024-09-13T15:06:19.444453+0000","last_clean":"2024-09-13T15:06:19.444453+0000","last_became_active":"2024-09-13T15:05:52.008634+0000","last_became_peered":"2024-09-13T15:05:52.008634+0000","last_unstale":"2024-09-13T15:06:19.444453+0000","last_undegraded":"2024-09-13T15:06:19.444453+0000","last_fullsized":"2024-09-13T15:06:19.444453+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:25.498296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":687,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.445490+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T15:06:19.445490+0000","last_peered":"2024-09-13T15:06:19.445490+0000","last_clean":"2024-09-13T15:06:19.445490+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T15:06:19.445490+0000","last_undegraded":"2024-09-13T15:06:19.445490+0000","last_fullsized":"2024-09-13T15:06:19.445490+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"397'16167","reported_seq":16217,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.700122+0000","last_change":"2024-09-13T15:02:39.921711+0000","last_active":"2024-09-13T15:06:21.700122+0000","last_peered":"2024-09-13T15:06:21.700122+0000","last_clean":"2024-09-13T15:06:21.700122+0000","last_became_active":"2024-09-13T15:02:39.921395+0000","last_became_peered":"2024-09-13T15:02:39.921395+0000","last_unstale":"2024-09-13T15:06:21.700122+0000","last_undegraded":"2024-09-13T15:06:21.700122+0000","last_fullsized":"2024-09-13T15:06:21.700122+0000","mapping_epoch":355,"log_start":"369'8400","ondisk_log_start":"369'8400","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":7767,"log_dups_size":0,"ondisk_log_size":7767,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:12.076600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38993920,"num_objects":595,"num_object_clones":0,"num_object_copies":1785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":595,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16167,"num_write_kb":536384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":174,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.442714+0000","last_change":"2024-09-13T14:55:59.574827+0000","last_active":"2024-09-13T15:06:19.442714+0000","last_peered":"2024-09-13T15:06:19.442714+0000","last_clean":"2024-09-13T15:06:19.442714+0000","last_became_active":"2024-09-13T14:55:51.162903+0000","last_became_peered":"2024-09-13T14:55:51.162903+0000","last_unstale":"2024-09-13T15:06:19.442714+0000","last_undegraded":"2024-09-13T15:06:19.442714+0000","last_fullsized":"2024-09-13T15:06:19.442714+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.574773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.574773+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:39.074072+0000","scrub_duration":0.001553708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":62,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.476788+0000","last_change":"2024-09-13T15:05:52.042632+0000","last_active":"2024-09-13T15:06:21.476788+0000","last_peered":"2024-09-13T15:06:21.476788+0000","last_clean":"2024-09-13T15:06:21.476788+0000","last_became_active":"2024-09-13T15:05:52.004324+0000","last_became_peered":"2024-09-13T15:05:52.004324+0000","last_unstale":"2024-09-13T15:06:21.476788+0000","last_undegraded":"2024-09-13T15:06:21.476788+0000","last_fullsized":"2024-09-13T15:06:21.476788+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:44.130757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":668,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.532691+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T15:06:22.532691+0000","last_peered":"2024-09-13T15:06:22.532691+0000","last_clean":"2024-09-13T15:06:22.532691+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T15:06:22.532691+0000","last_undegraded":"2024-09-13T15:06:22.532691+0000","last_fullsized":"2024-09-13T15:06:22.532691+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":686,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.533372+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T15:06:22.533372+0000","last_peered":"2024-09-13T15:06:22.533372+0000","last_clean":"2024-09-13T15:06:22.533372+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T15:06:22.533372+0000","last_undegraded":"2024-09-13T15:06:22.533372+0000","last_fullsized":"2024-09-13T15:06:22.533372+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"397'16396","reported_seq":16446,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.811532+0000","last_change":"2024-09-13T15:02:39.882940+0000","last_active":"2024-09-13T15:06:21.811532+0000","last_peered":"2024-09-13T15:06:21.811532+0000","last_clean":"2024-09-13T15:06:21.811532+0000","last_became_active":"2024-09-13T15:02:39.882641+0000","last_became_peered":"2024-09-13T15:02:39.882641+0000","last_unstale":"2024-09-13T15:06:21.811532+0000","last_undegraded":"2024-09-13T15:06:21.811532+0000","last_fullsized":"2024-09-13T15:06:21.811532+0000","mapping_epoch":355,"log_start":"373'10000","ondisk_log_start":"373'10000","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6396,"log_dups_size":0,"ondisk_log_size":6396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:10.481971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41025536,"num_objects":626,"num_object_clones":0,"num_object_copies":1878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16396,"num_write_kb":544704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":317,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.494568+0000","last_change":"2024-09-13T14:56:01.658240+0000","last_active":"2024-09-13T15:06:19.494568+0000","last_peered":"2024-09-13T15:06:19.494568+0000","last_clean":"2024-09-13T15:06:19.494568+0000","last_became_active":"2024-09-13T14:55:51.160480+0000","last_became_peered":"2024-09-13T14:55:51.160480+0000","last_unstale":"2024-09-13T15:06:19.494568+0000","last_undegraded":"2024-09-13T15:06:19.494568+0000","last_fullsized":"2024-09-13T15:06:19.494568+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.658143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.658143+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:19.262955+0000","scrub_duration":0.0038903079999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":1023,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.497085+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T15:06:19.497085+0000","last_peered":"2024-09-13T15:06:19.497085+0000","last_clean":"2024-09-13T15:06:19.497085+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T15:06:19.497085+0000","last_undegraded":"2024-09-13T15:06:19.497085+0000","last_fullsized":"2024-09-13T15:06:19.497085+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":382,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443644+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T15:06:19.443644+0000","last_peered":"2024-09-13T15:06:19.443644+0000","last_clean":"2024-09-13T15:06:19.443644+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T15:06:19.443644+0000","last_undegraded":"2024-09-13T15:06:19.443644+0000","last_fullsized":"2024-09-13T15:06:19.443644+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":700,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.532343+0000","last_change":"2024-09-13T15:05:52.008066+0000","last_active":"2024-09-13T15:06:22.532343+0000","last_peered":"2024-09-13T15:06:22.532343+0000","last_clean":"2024-09-13T15:06:22.532343+0000","last_became_active":"2024-09-13T15:05:52.007749+0000","last_became_peered":"2024-09-13T15:05:52.007749+0000","last_unstale":"2024-09-13T15:06:22.532343+0000","last_undegraded":"2024-09-13T15:06:22.532343+0000","last_fullsized":"2024-09-13T15:06:22.532343+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:11.201592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"397'16360","reported_seq":16410,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.655964+0000","last_change":"2024-09-13T15:02:39.882990+0000","last_active":"2024-09-13T15:06:21.655964+0000","last_peered":"2024-09-13T15:06:21.655964+0000","last_clean":"2024-09-13T15:06:21.655964+0000","last_became_active":"2024-09-13T15:02:39.882663+0000","last_became_peered":"2024-09-13T15:02:39.882663+0000","last_unstale":"2024-09-13T15:06:21.655964+0000","last_undegraded":"2024-09-13T15:06:21.655964+0000","last_fullsized":"2024-09-13T15:06:21.655964+0000","mapping_epoch":355,"log_start":"373'10100","ondisk_log_start":"373'10100","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6260,"log_dups_size":0,"ondisk_log_size":6260,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:20.368697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40239104,"num_objects":614,"num_object_clones":0,"num_object_copies":1842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16360,"num_write_kb":543168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":275,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.478666+0000","last_change":"2024-09-13T15:06:10.445608+0000","last_active":"2024-09-13T15:06:21.478666+0000","last_peered":"2024-09-13T15:06:21.478666+0000","last_clean":"2024-09-13T15:06:21.478666+0000","last_became_active":"2024-09-13T15:06:10.445259+0000","last_became_peered":"2024-09-13T15:06:10.445259+0000","last_unstale":"2024-09-13T15:06:21.478666+0000","last_undegraded":"2024-09-13T15:06:21.478666+0000","last_fullsized":"2024-09-13T15:06:21.478666+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:49.771572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:49.771572+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:41.953119+0000","scrub_duration":0.0035237329999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":398,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.494912+0000","last_change":"2024-09-13T15:05:52.014958+0000","last_active":"2024-09-13T15:06:19.494912+0000","last_peered":"2024-09-13T15:06:19.494912+0000","last_clean":"2024-09-13T15:06:19.494912+0000","last_became_active":"2024-09-13T15:05:52.014471+0000","last_became_peered":"2024-09-13T15:05:52.014471+0000","last_unstale":"2024-09-13T15:06:19.494912+0000","last_undegraded":"2024-09-13T15:06:19.494912+0000","last_fullsized":"2024-09-13T15:06:19.494912+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:38.514748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":1171,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.532580+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T15:06:22.532580+0000","last_peered":"2024-09-13T15:06:22.532580+0000","last_clean":"2024-09-13T15:06:22.532580+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T15:06:22.532580+0000","last_undegraded":"2024-09-13T15:06:22.532580+0000","last_fullsized":"2024-09-13T15:06:22.532580+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":643,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.532622+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T15:06:22.532622+0000","last_peered":"2024-09-13T15:06:22.532622+0000","last_clean":"2024-09-13T15:06:22.532622+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T15:06:22.532622+0000","last_undegraded":"2024-09-13T15:06:22.532622+0000","last_fullsized":"2024-09-13T15:06:22.532622+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"397'16495","reported_seq":16559,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.728925+0000","last_change":"2024-09-13T15:06:10.513775+0000","last_active":"2024-09-13T15:06:21.728925+0000","last_peered":"2024-09-13T15:06:21.728925+0000","last_clean":"2024-09-13T15:06:21.728925+0000","last_became_active":"2024-09-13T15:06:10.445216+0000","last_became_peered":"2024-09-13T15:06:10.445216+0000","last_unstale":"2024-09-13T15:06:21.728925+0000","last_undegraded":"2024-09-13T15:06:21.728925+0000","last_fullsized":"2024-09-13T15:06:21.728925+0000","mapping_epoch":389,"log_start":"373'10292","ondisk_log_start":"373'10292","created":355,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6203,"log_dups_size":0,"ondisk_log_size":6203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:59:24.815061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42598424,"num_objects":651,"num_object_clones":0,"num_object_copies":1953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":651,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":16495,"num_write_kb":548609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":420,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.444868+0000","last_change":"2024-09-13T15:06:18.431867+0000","last_active":"2024-09-13T15:06:19.444868+0000","last_peered":"2024-09-13T15:06:19.444868+0000","last_clean":"2024-09-13T15:06:19.444868+0000","last_became_active":"2024-09-13T15:06:10.446381+0000","last_became_peered":"2024-09-13T15:06:10.446381+0000","last_unstale":"2024-09-13T15:06:19.444868+0000","last_undegraded":"2024-09-13T15:06:19.444868+0000","last_fullsized":"2024-09-13T15:06:19.444868+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:51.748121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:51.748121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:06:21.271030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":668,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.532358+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T15:06:22.532358+0000","last_peered":"2024-09-13T15:06:22.532358+0000","last_clean":"2024-09-13T15:06:22.532358+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T15:06:22.532358+0000","last_undegraded":"2024-09-13T15:06:22.532358+0000","last_fullsized":"2024-09-13T15:06:22.532358+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":610,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.444074+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T15:06:19.444074+0000","last_peered":"2024-09-13T15:06:19.444074+0000","last_clean":"2024-09-13T15:06:19.444074+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T15:06:19.444074+0000","last_undegraded":"2024-09-13T15:06:19.444074+0000","last_fullsized":"2024-09-13T15:06:19.444074+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":651,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.445076+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T15:06:19.445076+0000","last_peered":"2024-09-13T15:06:19.445076+0000","last_clean":"2024-09-13T15:06:19.445076+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T15:06:19.445076+0000","last_undegraded":"2024-09-13T15:06:19.445076+0000","last_fullsized":"2024-09-13T15:06:19.445076+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"397'16610","reported_seq":16660,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.705023+0000","last_change":"2024-09-13T15:02:39.921655+0000","last_active":"2024-09-13T15:06:21.705023+0000","last_peered":"2024-09-13T15:06:21.705023+0000","last_clean":"2024-09-13T15:06:21.705023+0000","last_became_active":"2024-09-13T15:02:39.921331+0000","last_became_peered":"2024-09-13T15:02:39.921331+0000","last_unstale":"2024-09-13T15:06:21.705023+0000","last_undegraded":"2024-09-13T15:06:21.705023+0000","last_fullsized":"2024-09-13T15:06:21.705023+0000","mapping_epoch":355,"log_start":"374'10400","ondisk_log_start":"374'10400","created":355,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6210,"log_dups_size":0,"ondisk_log_size":6210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:48.004179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38928384,"num_objects":594,"num_object_clones":0,"num_object_copies":1782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16610,"num_write_kb":550528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":368,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443031+0000","last_change":"2024-09-13T15:02:52.361444+0000","last_active":"2024-09-13T15:06:19.443031+0000","last_peered":"2024-09-13T15:06:19.443031+0000","last_clean":"2024-09-13T15:06:19.443031+0000","last_became_active":"2024-09-13T14:59:53.220135+0000","last_became_peered":"2024-09-13T14:59:53.220135+0000","last_unstale":"2024-09-13T15:06:19.443031+0000","last_undegraded":"2024-09-13T15:06:19.443031+0000","last_fullsized":"2024-09-13T15:06:19.443031+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.361359+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.361359+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:35.003140+0000","scrub_duration":0.005845154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":42,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.478573+0000","last_change":"2024-09-13T15:05:52.014941+0000","last_active":"2024-09-13T15:06:21.478573+0000","last_peered":"2024-09-13T15:06:21.478573+0000","last_clean":"2024-09-13T15:06:21.478573+0000","last_became_active":"2024-09-13T15:05:52.002400+0000","last_became_peered":"2024-09-13T15:05:52.002400+0000","last_unstale":"2024-09-13T15:06:21.478573+0000","last_undegraded":"2024-09-13T15:06:21.478573+0000","last_fullsized":"2024-09-13T15:06:21.478573+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:06.227994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":1025,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.533218+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T15:06:22.533218+0000","last_peered":"2024-09-13T15:06:22.533218+0000","last_clean":"2024-09-13T15:06:22.533218+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T15:06:22.533218+0000","last_undegraded":"2024-09-13T15:06:22.533218+0000","last_fullsized":"2024-09-13T15:06:22.533218+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":38,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.476967+0000","last_change":"2024-09-13T15:05:52.003257+0000","last_active":"2024-09-13T15:06:21.476967+0000","last_peered":"2024-09-13T15:06:21.476967+0000","last_clean":"2024-09-13T15:06:21.476967+0000","last_became_active":"2024-09-13T15:05:52.002919+0000","last_became_peered":"2024-09-13T15:05:52.002919+0000","last_unstale":"2024-09-13T15:06:21.476967+0000","last_undegraded":"2024-09-13T15:06:21.476967+0000","last_fullsized":"2024-09-13T15:06:21.476967+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:04.833597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"397'16434","reported_seq":19177,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.782702+0000","last_change":"2024-09-13T15:06:11.474510+0000","last_active":"2024-09-13T15:06:21.782702+0000","last_peered":"2024-09-13T15:06:21.782702+0000","last_clean":"2024-09-13T15:06:21.782702+0000","last_became_active":"2024-09-13T15:06:11.443268+0000","last_became_peered":"2024-09-13T15:06:11.443268+0000","last_unstale":"2024-09-13T15:06:21.782702+0000","last_undegraded":"2024-09-13T15:06:21.782702+0000","last_fullsized":"2024-09-13T15:06:21.782702+0000","mapping_epoch":390,"log_start":"373'10000","ondisk_log_start":"373'10000","created":355,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:02:38.864513+0000","last_clean_scrub_stamp":"2024-09-13T15:02:38.864513+0000","objects_scrubbed":0,"log_size":6434,"log_dups_size":0,"ondisk_log_size":6434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:07:26.744798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40763392,"num_objects":622,"num_object_clones":0,"num_object_copies":1866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16434,"num_write_kb":545792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1332,"num_bytes_recovered":87228416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":38,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.477303+0000","last_change":"2024-09-13T15:05:52.005786+0000","last_active":"2024-09-13T15:06:21.477303+0000","last_peered":"2024-09-13T15:06:21.477303+0000","last_clean":"2024-09-13T15:06:21.477303+0000","last_became_active":"2024-09-13T15:05:52.005453+0000","last_became_peered":"2024-09-13T15:05:52.005453+0000","last_unstale":"2024-09-13T15:06:21.477303+0000","last_undegraded":"2024-09-13T15:06:21.477303+0000","last_fullsized":"2024-09-13T15:06:21.477303+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:54.193156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:54.193156+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:51.141832+0000","scrub_duration":0.0040069900000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":673,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.444963+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T15:06:19.444963+0000","last_peered":"2024-09-13T15:06:19.444963+0000","last_clean":"2024-09-13T15:06:19.444963+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T15:06:19.444963+0000","last_undegraded":"2024-09-13T15:06:19.444963+0000","last_fullsized":"2024-09-13T15:06:19.444963+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1210,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.449215+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T15:06:19.449215+0000","last_peered":"2024-09-13T15:06:19.449215+0000","last_clean":"2024-09-13T15:06:19.449215+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T15:06:19.449215+0000","last_undegraded":"2024-09-13T15:06:19.449215+0000","last_fullsized":"2024-09-13T15:06:19.449215+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":364,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.442814+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T15:06:19.442814+0000","last_peered":"2024-09-13T15:06:19.442814+0000","last_clean":"2024-09-13T15:06:19.442814+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T15:06:19.442814+0000","last_undegraded":"2024-09-13T15:06:19.442814+0000","last_fullsized":"2024-09-13T15:06:19.442814+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"82'100","reported_seq":136,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.533076+0000","last_change":"2024-09-13T15:05:52.014434+0000","last_active":"2024-09-13T15:06:22.533076+0000","last_peered":"2024-09-13T15:06:22.533076+0000","last_clean":"2024-09-13T15:06:22.533076+0000","last_became_active":"2024-09-13T15:05:52.014086+0000","last_became_peered":"2024-09-13T15:05:52.014086+0000","last_unstale":"2024-09-13T15:06:22.533076+0000","last_undegraded":"2024-09-13T15:06:22.533076+0000","last_fullsized":"2024-09-13T15:06:22.533076+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:51.192503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:51.192503+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:53.661130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":381,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443492+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T15:06:19.443492+0000","last_peered":"2024-09-13T15:06:19.443492+0000","last_clean":"2024-09-13T15:06:19.443492+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T15:06:19.443492+0000","last_undegraded":"2024-09-13T15:06:19.443492+0000","last_fullsized":"2024-09-13T15:06:19.443492+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":680,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.444877+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T15:06:19.444877+0000","last_peered":"2024-09-13T15:06:19.444877+0000","last_clean":"2024-09-13T15:06:19.444877+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T15:06:19.444877+0000","last_undegraded":"2024-09-13T15:06:19.444877+0000","last_fullsized":"2024-09-13T15:06:19.444877+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.17","version":"82'100","reported_seq":209,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.494964+0000","last_change":"2024-09-13T15:03:03.108031+0000","last_active":"2024-09-13T15:06:19.494964+0000","last_peered":"2024-09-13T15:06:19.494964+0000","last_clean":"2024-09-13T15:06:19.494964+0000","last_became_active":"2024-09-13T15:02:23.883373+0000","last_became_peered":"2024-09-13T15:02:23.883373+0000","last_unstale":"2024-09-13T15:06:19.494964+0000","last_undegraded":"2024-09-13T15:06:19.494964+0000","last_fullsized":"2024-09-13T15:06:19.494964+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:03.107786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:03.107786+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:17:10.041468+0000","scrub_duration":0.0049024139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":666,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.532386+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T15:06:22.532386+0000","last_peered":"2024-09-13T15:06:22.532386+0000","last_clean":"2024-09-13T15:06:22.532386+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T15:06:22.532386+0000","last_undegraded":"2024-09-13T15:06:22.532386+0000","last_fullsized":"2024-09-13T15:06:22.532386+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":403,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443767+0000","last_change":"2024-09-13T15:05:52.016761+0000","last_active":"2024-09-13T15:06:19.443767+0000","last_peered":"2024-09-13T15:06:19.443767+0000","last_clean":"2024-09-13T15:06:19.443767+0000","last_became_active":"2024-09-13T15:05:52.005623+0000","last_became_peered":"2024-09-13T15:05:52.005623+0000","last_unstale":"2024-09-13T15:06:19.443767+0000","last_undegraded":"2024-09-13T15:06:19.443767+0000","last_fullsized":"2024-09-13T15:06:19.443767+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:58:43.424615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":242,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-09-13T15:06:18.439701+0000","last_change":"2024-09-13T14:56:01.235655+0000","last_active":"2024-09-13T15:06:18.439701+0000","last_peered":"2024-09-13T15:06:18.439701+0000","last_clean":"2024-09-13T15:06:18.439701+0000","last_became_active":"2024-09-13T14:55:55.195703+0000","last_became_peered":"2024-09-13T14:55:55.195703+0000","last_unstale":"2024-09-13T15:06:18.439701+0000","last_undegraded":"2024-09-13T15:06:18.439701+0000","last_fullsized":"2024-09-13T15:06:18.439701+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.235497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.235497+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:36:33.861613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":361,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.495031+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T15:06:19.495031+0000","last_peered":"2024-09-13T15:06:19.495031+0000","last_clean":"2024-09-13T15:06:19.495031+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T15:06:19.495031+0000","last_undegraded":"2024-09-13T15:06:19.495031+0000","last_fullsized":"2024-09-13T15:06:19.495031+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":38,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.476936+0000","last_change":"2024-09-13T15:05:52.010150+0000","last_active":"2024-09-13T15:06:21.476936+0000","last_peered":"2024-09-13T15:06:21.476936+0000","last_clean":"2024-09-13T15:06:21.476936+0000","last_became_active":"2024-09-13T15:05:52.008977+0000","last_became_peered":"2024-09-13T15:05:52.008977+0000","last_unstale":"2024-09-13T15:06:21.476936+0000","last_undegraded":"2024-09-13T15:06:21.476936+0000","last_fullsized":"2024-09-13T15:06:21.476936+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:43.067814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"82'100","reported_seq":65,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443976+0000","last_change":"2024-09-13T15:02:55.870615+0000","last_active":"2024-09-13T15:06:19.443976+0000","last_peered":"2024-09-13T15:06:19.443976+0000","last_clean":"2024-09-13T15:06:19.443976+0000","last_became_active":"2024-09-13T15:02:23.883217+0000","last_became_peered":"2024-09-13T15:02:23.883217+0000","last_unstale":"2024-09-13T15:06:19.443976+0000","last_undegraded":"2024-09-13T15:06:19.443976+0000","last_fullsized":"2024-09-13T15:06:19.443976+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.870528+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.870528+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:10.023787+0000","scrub_duration":0.0047551850000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":673,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.445312+0000","last_change":"2024-09-13T15:05:52.006782+0000","last_active":"2024-09-13T15:06:19.445312+0000","last_peered":"2024-09-13T15:06:19.445312+0000","last_clean":"2024-09-13T15:06:19.445312+0000","last_became_active":"2024-09-13T15:05:52.006452+0000","last_became_peered":"2024-09-13T15:05:52.006452+0000","last_unstale":"2024-09-13T15:06:19.445312+0000","last_undegraded":"2024-09-13T15:06:19.445312+0000","last_fullsized":"2024-09-13T15:06:19.445312+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:37:23.855899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":38,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.476797+0000","last_change":"2024-09-13T15:05:51.999327+0000","last_active":"2024-09-13T15:06:21.476797+0000","last_peered":"2024-09-13T15:06:21.476797+0000","last_clean":"2024-09-13T15:06:21.476797+0000","last_became_active":"2024-09-13T15:05:51.994485+0000","last_became_peered":"2024-09-13T15:05:51.994485+0000","last_unstale":"2024-09-13T15:06:21.476797+0000","last_undegraded":"2024-09-13T15:06:21.476797+0000","last_fullsized":"2024-09-13T15:06:21.476797+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:55:49.041649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"82'100","reported_seq":65,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.442888+0000","last_change":"2024-09-13T15:02:56.319389+0000","last_active":"2024-09-13T15:06:19.442888+0000","last_peered":"2024-09-13T15:06:19.442888+0000","last_clean":"2024-09-13T15:06:19.442888+0000","last_became_active":"2024-09-13T15:02:23.876739+0000","last_became_peered":"2024-09-13T15:02:23.876739+0000","last_unstale":"2024-09-13T15:06:19.442888+0000","last_undegraded":"2024-09-13T15:06:19.442888+0000","last_fullsized":"2024-09-13T15:06:19.442888+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:56.319302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:56.319302+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:58.230158+0000","scrub_duration":0.0031182190000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":181,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.449170+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T15:06:19.449170+0000","last_peered":"2024-09-13T15:06:19.449170+0000","last_clean":"2024-09-13T15:06:19.449170+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T15:06:19.449170+0000","last_undegraded":"2024-09-13T15:06:19.449170+0000","last_fullsized":"2024-09-13T15:06:19.449170+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":38,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.476879+0000","last_change":"2024-09-13T15:05:52.003501+0000","last_active":"2024-09-13T15:06:21.476879+0000","last_peered":"2024-09-13T15:06:21.476879+0000","last_clean":"2024-09-13T15:06:21.476879+0000","last_became_active":"2024-09-13T15:05:52.003200+0000","last_became_peered":"2024-09-13T15:05:52.003200+0000","last_unstale":"2024-09-13T15:06:21.476879+0000","last_undegraded":"2024-09-13T15:06:21.476879+0000","last_fullsized":"2024-09-13T15:06:21.476879+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:46:07.106935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"82'100","reported_seq":647,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.494767+0000","last_change":"2024-09-13T15:03:07.166808+0000","last_active":"2024-09-13T15:06:19.494767+0000","last_peered":"2024-09-13T15:06:19.494767+0000","last_clean":"2024-09-13T15:06:19.494767+0000","last_became_active":"2024-09-13T15:02:23.883387+0000","last_became_peered":"2024-09-13T15:02:23.883387+0000","last_unstale":"2024-09-13T15:06:19.494767+0000","last_undegraded":"2024-09-13T15:06:19.494767+0000","last_fullsized":"2024-09-13T15:06:19.494767+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:07.166034+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:07.166034+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:32:42.605595+0000","scrub_duration":0.01094734,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":698,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.445166+0000","last_change":"2024-09-13T15:05:52.008086+0000","last_active":"2024-09-13T15:06:19.445166+0000","last_peered":"2024-09-13T15:06:19.445166+0000","last_clean":"2024-09-13T15:06:19.445166+0000","last_became_active":"2024-09-13T15:05:52.007347+0000","last_became_peered":"2024-09-13T15:05:52.007347+0000","last_unstale":"2024-09-13T15:06:19.445166+0000","last_undegraded":"2024-09-13T15:06:19.445166+0000","last_fullsized":"2024-09-13T15:06:19.445166+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:12:09.381403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":666,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.532228+0000","last_change":"2024-09-13T15:05:52.007634+0000","last_active":"2024-09-13T15:06:22.532228+0000","last_peered":"2024-09-13T15:06:22.532228+0000","last_clean":"2024-09-13T15:06:22.532228+0000","last_became_active":"2024-09-13T15:05:52.007302+0000","last_became_peered":"2024-09-13T15:05:52.007302+0000","last_unstale":"2024-09-13T15:06:22.532228+0000","last_undegraded":"2024-09-13T15:06:22.532228+0000","last_fullsized":"2024-09-13T15:06:22.532228+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:36.203235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"82'100","reported_seq":352,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.494671+0000","last_change":"2024-09-13T15:03:04.101898+0000","last_active":"2024-09-13T15:06:19.494671+0000","last_peered":"2024-09-13T15:06:19.494671+0000","last_clean":"2024-09-13T15:06:19.494671+0000","last_became_active":"2024-09-13T15:02:23.884406+0000","last_became_peered":"2024-09-13T15:02:23.884406+0000","last_unstale":"2024-09-13T15:06:19.494671+0000","last_undegraded":"2024-09-13T15:06:19.494671+0000","last_fullsized":"2024-09-13T15:06:19.494671+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:04.101802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:04.101802+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:34.459623+0000","scrub_duration":0.003875819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":38,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.477106+0000","last_change":"2024-09-13T15:05:52.001307+0000","last_active":"2024-09-13T15:06:21.477106+0000","last_peered":"2024-09-13T15:06:21.477106+0000","last_clean":"2024-09-13T15:06:21.477106+0000","last_became_active":"2024-09-13T15:05:52.001040+0000","last_became_peered":"2024-09-13T15:05:52.001040+0000","last_unstale":"2024-09-13T15:06:21.477106+0000","last_undegraded":"2024-09-13T15:06:21.477106+0000","last_fullsized":"2024-09-13T15:06:21.477106+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:49:48.758670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":380,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443833+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T15:06:19.443833+0000","last_peered":"2024-09-13T15:06:19.443833+0000","last_clean":"2024-09-13T15:06:19.443833+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T15:06:19.443833+0000","last_undegraded":"2024-09-13T15:06:19.443833+0000","last_fullsized":"2024-09-13T15:06:19.443833+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"82'100","reported_seq":38,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.477090+0000","last_change":"2024-09-13T15:05:52.002778+0000","last_active":"2024-09-13T15:06:21.477090+0000","last_peered":"2024-09-13T15:06:21.477090+0000","last_clean":"2024-09-13T15:06:21.477090+0000","last_became_active":"2024-09-13T15:05:52.002405+0000","last_became_peered":"2024-09-13T15:05:52.002405+0000","last_unstale":"2024-09-13T15:06:21.477090+0000","last_undegraded":"2024-09-13T15:06:21.477090+0000","last_fullsized":"2024-09-13T15:06:21.477090+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.429543+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.429543+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:04:50.457862+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":499,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.532773+0000","last_change":"2024-09-13T15:05:52.013949+0000","last_active":"2024-09-13T15:06:22.532773+0000","last_peered":"2024-09-13T15:06:22.532773+0000","last_clean":"2024-09-13T15:06:22.532773+0000","last_became_active":"2024-09-13T15:05:52.013611+0000","last_became_peered":"2024-09-13T15:05:52.013611+0000","last_unstale":"2024-09-13T15:06:22.532773+0000","last_undegraded":"2024-09-13T15:06:22.532773+0000","last_fullsized":"2024-09-13T15:06:22.532773+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:29.374445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":652,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.444965+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T15:06:19.444965+0000","last_peered":"2024-09-13T15:06:19.444965+0000","last_clean":"2024-09-13T15:06:19.444965+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T15:06:19.444965+0000","last_undegraded":"2024-09-13T15:06:19.444965+0000","last_fullsized":"2024-09-13T15:06:19.444965+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"82'100","reported_seq":46,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.477342+0000","last_change":"2024-09-13T15:05:52.535449+0000","last_active":"2024-09-13T15:06:21.477342+0000","last_peered":"2024-09-13T15:06:21.477342+0000","last_clean":"2024-09-13T15:06:21.477342+0000","last_became_active":"2024-09-13T15:05:51.991505+0000","last_became_peered":"2024-09-13T15:05:51.991505+0000","last_unstale":"2024-09-13T15:06:21.477342+0000","last_undegraded":"2024-09-13T15:06:21.477342+0000","last_fullsized":"2024-09-13T15:06:21.477342+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:05:52.535365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:05:52.535365+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:39:36.016324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":398,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443428+0000","last_change":"2024-09-13T15:05:51.998541+0000","last_active":"2024-09-13T15:06:19.443428+0000","last_peered":"2024-09-13T15:06:19.443428+0000","last_clean":"2024-09-13T15:06:19.443428+0000","last_became_active":"2024-09-13T15:05:51.998234+0000","last_became_peered":"2024-09-13T15:05:51.998234+0000","last_unstale":"2024-09-13T15:06:19.443428+0000","last_undegraded":"2024-09-13T15:06:19.443428+0000","last_fullsized":"2024-09-13T15:06:19.443428+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:25.705700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":363,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.494769+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T15:06:19.494769+0000","last_peered":"2024-09-13T15:06:19.494769+0000","last_clean":"2024-09-13T15:06:19.494769+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T15:06:19.494769+0000","last_undegraded":"2024-09-13T15:06:19.494769+0000","last_fullsized":"2024-09-13T15:06:19.494769+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":398,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.443884+0000","last_change":"2024-09-13T15:05:51.995355+0000","last_active":"2024-09-13T15:06:19.443884+0000","last_peered":"2024-09-13T15:06:19.443884+0000","last_clean":"2024-09-13T15:06:19.443884+0000","last_became_active":"2024-09-13T15:05:51.995038+0000","last_became_peered":"2024-09-13T15:05:51.995038+0000","last_unstale":"2024-09-13T15:06:19.443884+0000","last_undegraded":"2024-09-13T15:06:19.443884+0000","last_fullsized":"2024-09-13T15:06:19.443884+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:24:14.413713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":402,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.494887+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T15:06:19.494887+0000","last_peered":"2024-09-13T15:06:19.494887+0000","last_clean":"2024-09-13T15:06:19.494887+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T15:06:19.494887+0000","last_undegraded":"2024-09-13T15:06:19.494887+0000","last_fullsized":"2024-09-13T15:06:19.494887+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"82'100","reported_seq":55,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.442761+0000","last_change":"2024-09-13T15:02:23.879862+0000","last_active":"2024-09-13T15:06:19.442761+0000","last_peered":"2024-09-13T15:06:19.442761+0000","last_clean":"2024-09-13T15:06:19.442761+0000","last_became_active":"2024-09-13T15:02:23.879170+0000","last_became_peered":"2024-09-13T15:02:23.879170+0000","last_unstale":"2024-09-13T15:06:19.442761+0000","last_undegraded":"2024-09-13T15:06:19.442761+0000","last_fullsized":"2024-09-13T15:06:19.442761+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:06.945525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:06.945525+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:48.496118+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":38,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.476689+0000","last_change":"2024-09-13T15:05:52.004131+0000","last_active":"2024-09-13T15:06:21.476689+0000","last_peered":"2024-09-13T15:06:21.476689+0000","last_clean":"2024-09-13T15:06:21.476689+0000","last_became_active":"2024-09-13T15:05:52.003845+0000","last_became_peered":"2024-09-13T15:05:52.003845+0000","last_unstale":"2024-09-13T15:06:21.476689+0000","last_undegraded":"2024-09-13T15:06:21.476689+0000","last_fullsized":"2024-09-13T15:06:21.476689+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:01.268036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":38,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-09-13T15:06:21.477438+0000","last_change":"2024-09-13T15:05:51.995035+0000","last_active":"2024-09-13T15:06:21.477438+0000","last_peered":"2024-09-13T15:06:21.477438+0000","last_clean":"2024-09-13T15:06:21.477438+0000","last_became_active":"2024-09-13T15:05:51.994496+0000","last_became_peered":"2024-09-13T15:05:51.994496+0000","last_unstale":"2024-09-13T15:06:21.477438+0000","last_undegraded":"2024-09-13T15:06:21.477438+0000","last_fullsized":"2024-09-13T15:06:21.477438+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:11:51.103755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"82'100","reported_seq":138,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:06:22.532702+0000","last_change":"2024-09-13T15:05:52.013759+0000","last_active":"2024-09-13T15:06:22.532702+0000","last_peered":"2024-09-13T15:06:22.532702+0000","last_clean":"2024-09-13T15:06:22.532702+0000","last_became_active":"2024-09-13T15:05:52.013210+0000","last_became_peered":"2024-09-13T15:05:52.013210+0000","last_unstale":"2024-09-13T15:06:22.532702+0000","last_undegraded":"2024-09-13T15:06:22.532702+0000","last_fullsized":"2024-09-13T15:06:22.532702+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.149290+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.149290+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:54.818210+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":384,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.442346+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T15:06:19.442346+0000","last_peered":"2024-09-13T15:06:19.442346+0000","last_clean":"2024-09-13T15:06:19.442346+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T15:06:19.442346+0000","last_undegraded":"2024-09-13T15:06:19.442346+0000","last_fullsized":"2024-09-13T15:06:19.442346+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":704,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-09-13T15:06:19.445274+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T15:06:19.445274+0000","last_peered":"2024-09-13T15:06:19.445274+0000","last_clean":"2024-09-13T15:06:19.445274+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T15:06:19.445274+0000","last_undegraded":"2024-09-13T15:06:19.445274+0000","last_fullsized":"2024-09-13T15:06:19.445274+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":671612952,"num_objects":10249,"num_object_clones":0,"num_object_copies":30747,"num_objects_missing_on_primary":0,"num_objects_missing":1,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10249,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":261131,"num_write_kb":8684097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5994,"num_bytes_recovered":392626176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2307338240,"data_stored":2307326024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105639,"ondisk_log_size":105639,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":16262,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24498,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397989,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1122600,"kb_used_data":237344,"kb_used_omap":1,"kb_used_meta":885246,"kb_avail":92622552,"statfs":{"total":95995035648,"available":94845493248,"internally_reserved":0,"allocated":243040256,"data_stored":240272438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":906492248},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594218,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":606756,"kb_used_data":296856,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93138396,"statfs":{"total":95995035648,"available":95373717504,"internally_reserved":0,"allocated":303980544,"data_stored":301219970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790447,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1387308,"kb_used_data":214368,"kb_used_omap":1,"kb_used_meta":1172926,"kb_avail":92357844,"statfs":{"total":95995035648,"available":94574432256,"internally_reserved":0,"allocated":219512832,"data_stored":216749107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1201076568},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,1,0,1,7,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986676,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1330420,"kb_used_data":429028,"kb_used_omap":1,"kb_used_meta":901374,"kb_avail":92414732,"statfs":{"total":95995035648,"available":94632685568,"internally_reserved":0,"allocated":439324672,"data_stored":436556062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923007320},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":335,"seq":1438814044223,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":340556,"kb_used_data":219012,"kb_used_omap":10,"kb_used_meta":121525,"kb_avail":93404596,"statfs":{"total":95995035648,"available":95646306304,"internally_reserved":0,"allocated":224268288,"data_stored":221523526,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10273,"internal_metadata":124442591},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762650,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1407820,"kb_used_data":358528,"kb_used_omap":8,"kb_used_meta":1049271,"kb_avail":92337332,"statfs":{"total":95995035648,"available":94553427968,"internally_reserved":0,"allocated":367132672,"data_stored":364384450,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":1074453962},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926177,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1117236,"kb_used_data":112168,"kb_used_omap":7,"kb_used_meta":1005048,"kb_avail":92627916,"statfs":{"total":95995035648,"available":94850985984,"internally_reserved":0,"allocated":114860032,"data_stored":112095806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":1029169865},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122416,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1520084,"kb_used_data":496332,"kb_used_omap":7,"kb_used_meta":1023736,"kb_avail":92225068,"statfs":{"total":95995035648,"available":94438469632,"internally_reserved":0,"allocated":508243968,"data_stored":505487892,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":1048306255},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7266,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":496504832,"data_stored":496500760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":98172928,"data_stored":98172928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":357433344,"data_stored":357433344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":210370560,"data_stored":210370560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":433393664,"data_stored":433389592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":209125376,"data_stored":209125376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":285020160,"data_stored":285016088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":217317376,"data_stored":217317376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:06:26.701 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_5 pgp_num 16 2024-09-13T15:06:26.770 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:26.772 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:26 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2784921394' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:26.772 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:26 smithi096 ceph-mon[102928]: 1.6 scrub starts 2024-09-13T15:06:26.773 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:26 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2784921394' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:26.773 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:26 smithi096 ceph-mon[98901]: 1.6 scrub starts 2024-09-13T15:06:26.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.825 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:26.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:26.926 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:26.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.965 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:26.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:26.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:26 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2784921394' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:26.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:26 smithi204 ceph-mon[87071]: 1.6 scrub starts 2024-09-13T15:06:27.066 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:27.100 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:27.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:27.202 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:27.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.245 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:27.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:27.346 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:27.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.401 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:27.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:27.502 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:27.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.548 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:27.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:27.610 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:27 smithi096 ceph-mon[98901]: pgmap v1334: 119 pgs: 1 peering, 118 active+clean; 594 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 15 MiB/s, 242 objects/s recovering 2024-09-13T15:06:27.611 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:27 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:27.611 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:27 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2784921394' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:06:27.611 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:27 smithi096 ceph-mon[98901]: osdmap e399: 8 total, 8 up, 8 in 2024-09-13T15:06:27.611 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:27 smithi096 ceph-mon[102928]: pgmap v1334: 119 pgs: 1 peering, 118 active+clean; 594 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 15 MiB/s, 242 objects/s recovering 2024-09-13T15:06:27.611 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:27 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:27.611 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:27 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2784921394' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:06:27.611 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:27 smithi096 ceph-mon[102928]: osdmap e399: 8 total, 8 up, 8 in 2024-09-13T15:06:27.649 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:27.660 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:27.660 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T15:06:27.660 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [ 2024-09-13T15:06:27.660 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.660 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.36552.0:266866 10.b e399/355 10:d6b78635:::benchmark_data_smithi096_288650_object131174:head v 399'16525, mlcod=399'16521)", 2024-09-13T15:06:27.660 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:06:27.640579+0000", 2024-09-13T15:06:27.661 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.017079449999999999, 2024-09-13T15:06:27.661 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.017110276000000001, 2024-09-13T15:06:27.661 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:06:27.661 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:06:27.661 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T15:06:27.661 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:06:27.661 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.661 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:06:27.661 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.640579+0000", 2024-09-13T15:06:27.661 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:06:27.661 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.661 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.662 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:06:27.662 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.640579+0000", 2024-09-13T15:06:27.662 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:06:27.662 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.662 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.662 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:06:27.662 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.640580+0000", 2024-09-13T15:06:27.662 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.850000000000001e-07 2024-09-13T15:06:27.662 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.663 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.663 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:06:27.663 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.640588+0000", 2024-09-13T15:06:27.663 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.114e-06 2024-09-13T15:06:27.663 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.663 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.663 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:06:27.663 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.640601+0000", 2024-09-13T15:06:27.663 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.3455000000000001e-05 2024-09-13T15:06:27.663 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.663 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.663 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:06:27.663 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.640609+0000", 2024-09-13T15:06:27.664 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.3000000000000002e-06 2024-09-13T15:06:27.664 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.664 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.664 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:06:27.664 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.641432+0000", 2024-09-13T15:06:27.664 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00082258899999999998 2024-09-13T15:06:27.664 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.664 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.664 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:06:27.664 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.641450+0000", 2024-09-13T15:06:27.665 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7682000000000001e-05 2024-09-13T15:06:27.665 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:06:27.665 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:06:27.665 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:06:27.665 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.665 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.665 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.36552.0:266870 10.b e399/355 10:d6cfc3e8:::benchmark_data_smithi096_288650_object131178:head v 399'16526, mlcod=399'16521)", 2024-09-13T15:06:27.665 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:06:27.642062+0000", 2024-09-13T15:06:27.665 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.015597066999999999, 2024-09-13T15:06:27.665 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.015681941000000001, 2024-09-13T15:06:27.665 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:06:27.665 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:06:27.666 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T15:06:27.666 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:06:27.666 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.666 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:06:27.666 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.642062+0000", 2024-09-13T15:06:27.666 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:06:27.666 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.666 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.666 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:06:27.666 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.642062+0000", 2024-09-13T15:06:27.666 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:06:27.666 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.666 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.666 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:06:27.667 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.642063+0000", 2024-09-13T15:06:27.667 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 8.8100000000000001e-07 2024-09-13T15:06:27.667 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.667 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.667 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:06:27.667 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.642069+0000", 2024-09-13T15:06:27.667 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.1789999999999996e-06 2024-09-13T15:06:27.667 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.667 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.667 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:06:27.667 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.642079+0000", 2024-09-13T15:06:27.668 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.0030999999999999e-05 2024-09-13T15:06:27.668 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.668 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.668 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:06:27.668 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.642085+0000", 2024-09-13T15:06:27.668 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 6.5069999999999997e-06 2024-09-13T15:06:27.668 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.668 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.668 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:06:27.668 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.645063+0000", 2024-09-13T15:06:27.668 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0029775180000000002 2024-09-13T15:06:27.668 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.668 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.669 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:06:27.669 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.645082+0000", 2024-09-13T15:06:27.669 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.8816000000000001e-05 2024-09-13T15:06:27.669 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:06:27.669 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:06:27.669 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:06:27.669 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.669 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.669 INFO:teuthology.orchestra.run.smithi096.stdout: "description": "osd_repop(client.36552.0:266871 10.b e399/355 10:d102d865:::benchmark_data_smithi096_288650_object131179:head v 399'16527, mlcod=399'16521)", 2024-09-13T15:06:27.669 INFO:teuthology.orchestra.run.smithi096.stdout: "initiated_at": "2024-09-13T15:06:27.648573+0000", 2024-09-13T15:06:27.669 INFO:teuthology.orchestra.run.smithi096.stdout: "age": 0.0090855669999999993, 2024-09-13T15:06:27.669 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.0092189500000000001, 2024-09-13T15:06:27.670 INFO:teuthology.orchestra.run.smithi096.stdout: "continuous": false, 2024-09-13T15:06:27.670 INFO:teuthology.orchestra.run.smithi096.stdout: "type_data": { 2024-09-13T15:06:27.670 INFO:teuthology.orchestra.run.smithi096.stdout: "flag_point": "started", 2024-09-13T15:06:27.670 INFO:teuthology.orchestra.run.smithi096.stdout: "events": [ 2024-09-13T15:06:27.670 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.670 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "initiated", 2024-09-13T15:06:27.670 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.648573+0000", 2024-09-13T15:06:27.670 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:06:27.670 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.670 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.670 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "header_read", 2024-09-13T15:06:27.670 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.648573+0000", 2024-09-13T15:06:27.670 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0 2024-09-13T15:06:27.670 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.671 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.671 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "throttled", 2024-09-13T15:06:27.671 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.648574+0000", 2024-09-13T15:06:27.671 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.3099999999999996e-07 2024-09-13T15:06:27.671 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.671 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.671 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "all_read", 2024-09-13T15:06:27.671 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.648580+0000", 2024-09-13T15:06:27.671 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 5.2800000000000003e-06 2024-09-13T15:06:27.671 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.671 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.671 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "dispatched", 2024-09-13T15:06:27.671 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.648589+0000", 2024-09-13T15:06:27.671 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 9.1290000000000002e-06 2024-09-13T15:06:27.672 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.672 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.672 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "queued_for_pg", 2024-09-13T15:06:27.672 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.648596+0000", 2024-09-13T15:06:27.672 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 7.1709999999999996e-06 2024-09-13T15:06:27.672 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.672 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.672 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "reached_pg", 2024-09-13T15:06:27.672 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.648750+0000", 2024-09-13T15:06:27.673 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 0.00015454499999999999 2024-09-13T15:06:27.673 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2024-09-13T15:06:27.673 INFO:teuthology.orchestra.run.smithi096.stdout: { 2024-09-13T15:06:27.673 INFO:teuthology.orchestra.run.smithi096.stdout: "event": "started", 2024-09-13T15:06:27.673 INFO:teuthology.orchestra.run.smithi096.stdout: "time": "2024-09-13T15:06:27.648768+0000", 2024-09-13T15:06:27.673 INFO:teuthology.orchestra.run.smithi096.stdout: "duration": 1.7841000000000001e-05 2024-09-13T15:06:27.673 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:06:27.673 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-13T15:06:27.673 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:06:27.673 INFO:teuthology.orchestra.run.smithi096.stdout: } 2024-09-13T15:06:27.673 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2024-09-13T15:06:27.673 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 3 2024-09-13T15:06:27.673 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T15:06:27.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.679 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:27.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:27.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:27 smithi204 ceph-mon[87071]: pgmap v1334: 119 pgs: 1 peering, 118 active+clean; 594 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 15 MiB/s, 242 objects/s recovering 2024-09-13T15:06:27.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:27 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:27.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:27 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2784921394' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:06:27.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:27 smithi204 ceph-mon[87071]: osdmap e399: 8 total, 8 up, 8 in 2024-09-13T15:06:27.780 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:27.822 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:27.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:27.924 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:27.965 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:27.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:28.067 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:28.100 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:28.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:28.202 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:28.229 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:06:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:06:27] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.51.0" 2024-09-13T15:06:28.233 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:28.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:28.335 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:28.380 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:28.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:28.482 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:28.520 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:28.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:28.622 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:28.684 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:28.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:28.723 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T15:06:28.786 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:28.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.840 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:28.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:28.941 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:28.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.984 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:28.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:29.085 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:29.105 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:29.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.126 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:29.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:29.228 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:29.266 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:29.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:29.368 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:29.398 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:06:29.405 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:29.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:29.507 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:29.555 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:29.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:29.657 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:29.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.694 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:29.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:29.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:29 smithi204 ceph-mon[87071]: pgmap v1336: 119 pgs: 119 active+clean; 375 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 7.4 MiB/s, 118 objects/s recovering 2024-09-13T15:06:29.795 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:29.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:29 smithi096 ceph-mon[98901]: pgmap v1336: 119 pgs: 119 active+clean; 375 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 7.4 MiB/s, 118 objects/s recovering 2024-09-13T15:06:29.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:29 smithi096 ceph-mon[102928]: pgmap v1336: 119 pgs: 119 active+clean; 375 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 7.4 MiB/s, 118 objects/s recovering 2024-09-13T15:06:29.833 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:29.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:29.935 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:29.973 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:29.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:30.075 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:30.119 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:30.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:30.221 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:30.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.264 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:30.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:30.365 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:30.413 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:30.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:06:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:06:30.215Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:06:30.516 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:30.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.553 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:30.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:30.654 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:30.692 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:30.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:30.794 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:30.829 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:30.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:30.930 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:30.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:30 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:30.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:30 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:30.971 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:30.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:31.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:30 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:31.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:30 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:31.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:30 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:31.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:30 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:31.073 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:31.106 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:31.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:31.208 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:31.242 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:31.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:31.344 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:31.377 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:31.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:31.479 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:31.516 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:31.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:31.618 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:31.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.663 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:31.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:31.764 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:31.795 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:31.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:31.896 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:31.905 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Removed 135689 objects 2024-09-13T15:06:31.905 INFO:tasks.radosbench.radosbench.0.smithi096.stdout:Clean up completed and total clean up time :131.723 2024-09-13T15:06:31.919 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:31.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:31.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:31 smithi204 ceph-mon[87071]: pgmap v1337: 119 pgs: 119 active+clean; 295 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T15:06:31.995 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:31 smithi096 ceph-mon[98901]: pgmap v1337: 119 pgs: 119 active+clean; 295 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T15:06:31.995 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:31 smithi096 ceph-mon[102928]: pgmap v1337: 119 pgs: 119 active+clean; 295 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T15:06:32.021 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:32.029 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:32.029 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:32.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.053 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:32.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:32.154 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:32.176 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:32.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:32.278 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:32.336 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:32.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:32.438 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:32.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.461 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:32.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:32.562 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:32.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.586 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:32.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:32.687 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:32.710 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:32.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:32.812 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:32.819 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:32 smithi096 ceph-mon[98901]: 1.6 scrub starts 2024-09-13T15:06:32.819 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:32 smithi096 ceph-mon[98901]: 1.6 scrub ok 2024-09-13T15:06:32.819 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:32 smithi096 ceph-mon[102928]: 1.6 scrub starts 2024-09-13T15:06:32.819 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:32 smithi096 ceph-mon[102928]: 1.6 scrub ok 2024-09-13T15:06:32.833 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:32.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:32.935 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:32.957 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:32.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:32.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:32 smithi204 ceph-mon[87071]: 1.6 scrub starts 2024-09-13T15:06:32.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:32 smithi204 ceph-mon[87071]: 1.6 scrub ok 2024-09-13T15:06:33.060 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:33.083 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:33.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:33.185 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:33.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.209 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:33.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:33.311 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:33.334 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:33.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:33.436 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:33.458 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:33.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:33.560 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:33.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.582 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:33.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:33.642 INFO:teuthology.orchestra.run.smithi096.stderr:set pool 10 pgp_num to 16 2024-09-13T15:06:33.684 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:33.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.710 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:33.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:33.811 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:33 smithi096 ceph-mon[98901]: pgmap v1338: 119 pgs: 119 active+clean; 47 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T15:06:33.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:33 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3431774669' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T15:06:33.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:33 smithi096 ceph-mon[102928]: pgmap v1338: 119 pgs: 119 active+clean; 47 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T15:06:33.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:33 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3431774669' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T15:06:33.833 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:33.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:33.934 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:33.959 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:33.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:33.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:33 smithi204 ceph-mon[87071]: pgmap v1338: 119 pgs: 119 active+clean; 47 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T15:06:33.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:33 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3431774669' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T15:06:34.060 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:34.081 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:34.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:34.156 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:06:34.156 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:06:34.173 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:06:34.173 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:06:34.184 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:34.192 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:06:34.192 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:06:34.204 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:34.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:34.208 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:06:34.209 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:06:34.224 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:06:34.225 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:06:34.240 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:06:34.241 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:06:34.256 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:06:34.257 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:06:34.273 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:06:34.273 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:06:34.306 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:34.332 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:34.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:34.434 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:34.458 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:34.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:34.560 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:34.584 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:34.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:34.686 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:34.711 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:34.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:34.813 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:34.837 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:34.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:34.939 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:34.965 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:34.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:34.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:34 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3431774669' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T15:06:34.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:34 smithi204 ceph-mon[87071]: osdmap e400: 8 total, 8 up, 8 in 2024-09-13T15:06:34.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:34 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4186878186' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:35.067 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:35.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:34 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3431774669' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T15:06:35.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:34 smithi096 ceph-mon[98901]: osdmap e400: 8 total, 8 up, 8 in 2024-09-13T15:06:35.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:34 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4186878186' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:35.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:34 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3431774669' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T15:06:35.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:34 smithi096 ceph-mon[102928]: osdmap e400: 8 total, 8 up, 8 in 2024-09-13T15:06:35.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:34 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4186878186' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:35.093 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:35.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:35.194 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:35.220 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:35.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:35.322 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:35.347 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:35.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:35.449 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:35.475 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:35.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:35.577 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:35.602 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:35.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:35.704 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:35.730 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:35.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:35.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:35 smithi096 ceph-mon[98901]: pgmap v1340: 119 pgs: 119 active+clean; 35 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T15:06:35.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:35 smithi096 ceph-mon[102928]: pgmap v1340: 119 pgs: 119 active+clean; 35 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T15:06:35.833 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:35.859 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:35.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:35.961 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:35.968 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:35 smithi204 ceph-mon[87071]: pgmap v1340: 119 pgs: 119 active+clean; 35 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T15:06:35.987 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:35.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:36.089 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:36.114 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:36.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:36.216 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:36.239 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:36.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:36.341 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:36.366 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:36.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:36.468 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:36.490 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:36.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:36.592 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:36.618 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:36.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:36.719 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:36.745 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:36.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:36.847 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:36.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.869 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:36.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:36.935 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:36.935 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:36.935 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:36.935 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:36.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:36.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:36.970 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:36.991 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:36.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:37.093 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:37.114 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:37.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:37.216 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:37.237 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:37.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:37.339 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:37.360 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:37.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:37.462 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:37.483 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:37.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:37.584 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:37.589 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2024-09-13T15:06:37.589 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2024-09-13T15:06:37.611 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:37.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:37.682 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:37 smithi096 ceph-mon[98901]: pgmap v1341: 119 pgs: 119 active+clean; 5.4 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 967 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T15:06:37.682 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:37 smithi096 ceph-mon[102928]: pgmap v1341: 119 pgs: 119 active+clean; 5.4 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 967 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T15:06:37.712 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:37.734 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:37.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:37.836 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:37.862 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:37.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:37.964 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:37.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:37 smithi204 ceph-mon[87071]: pgmap v1341: 119 pgs: 119 active+clean; 5.4 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 967 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T15:06:37.989 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:37.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:38.091 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:38.113 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:38.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:38.215 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:38.241 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:38.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:38.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:06:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:06:37] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.51.0" 2024-09-13T15:06:38.343 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:38.368 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:38.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:38.470 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:38.492 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:38.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:38.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:38.620 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:38.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:38.661 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T15:06:38.722 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:38.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.744 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:38.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:38.779 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:38 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/942146443' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:38.779 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:38 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/942146443' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:38.781 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:06:38.781 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:06:38.846 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:38.868 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:38.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:38.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:38 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/942146443' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:38.969 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:38.995 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:38.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:39.097 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:39.124 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:39.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:39.225 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:39.247 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:39.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:39.349 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:39.375 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:39.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:39.477 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:39.499 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:39.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:39.501 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T15:06:39.601 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:39.622 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:39.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:39.724 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:39.725 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:39.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.752 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:39.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:39.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:39 smithi096 ceph-mon[98901]: pgmap v1342: 119 pgs: 119 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 593 op/s 2024-09-13T15:06:39.900 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:39 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:06:39.900 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:39 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T15:06:39.901 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:39 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/942146443' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:06:39.901 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:39 smithi096 ceph-mon[98901]: osdmap e401: 8 total, 8 up, 8 in 2024-09-13T15:06:39.901 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:39 smithi096 ceph-mon[102928]: pgmap v1342: 119 pgs: 119 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 593 op/s 2024-09-13T15:06:39.902 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:39 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:06:39.902 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:39 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T15:06:39.902 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:39 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/942146443' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:06:39.902 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:39 smithi096 ceph-mon[102928]: osdmap e401: 8 total, 8 up, 8 in 2024-09-13T15:06:39.903 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:39.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.926 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:39.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:39.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:39 smithi204 ceph-mon[87071]: pgmap v1342: 119 pgs: 119 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 593 op/s 2024-09-13T15:06:39.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:39 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:06:39.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:39 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T15:06:39.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:39 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/942146443' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:06:39.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:39 smithi204 ceph-mon[87071]: osdmap e401: 8 total, 8 up, 8 in 2024-09-13T15:06:40.028 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:40.049 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:40.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:40.151 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:40.172 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:40.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:40.274 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:40.296 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:40.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:40.397 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:40.419 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:40.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:06:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:06:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:06:40.519 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:06:40.520 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:40.542 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:40.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:40.644 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:40.666 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:40.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:40.767 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:40.794 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:40.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:40.896 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:40.918 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:40.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:40.943 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:41.020 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:41.041 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:41.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:41.143 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:41.169 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:41.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:41.272 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:41.293 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:41.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:41.395 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:41.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.416 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:41.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:41.517 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:41.539 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:41.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:41.640 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:41.662 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:41.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:41.669 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:41.764 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:41.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.785 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:41.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:41.886 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:41.908 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:41.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:41.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:41 smithi204 ceph-mon[87071]: pgmap v1344: 119 pgs: 119 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 85 op/s 2024-09-13T15:06:42.010 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:42.031 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:42.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:42.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:41 smithi096 ceph-mon[98901]: pgmap v1344: 119 pgs: 119 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 85 op/s 2024-09-13T15:06:42.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:41 smithi096 ceph-mon[102928]: pgmap v1344: 119 pgs: 119 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 85 op/s 2024-09-13T15:06:42.134 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:42.160 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:42.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:42.245 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:06:42.262 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:42.310 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:42.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:42.412 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:42.438 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:42.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:42.540 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:42.566 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:42.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:42.590 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 6, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:06:42.590 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:06:42.590 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-09-13T15:06:42.590 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T15:06:42.590 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T15:06:42.591 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-09-13T15:06:42.667 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:42.690 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:42.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:42.792 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:42.814 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:42.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:42.867 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:06:42.917 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:42.939 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:42.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:43.040 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:43.061 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:43.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:43.163 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:43.185 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:43.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:43.287 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:43.309 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:43.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:43.410 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:43.433 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:43.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:43.535 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:43.557 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:43.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:43.658 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:43.680 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:43.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:43.782 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:43.804 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:43.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:43.906 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:43.928 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:43.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:43.933 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:43 smithi096 ceph-mon[98901]: pgmap v1345: 119 pgs: 119 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 79 op/s 2024-09-13T15:06:43.933 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:43 smithi096 ceph-mon[102928]: pgmap v1345: 119 pgs: 119 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 79 op/s 2024-09-13T15:06:43.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:43 smithi204 ceph-mon[87071]: pgmap v1345: 119 pgs: 119 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 79 op/s 2024-09-13T15:06:44.030 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:44.052 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:44.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:44.154 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:44.175 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:44.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:44.244 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:06:44.245 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:06:44.277 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:44.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.299 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:44.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:44.400 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:44.421 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:44.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:44.523 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:44.545 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:44.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:44.649 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:44.666 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'unique_pool_5' removed 2024-09-13T15:06:44.671 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:44.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:44.772 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:44.784 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-09-13T15:06:44.784 INFO:teuthology.orchestra.run.smithi204.stdout: "success": "" 2024-09-13T15:06:44.784 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-09-13T15:06:44.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.795 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:44.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:44.896 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:44.922 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:44.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:44.942 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:44 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2499893530' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:06:44.942 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:44 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2499893530' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:06:44.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:44 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2499893530' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:06:45.024 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:45.047 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:45.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:45.148 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:45.170 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:45.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:45.272 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:45.294 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:45.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:45.396 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:45.418 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:45.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:45.520 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:45.541 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:45.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:45.643 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:45.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.666 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:45.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:45.714 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-13T15:06:45.714 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-09-13T15:06:45.714 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-09-13T15:06:45.715 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-13T15:06:45.715 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-13T15:06:45.716 DEBUG:teuthology.orchestra.run.smithi096:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-13T15:06:45.718 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:45 smithi204 ceph-mon[87071]: pgmap v1346: 119 pgs: 119 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 48 op/s 2024-09-13T15:06:45.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:45 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2499893530' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:06:45.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:45 smithi204 ceph-mon[87071]: osdmap e402: 8 total, 8 up, 8 in 2024-09-13T15:06:45.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.746 INFO:teuthology.orchestra.run.smithi096.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-13T15:06:45.747 DEBUG:teuthology.orchestra.run.smithi096:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T15:06:45.767 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:45.789 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:45.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:45.819 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-13T15:06:45.819 DEBUG:teuthology.orchestra.run.smithi096:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-13T15:06:45.875 INFO:tasks.workunit:timeout=3h 2024-09-13T15:06:45.875 INFO:tasks.workunit:cleanup=True 2024-09-13T15:06:45.876 DEBUG:teuthology.orchestra.run.smithi096:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-13T15:06:45.890 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:45.916 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:45.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:45.958 INFO:tasks.workunit.client.0.smithi096.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-13T15:06:46.018 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:46.039 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:46.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:46.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:45 smithi096 ceph-mon[102928]: pgmap v1346: 119 pgs: 119 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 48 op/s 2024-09-13T15:06:46.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:45 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2499893530' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:06:46.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:45 smithi096 ceph-mon[102928]: osdmap e402: 8 total, 8 up, 8 in 2024-09-13T15:06:46.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:45 smithi096 ceph-mon[98901]: pgmap v1346: 119 pgs: 119 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 48 op/s 2024-09-13T15:06:46.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:45 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2499893530' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:06:46.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:45 smithi096 ceph-mon[98901]: osdmap e402: 8 total, 8 up, 8 in 2024-09-13T15:06:46.141 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:46.163 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:46.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:46.265 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:46.286 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:46.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:46.387 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:46.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.410 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:46.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:46.511 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:46.532 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:46.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:46.553 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:46.634 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:46.656 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:46.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:46.758 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:46.779 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:46.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:46.881 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:46.902 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:46.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:47.004 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:47.025 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:47.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:47.127 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:47.148 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:47.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:47.250 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:47.272 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:47.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:47.374 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:47.398 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:47.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:47.499 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:47.521 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:47.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:47.623 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:47.645 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:47.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:47.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:47 smithi204 ceph-mon[87071]: pgmap v1348: 103 pgs: 103 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:06:47.746 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:47.769 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:47.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:47.870 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:47.892 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:47.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:47.919 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:47 smithi096 ceph-mon[98901]: pgmap v1348: 103 pgs: 103 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:06:47.919 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:47 smithi096 ceph-mon[102928]: pgmap v1348: 103 pgs: 103 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:06:47.994 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:48.017 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:48.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:48.119 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:48.141 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:48.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:48.243 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:48.264 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:48.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:48.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:06:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:06:47] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.51.0" 2024-09-13T15:06:48.366 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:48.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.389 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:48.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:48.490 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:48.511 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:48.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:48.613 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:48.636 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:48.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:48.738 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:48.761 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:48.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:48.849 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T15:06:48.863 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:48.886 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:48.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:48.988 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:49.010 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:49.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:49.111 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:49.134 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:49.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:49.236 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:49.260 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:49.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:49.328 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:06:49.329 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:06:49.337 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:06:49.338 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:06:49.354 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:06:49.354 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:06:49.361 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:49.370 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:06:49.370 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:06:49.383 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:49.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:49.386 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:06:49.386 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:06:49.403 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:06:49.403 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:06:49.419 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:06:49.419 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:06:49.434 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:06:49.435 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:06:49.485 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:49.508 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:49.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:49.609 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:49.631 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:49.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:49.733 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:49.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.754 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:49.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:49.808 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:49 smithi096 ceph-mon[102928]: pgmap v1349: 103 pgs: 103 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:06:49.808 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:49 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1121274430' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:49.809 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:49 smithi096 ceph-mon[98901]: pgmap v1349: 103 pgs: 103 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:06:49.809 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:49 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1121274430' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:49.855 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:49.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.881 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:49.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:49.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:49 smithi204 ceph-mon[87071]: pgmap v1349: 103 pgs: 103 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:06:49.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:49 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1121274430' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:49.982 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:50.004 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:50.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:50.106 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:50.128 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:50.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:50.230 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:50.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.253 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:50.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:50.354 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:50.376 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:50.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:50.454 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 6, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:06:50.454 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:06:50.454 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T15:06:50.454 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:06:50.465 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:06:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:06:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:06:50.478 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:50.502 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:50.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:50.605 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:50.625 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:50.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:50.727 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:50.748 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:50.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:50.850 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:50.872 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:50.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:50.974 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:50.995 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:50.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:51.097 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:51.117 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:51.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:51.219 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:51.245 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:51.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:51.263 DEBUG:teuthology.orchestra.run.smithi096:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-13T15:06:51.286 INFO:tasks.workunit.client.0.smithi096.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-13T15:06:51.289 INFO:tasks.workunit.client.0.smithi096.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T15:06:51.289 INFO:tasks.workunit.client.0.smithi096.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-13T15:06:51.346 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:51.371 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:51.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:51.472 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:51.497 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:51.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:51.574 INFO:tasks.workunit.client.0.smithi096.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-13T15:06:51.598 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:51.622 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:51.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:51.624 INFO:tasks.workunit.client.0.smithi096.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-13T15:06:51.664 INFO:tasks.workunit.client.0.smithi096.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T15:06:51.666 INFO:tasks.workunit.client.0.smithi096.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T15:06:51.666 INFO:tasks.workunit.client.0.smithi096.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-13T15:06:51.709 INFO:tasks.workunit.client.0.smithi096.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T15:06:51.724 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:51.735 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-13T15:06:51.735 DEBUG:teuthology.orchestra.run.smithi096:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-13T15:06:51.747 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:51.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:51.820 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-09-13T15:06:51.821 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-09-13T15:06:51.821 DEBUG:teuthology.orchestra.run.smithi096:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-09-13T15:06:51.849 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:51.873 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:51.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:51.975 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:51.997 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:51.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:52.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:51 smithi096 ceph-mon[98901]: pgmap v1350: 103 pgs: 103 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:06:52.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:52.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:51 smithi096 ceph-mon[102928]: pgmap v1350: 103 pgs: 103 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:06:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:52.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:52.091 INFO:tasks.workunit.client.0.smithi096.stdout:Running main() from gmock_main.cc 2024-09-13T15:06:52.091 INFO:tasks.workunit.client.0.smithi096.stdout:[==========] Running 70 tests from 1 test suite. 2024-09-13T15:06:52.091 INFO:tasks.workunit.client.0.smithi096.stdout:[----------] Global test environment set-up. 2024-09-13T15:06:52.091 INFO:tasks.workunit.client.0.smithi096.stdout:[----------] 70 tests from TestClsRbd 2024-09-13T15:06:52.099 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:52.120 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:52.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:52.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:51 smithi204 ceph-mon[87071]: pgmap v1350: 103 pgs: 103 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:06:52.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:52.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:52.223 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:52.245 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:52.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:52.347 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:52.368 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:52.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:52.470 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:52.491 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:52.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:52.592 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:52.614 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:52.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:52.715 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:52.736 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:52.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:52.838 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:52.859 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:52.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:52.961 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:52.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.983 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:52.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:53.084 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:53.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.107 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:53.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:53.208 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:53.233 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:53.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:53.335 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:53.359 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:53.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:53.461 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:53.484 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:53.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:53.586 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:53.609 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:53.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:53.711 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:53.735 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:53.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:53.837 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:53.859 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:53.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:53.879 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is set 2024-09-13T15:06:53.888 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.get_all_features 2024-09-13T15:06:53.931 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.get_all_features (42 ms) 2024-09-13T15:06:53.931 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.copyup 2024-09-13T15:06:53.961 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:53.981 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:53.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:54.026 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:53 smithi096 ceph-mon[98901]: pgmap v1351: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:06:54.026 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:53 smithi096 ceph-mon[98901]: osdmap e403: 8 total, 8 up, 8 in 2024-09-13T15:06:54.027 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:53 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/365939274' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi096-315628-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:06:54.027 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:53 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi096-315628-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:06:54.027 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:53 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/960148578' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:06:54.027 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:53 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:06:54.027 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:53 smithi096 ceph-mon[102928]: pgmap v1351: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:06:54.027 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:53 smithi096 ceph-mon[102928]: osdmap e403: 8 total, 8 up, 8 in 2024-09-13T15:06:54.027 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:53 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/365939274' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi096-315628-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:06:54.027 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:53 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi096-315628-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:06:54.028 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:53 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/960148578' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:06:54.028 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:53 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:06:54.083 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:54.104 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:54.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:54.126 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.copyup (196 ms) 2024-09-13T15:06:54.126 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-09-13T15:06:54.134 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-13T15:06:54.135 INFO:teuthology.orchestra.run.smithi096.stdout: "ops": [], 2024-09-13T15:06:54.135 INFO:teuthology.orchestra.run.smithi096.stdout: "num_ops": 0 2024-09-13T15:06:54.135 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-13T15:06:54.144 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.sparse_copyup (18 ms) 2024-09-13T15:06:54.144 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-09-13T15:06:54.151 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.get_and_set_id (6 ms) 2024-09-13T15:06:54.151 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-09-13T15:06:54.162 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.add_remove_child (11 ms) 2024-09-13T15:06:54.162 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.directory_methods 2024-09-13T15:06:54.187 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.directory_methods (25 ms) 2024-09-13T15:06:54.187 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.create 2024-09-13T15:06:54.200 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.create (13 ms) 2024-09-13T15:06:54.200 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.get_features 2024-09-13T15:06:54.201 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2024-09-13T15:06:54.201 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-09-13T15:06:54.203 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2024-09-13T15:06:54.203 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-09-13T15:06:54.204 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.get_create_timestamp (1 ms) 2024-09-13T15:06:54.205 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-09-13T15:06:54.206 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:54.207 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2024-09-13T15:06:54.207 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-09-13T15:06:54.207 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-09-13T15:06:54.207 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-09-13T15:06:54.210 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2024-09-13T15:06:54.211 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.get_size 2024-09-13T15:06:54.215 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2024-09-13T15:06:54.215 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.set_size 2024-09-13T15:06:54.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:53 smithi204 ceph-mon[87071]: pgmap v1351: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:06:54.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:53 smithi204 ceph-mon[87071]: osdmap e403: 8 total, 8 up, 8 in 2024-09-13T15:06:54.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:53 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/365939274' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi096-315628-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:06:54.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:53 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi096-315628-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:06:54.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:53 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/960148578' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:06:54.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:53 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:06:54.220 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2024-09-13T15:06:54.220 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.protection_status 2024-09-13T15:06:54.229 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:54.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:54.236 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.protection_status (16 ms) 2024-09-13T15:06:54.236 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-09-13T15:06:54.242 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2024-09-13T15:06:54.242 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.parents_v1 2024-09-13T15:06:54.292 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.parents_v1 (50 ms) 2024-09-13T15:06:54.292 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.parents_v2 2024-09-13T15:06:54.321 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.parents_v2 (29 ms) 2024-09-13T15:06:54.321 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.snapshots 2024-09-13T15:06:54.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:54.341 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.snapshots (19 ms) 2024-09-13T15:06:54.341 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.snapid_race 2024-09-13T15:06:54.345 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2024-09-13T15:06:54.345 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.stripingv2 2024-09-13T15:06:54.354 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.stripingv2 (8 ms) 2024-09-13T15:06:54.354 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.object_map_save 2024-09-13T15:06:54.354 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:54.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:54.356 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.object_map_save (3 ms) 2024-09-13T15:06:54.356 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-09-13T15:06:54.360 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.object_map_resize (4 ms) 2024-09-13T15:06:54.360 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.object_map_update 2024-09-13T15:06:54.366 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.object_map_update (5 ms) 2024-09-13T15:06:54.366 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-09-13T15:06:54.366 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-09-13T15:06:54.366 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-09-13T15:06:54.371 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.object_map_snap_add (6 ms) 2024-09-13T15:06:54.371 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-09-13T15:06:54.376 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.object_map_snap_remove (5 ms) 2024-09-13T15:06:54.377 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.flags 2024-09-13T15:06:54.381 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.flags (4 ms) 2024-09-13T15:06:54.381 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.metadata 2024-09-13T15:06:54.406 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.metadata (26 ms) 2024-09-13T15:06:54.406 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.set_features 2024-09-13T15:06:54.413 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2024-09-13T15:06:54.413 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.mirror 2024-09-13T15:06:54.439 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:06:54.455 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.mirror (43 ms) 2024-09-13T15:06:54.455 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.mirror_image 2024-09-13T15:06:54.456 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:54.471 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.mirror_image (15 ms) 2024-09-13T15:06:54.471 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-09-13T15:06:54.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.478 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:54.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:54.580 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:54.606 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:54.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:54.708 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:54.729 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:54.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:54.831 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:54.857 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:54.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:54.959 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:54.985 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:54.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:55.086 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:55.111 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:55.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:55.213 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:54 smithi204 ceph-mon[87071]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:54 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi096-315628-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T15:06:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:54 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:06:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:54 smithi204 ceph-mon[87071]: osdmap e404: 8 total, 8 up, 8 in 2024-09-13T15:06:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:54 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:54 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:55.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:54 smithi204 ceph-mon[87071]: osdmap e405: 8 total, 8 up, 8 in 2024-09-13T15:06:55.238 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:55.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:54 smithi096 ceph-mon[98901]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:54 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi096-315628-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T15:06:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:54 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:06:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:54 smithi096 ceph-mon[98901]: osdmap e404: 8 total, 8 up, 8 in 2024-09-13T15:06:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:54 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:54 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:55.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:54 smithi096 ceph-mon[98901]: osdmap e405: 8 total, 8 up, 8 in 2024-09-13T15:06:55.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:54 smithi096 ceph-mon[102928]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:54 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi096-315628-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T15:06:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:54 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:06:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:54 smithi096 ceph-mon[102928]: osdmap e404: 8 total, 8 up, 8 in 2024-09-13T15:06:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:54 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:54 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:55.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:54 smithi096 ceph-mon[102928]: osdmap e405: 8 total, 8 up, 8 in 2024-09-13T15:06:55.340 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:55.364 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:55.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:55.465 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:55.490 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:55.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:55.543 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:55.592 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:55.618 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:55.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:55.720 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:55.743 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:55.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:55.845 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:55.869 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:55.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:55.971 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:55.995 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:55.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:56.056 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T15:06:56.065 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:56.097 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:56.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.123 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:56.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:56.192 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:55 smithi096 ceph-mon[98901]: pgmap v1354: 135 pgs: 29 unknown, 106 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:06:56.192 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:55 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:06:56.193 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:55 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:06:56.193 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:55 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:56.193 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:55 smithi096 ceph-mon[102928]: pgmap v1354: 135 pgs: 29 unknown, 106 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:06:56.193 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:55 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:06:56.193 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:55 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:06:56.193 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:55 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:56.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:55 smithi204 ceph-mon[87071]: pgmap v1354: 135 pgs: 29 unknown, 106 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:06:56.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:55 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:06:56.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:55 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:06:56.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:55 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:06:56.223 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:56.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.247 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:56.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:56.349 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:56.374 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:56.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:56.475 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:56.500 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:56.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:56.602 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:56.627 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:56.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:56.729 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:56.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.756 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:56.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:56.857 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:56.881 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:56.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:56.983 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:57.008 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:57.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:57.076 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.mirror_image_status (2604 ms) 2024-09-13T15:06:57.076 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-09-13T15:06:57.110 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:57.135 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:57.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:57.164 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.mirror_image_map (89 ms) 2024-09-13T15:06:57.164 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-09-13T15:06:57.172 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2024-09-13T15:06:57.172 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-09-13T15:06:57.202 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.mirror_snapshot (31 ms) 2024-09-13T15:06:57.203 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-09-13T15:06:57.205 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-09-13T15:06:57.205 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-09-13T15:06:57.207 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-09-13T15:06:57.207 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-09-13T15:06:57.210 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.dir_add_already_existing (4 ms) 2024-09-13T15:06:57.210 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-09-13T15:06:57.216 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2024-09-13T15:06:57.216 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-09-13T15:06:57.219 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-09-13T15:06:57.219 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-09-13T15:06:57.223 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2024-09-13T15:06:57.223 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_image_add 2024-09-13T15:06:57.226 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2024-09-13T15:06:57.226 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-09-13T15:06:57.229 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-09-13T15:06:57.229 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_image_list 2024-09-13T15:06:57.233 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_image_list (4 ms) 2024-09-13T15:06:57.233 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-09-13T15:06:57.237 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2024-09-13T15:06:57.237 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.image_group_add 2024-09-13T15:06:57.237 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:57.239 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2024-09-13T15:06:57.239 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-09-13T15:06:57.242 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-09-13T15:06:57.242 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.image_group_get 2024-09-13T15:06:57.244 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2024-09-13T15:06:57.245 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-09-13T15:06:57.246 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-09-13T15:06:57.246 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-09-13T15:06:57.248 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-09-13T15:06:57.248 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-09-13T15:06:57.251 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (2 ms) 2024-09-13T15:06:57.251 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-09-13T15:06:57.254 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2024-09-13T15:06:57.254 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-09-13T15:06:57.256 INFO:tasks.workunit.client.0.smithi096.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1627-gbbdadfa3/rpm/el9/BUILD/ceph-17.2.7-1627-gbbdadfa3/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-09-13T15:06:57.256 INFO:tasks.workunit.client.0.smithi096.stdout:Expected equality of these values: 2024-09-13T15:06:57.256 INFO:tasks.workunit.client.0.smithi096.stdout: 1U 2024-09-13T15:06:57.256 INFO:tasks.workunit.client.0.smithi096.stdout: Which is: 1 2024-09-13T15:06:57.256 INFO:tasks.workunit.client.0.smithi096.stdout: keys.size() 2024-09-13T15:06:57.256 INFO:tasks.workunit.client.0.smithi096.stdout: Which is: 3 2024-09-13T15:06:57.256 INFO:tasks.workunit.client.0.smithi096.stdout:[ FAILED ] TestClsRbd.group_snap_set (2 ms) 2024-09-13T15:06:57.256 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-09-13T15:06:57.259 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_snap_list (4 ms) 2024-09-13T15:06:57.259 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-09-13T15:06:57.261 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:57.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:57.275 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (16 ms) 2024-09-13T15:06:57.275 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-09-13T15:06:57.366 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:57.387 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:57.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:57.469 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:06:57.476 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (200 ms) 2024-09-13T15:06:57.476 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-09-13T15:06:57.479 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2024-09-13T15:06:57.479 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-09-13T15:06:57.481 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2024-09-13T15:06:57.481 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.trash_methods 2024-09-13T15:06:57.488 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.trash_methods (6 ms) 2024-09-13T15:06:57.488 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.op_features 2024-09-13T15:06:57.488 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:57.495 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2024-09-13T15:06:57.495 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.clone_parent 2024-09-13T15:06:57.513 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:57.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:57.517 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.clone_parent (23 ms) 2024-09-13T15:06:57.517 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-09-13T15:06:57.525 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2024-09-13T15:06:57.525 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.clone_child 2024-09-13T15:06:57.536 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.clone_child (10 ms) 2024-09-13T15:06:57.536 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-09-13T15:06:57.543 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2024-09-13T15:06:57.543 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.migration 2024-09-13T15:06:57.551 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.migration (9 ms) 2024-09-13T15:06:57.552 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.migration_v1 2024-09-13T15:06:57.562 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.migration_v1 (10 ms) 2024-09-13T15:06:57.562 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-09-13T15:06:57.615 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:57.638 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:57.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:57.740 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:57.762 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:57.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:57.864 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:57.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.886 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:57.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:57.988 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:58.009 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:58.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:58.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:57 smithi096 ceph-mon[98901]: pgmap v1356: 135 pgs: 25 unknown, 110 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 107 KiB/s rd, 48 KiB/s wr, 173 op/s 2024-09-13T15:06:58.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:06:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:06:57] "GET /metrics HTTP/1.1" 200 34163 "" "Prometheus/2.51.0" 2024-09-13T15:06:58.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:57 smithi096 ceph-mon[102928]: pgmap v1356: 135 pgs: 25 unknown, 110 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 107 KiB/s rd, 48 KiB/s wr, 173 op/s 2024-09-13T15:06:58.111 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:58.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.133 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:58.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:58.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:57 smithi204 ceph-mon[87071]: pgmap v1356: 135 pgs: 25 unknown, 110 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 107 KiB/s rd, 48 KiB/s wr, 173 op/s 2024-09-13T15:06:58.234 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:58.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.256 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:58.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:58.357 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:58.380 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:58.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:58.482 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:58.504 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:58.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:58.605 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:58.627 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:58.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:58.728 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:58.749 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:58.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:58.841 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T15:06:58.843 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T15:06:58.851 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:58.873 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:58.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:58.919 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1358 ms) 2024-09-13T15:06:58.920 INFO:tasks.workunit.client.0.smithi096.stdout:[ RUN ] TestClsRbd.sparsify 2024-09-13T15:06:58.953 INFO:tasks.workunit.client.0.smithi096.stdout:[ OK ] TestClsRbd.sparsify (33 ms) 2024-09-13T15:06:58.975 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:58.996 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:58.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:59.098 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:59.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.120 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:59.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:59.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:58 smithi204 ceph-mon[87071]: osdmap e406: 8 total, 8 up, 8 in 2024-09-13T15:06:59.222 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:59.243 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:59.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:59.244 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:58 smithi096 ceph-mon[98901]: osdmap e406: 8 total, 8 up, 8 in 2024-09-13T15:06:59.244 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:58 smithi096 ceph-mon[102928]: osdmap e406: 8 total, 8 up, 8 in 2024-09-13T15:06:59.345 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:59.367 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:59.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:59.469 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:59.491 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:59.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:59.593 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:59.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.616 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:59.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:59.717 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:59.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.742 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:59.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:59.843 DEBUG:teuthology.orchestra.run.smithi096:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:59.865 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: docker: command not found 2024-09-13T15:06:59.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi096 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:59.925 INFO:tasks.workunit.client.0.smithi096.stdout:[----------] 70 tests from TestClsRbd (5064 ms total) 2024-09-13T15:06:59.926 INFO:tasks.workunit.client.0.smithi096.stdout: 2024-09-13T15:06:59.926 INFO:tasks.workunit.client.0.smithi096.stdout:[----------] Global test environment tear-down 2024-09-13T15:06:59.926 INFO:tasks.workunit.client.0.smithi096.stdout:[==========] 70 tests from 1 test suite ran. (7834 ms total) 2024-09-13T15:06:59.926 INFO:tasks.workunit.client.0.smithi096.stdout:[ PASSED ] 69 tests. 2024-09-13T15:06:59.926 INFO:tasks.workunit.client.0.smithi096.stdout:[ FAILED ] 1 test, listed below: 2024-09-13T15:06:59.926 INFO:tasks.workunit.client.0.smithi096.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-09-13T15:06:59.926 INFO:tasks.workunit.client.0.smithi096.stdout: 2024-09-13T15:06:59.926 INFO:tasks.workunit.client.0.smithi096.stdout: 1 FAILED TEST 2024-09-13T15:06:59.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.935 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-09-13T15:06:59.935 DEBUG:teuthology.orchestra.run.smithi096:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-13T15:06:59.967 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:59.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.991 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:06:59.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:00.093 DEBUG:teuthology.orchestra.run.smithi204:> sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:00.116 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: docker: command not found 2024-09-13T15:07:00.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi204 with status 1: 'sudo docker kill -s 1 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:00.140 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:59 smithi096 ceph-mon[98901]: pgmap v1358: 135 pgs: 135 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1.7 MiB/s rd, 843 KiB/s wr, 319 op/s 2024-09-13T15:07:00.141 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:59 smithi096 ceph-mon[98901]: from='client.36834 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:07:00.141 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:06:59 smithi096 ceph-mon[98901]: osdmap e407: 8 total, 8 up, 8 in 2024-09-13T15:07:00.141 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:59 smithi096 ceph-mon[102928]: pgmap v1358: 135 pgs: 135 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1.7 MiB/s rd, 843 KiB/s wr, 319 op/s 2024-09-13T15:07:00.141 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:59 smithi096 ceph-mon[102928]: from='client.36834 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:07:00.141 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:06:59 smithi096 ceph-mon[102928]: osdmap e407: 8 total, 8 up, 8 in 2024-09-13T15:07:00.142 INFO:tasks.thrashosds:joining thrashosds 2024-09-13T15:07:00.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:59 smithi204 ceph-mon[87071]: pgmap v1358: 135 pgs: 135 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1.7 MiB/s rd, 843 KiB/s wr, 319 op/s 2024-09-13T15:07:00.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:59 smithi204 ceph-mon[87071]: from='client.36834 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:07:00.214 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:06:59 smithi204 ceph-mon[87071]: osdmap e407: 8 total, 8 up, 8 in 2024-09-13T15:07:00.309 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":1358,"stamp":"2024-09-13T15:06:58.243967+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790553,"num_objects":59,"num_object_clones":0,"num_object_copies":177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":59,"num_whiteouts":0,"num_read":7240,"num_read_kb":22345,"num_write":1379,"num_write_kb":17153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8732588,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4575,"ondisk_log_size":4575,"up":405,"acting":405,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":368,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6536096,"kb_used_data":61360,"kb_used_omap":39,"kb_used_meta":6474520,"kb_avail":743425120,"statfs":{"total":767960285184,"available":761267322880,"internally_reserved":0,"allocated":62832640,"data_stored":40227310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40927,"internal_metadata":6629908513},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":332,"num_objects":31,"num_object_clones":0,"num_object_copies":93,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31,"num_whiteouts":0,"num_read":1242,"num_read_kb":9334,"num_write":472,"num_write_kb":4530,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":24,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.372889"},"pg_stats":[{"pgid":"3.1f","version":"64'4","reported_seq":720,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.886560+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T15:06:53.886560+0000","last_peered":"2024-09-13T15:06:53.886560+0000","last_clean":"2024-09-13T15:06:53.886560+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T15:06:53.886560+0000","last_undegraded":"2024-09-13T15:06:53.886560+0000","last_fullsized":"2024-09-13T15:06:53.886560+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":392,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.874787+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T15:06:53.874787+0000","last_peered":"2024-09-13T15:06:53.874787+0000","last_clean":"2024-09-13T15:06:53.874787+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T15:06:53.874787+0000","last_undegraded":"2024-09-13T15:06:53.874787+0000","last_fullsized":"2024-09-13T15:06:53.874787+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"82'100","reported_seq":150,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.880959+0000","last_change":"2024-09-13T15:05:52.013759+0000","last_active":"2024-09-13T15:06:53.880959+0000","last_peered":"2024-09-13T15:06:53.880959+0000","last_clean":"2024-09-13T15:06:53.880959+0000","last_became_active":"2024-09-13T15:05:52.013210+0000","last_became_peered":"2024-09-13T15:05:52.013210+0000","last_unstale":"2024-09-13T15:06:53.880959+0000","last_undegraded":"2024-09-13T15:06:53.880959+0000","last_fullsized":"2024-09-13T15:06:53.880959+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.149290+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.149290+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:54.818210+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.17","version":"404'2","reported_seq":12,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.223064+0000","last_change":"2024-09-13T15:06:53.885841+0000","last_active":"2024-09-13T15:06:54.223064+0000","last_peered":"2024-09-13T15:06:54.223064+0000","last_clean":"2024-09-13T15:06:54.223064+0000","last_became_active":"2024-09-13T15:06:53.885521+0000","last_became_peered":"2024-09-13T15:06:53.885521+0000","last_unstale":"2024-09-13T15:06:54.223064+0000","last_undegraded":"2024-09-13T15:06:54.223064+0000","last_fullsized":"2024-09-13T15:06:54.223064+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:33:14.848635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987801+0000","last_change":"2024-09-13T15:05:51.995035+0000","last_active":"2024-09-13T15:06:54.987801+0000","last_peered":"2024-09-13T15:06:54.987801+0000","last_clean":"2024-09-13T15:06:54.987801+0000","last_became_active":"2024-09-13T15:05:51.994496+0000","last_became_peered":"2024-09-13T15:05:51.994496+0000","last_unstale":"2024-09-13T15:06:54.987801+0000","last_undegraded":"2024-09-13T15:06:54.987801+0000","last_fullsized":"2024-09-13T15:06:54.987801+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:11:51.103755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987468+0000","last_change":"2024-09-13T15:05:52.004131+0000","last_active":"2024-09-13T15:06:54.987468+0000","last_peered":"2024-09-13T15:06:54.987468+0000","last_clean":"2024-09-13T15:06:54.987468+0000","last_became_active":"2024-09-13T15:05:52.003845+0000","last_became_peered":"2024-09-13T15:05:52.003845+0000","last_unstale":"2024-09-13T15:06:54.987468+0000","last_undegraded":"2024-09-13T15:06:54.987468+0000","last_fullsized":"2024-09-13T15:06:54.987468+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:01.268036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"82'100","reported_seq":63,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.875339+0000","last_change":"2024-09-13T15:02:23.879862+0000","last_active":"2024-09-13T15:06:53.875339+0000","last_peered":"2024-09-13T15:06:53.875339+0000","last_clean":"2024-09-13T15:06:53.875339+0000","last_became_active":"2024-09-13T15:02:23.879170+0000","last_became_peered":"2024-09-13T15:02:23.879170+0000","last_unstale":"2024-09-13T15:06:53.875339+0000","last_undegraded":"2024-09-13T15:06:53.875339+0000","last_fullsized":"2024-09-13T15:06:53.875339+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:06.945525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:06.945525+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:48.496118+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.16","version":"404'22","reported_seq":42,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.412647+0000","last_change":"2024-09-13T15:06:53.887603+0000","last_active":"2024-09-13T15:06:54.412647+0000","last_peered":"2024-09-13T15:06:54.412647+0000","last_clean":"2024-09-13T15:06:54.412647+0000","last_became_active":"2024-09-13T15:06:53.887249+0000","last_became_peered":"2024-09-13T15:06:53.887249+0000","last_unstale":"2024-09-13T15:06:54.412647+0000","last_undegraded":"2024-09-13T15:06:54.412647+0000","last_fullsized":"2024-09-13T15:06:54.412647+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:15.930964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":23,"num_read_kb":44,"num_write":14,"num_write_kb":17,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":410,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878661+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T15:06:53.878661+0000","last_peered":"2024-09-13T15:06:53.878661+0000","last_clean":"2024-09-13T15:06:53.878661+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T15:06:53.878661+0000","last_undegraded":"2024-09-13T15:06:53.878661+0000","last_fullsized":"2024-09-13T15:06:53.878661+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":406,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.874726+0000","last_change":"2024-09-13T15:05:51.995355+0000","last_active":"2024-09-13T15:06:53.874726+0000","last_peered":"2024-09-13T15:06:53.874726+0000","last_clean":"2024-09-13T15:06:53.874726+0000","last_became_active":"2024-09-13T15:05:51.995038+0000","last_became_peered":"2024-09-13T15:05:51.995038+0000","last_unstale":"2024-09-13T15:06:53.874726+0000","last_undegraded":"2024-09-13T15:06:53.874726+0000","last_fullsized":"2024-09-13T15:06:53.874726+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:24:14.413713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"0'0","reported_seq":11,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.893041+0000","last_change":"2024-09-13T15:06:53.891530+0000","last_active":"2024-09-13T15:06:53.893041+0000","last_peered":"2024-09-13T15:06:53.893041+0000","last_clean":"2024-09-13T15:06:53.893041+0000","last_became_active":"2024-09-13T15:06:53.891260+0000","last_became_peered":"2024-09-13T15:06:53.891260+0000","last_unstale":"2024-09-13T15:06:53.893041+0000","last_undegraded":"2024-09-13T15:06:53.893041+0000","last_fullsized":"2024-09-13T15:06:53.893041+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:15:08.615676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":371,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.877951+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T15:06:53.877951+0000","last_peered":"2024-09-13T15:06:53.877951+0000","last_clean":"2024-09-13T15:06:53.877951+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T15:06:53.877951+0000","last_undegraded":"2024-09-13T15:06:53.877951+0000","last_fullsized":"2024-09-13T15:06:53.877951+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":406,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.875087+0000","last_change":"2024-09-13T15:05:51.998541+0000","last_active":"2024-09-13T15:06:53.875087+0000","last_peered":"2024-09-13T15:06:53.875087+0000","last_clean":"2024-09-13T15:06:53.875087+0000","last_became_active":"2024-09-13T15:05:51.998234+0000","last_became_peered":"2024-09-13T15:05:51.998234+0000","last_unstale":"2024-09-13T15:06:53.875087+0000","last_undegraded":"2024-09-13T15:06:53.875087+0000","last_fullsized":"2024-09-13T15:06:53.875087+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:25.705700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"82'100","reported_seq":62,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987625+0000","last_change":"2024-09-13T15:05:52.535449+0000","last_active":"2024-09-13T15:06:54.987625+0000","last_peered":"2024-09-13T15:06:54.987625+0000","last_clean":"2024-09-13T15:06:54.987625+0000","last_became_active":"2024-09-13T15:05:51.991505+0000","last_became_peered":"2024-09-13T15:05:51.991505+0000","last_unstale":"2024-09-13T15:06:54.987625+0000","last_undegraded":"2024-09-13T15:06:54.987625+0000","last_fullsized":"2024-09-13T15:06:54.987625+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:05:52.535365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:05:52.535365+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:39:36.016324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.14","version":"404'7","reported_seq":19,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.241948+0000","last_change":"2024-09-13T15:06:53.883414+0000","last_active":"2024-09-13T15:06:54.241948+0000","last_peered":"2024-09-13T15:06:54.241948+0000","last_clean":"2024-09-13T15:06:54.241948+0000","last_became_active":"2024-09-13T15:06:53.883072+0000","last_became_peered":"2024-09-13T15:06:53.883072+0000","last_unstale":"2024-09-13T15:06:54.241948+0000","last_undegraded":"2024-09-13T15:06:54.241948+0000","last_fullsized":"2024-09-13T15:06:54.241948+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:59:28.271646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":26,"num_write":8,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":668,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.888957+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T15:06:53.888957+0000","last_peered":"2024-09-13T15:06:53.888957+0000","last_clean":"2024-09-13T15:06:53.888957+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T15:06:53.888957+0000","last_undegraded":"2024-09-13T15:06:53.888957+0000","last_fullsized":"2024-09-13T15:06:53.888957+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":512,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.880161+0000","last_change":"2024-09-13T15:05:52.013949+0000","last_active":"2024-09-13T15:06:53.880161+0000","last_peered":"2024-09-13T15:06:53.880161+0000","last_clean":"2024-09-13T15:06:53.880161+0000","last_became_active":"2024-09-13T15:05:52.013611+0000","last_became_peered":"2024-09-13T15:05:52.013611+0000","last_unstale":"2024-09-13T15:06:53.880161+0000","last_undegraded":"2024-09-13T15:06:53.880161+0000","last_fullsized":"2024-09-13T15:06:53.880161+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:29.374445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.19","version":"82'100","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987350+0000","last_change":"2024-09-13T15:05:52.002778+0000","last_active":"2024-09-13T15:06:54.987350+0000","last_peered":"2024-09-13T15:06:54.987350+0000","last_clean":"2024-09-13T15:06:54.987350+0000","last_became_active":"2024-09-13T15:05:52.002405+0000","last_became_peered":"2024-09-13T15:05:52.002405+0000","last_unstale":"2024-09-13T15:06:54.987350+0000","last_undegraded":"2024-09-13T15:06:54.987350+0000","last_fullsized":"2024-09-13T15:06:54.987350+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.429543+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.429543+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:04:50.457862+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.13","version":"0'0","reported_seq":10,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.896614+0000","last_change":"2024-09-13T15:06:53.896372+0000","last_active":"2024-09-13T15:06:53.896614+0000","last_peered":"2024-09-13T15:06:53.896614+0000","last_clean":"2024-09-13T15:06:53.896614+0000","last_became_active":"2024-09-13T15:06:53.896124+0000","last_became_peered":"2024-09-13T15:06:53.896124+0000","last_unstale":"2024-09-13T15:06:53.896614+0000","last_undegraded":"2024-09-13T15:06:53.896614+0000","last_fullsized":"2024-09-13T15:06:53.896614+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:27.927135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":388,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.874813+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T15:06:53.874813+0000","last_peered":"2024-09-13T15:06:53.874813+0000","last_clean":"2024-09-13T15:06:53.874813+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T15:06:53.874813+0000","last_undegraded":"2024-09-13T15:06:53.874813+0000","last_fullsized":"2024-09-13T15:06:53.874813+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987734+0000","last_change":"2024-09-13T15:05:52.001307+0000","last_active":"2024-09-13T15:06:54.987734+0000","last_peered":"2024-09-13T15:06:54.987734+0000","last_clean":"2024-09-13T15:06:54.987734+0000","last_became_active":"2024-09-13T15:05:52.001040+0000","last_became_peered":"2024-09-13T15:05:52.001040+0000","last_unstale":"2024-09-13T15:06:54.987734+0000","last_undegraded":"2024-09-13T15:06:54.987734+0000","last_fullsized":"2024-09-13T15:06:54.987734+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:49:48.758670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"82'100","reported_seq":360,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878312+0000","last_change":"2024-09-13T15:03:04.101898+0000","last_active":"2024-09-13T15:06:53.878312+0000","last_peered":"2024-09-13T15:06:53.878312+0000","last_clean":"2024-09-13T15:06:53.878312+0000","last_became_active":"2024-09-13T15:02:23.884406+0000","last_became_peered":"2024-09-13T15:02:23.884406+0000","last_unstale":"2024-09-13T15:06:53.878312+0000","last_undegraded":"2024-09-13T15:06:53.878312+0000","last_fullsized":"2024-09-13T15:06:53.878312+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:04.101802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:04.101802+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:34.459623+0000","scrub_duration":0.003875819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.12","version":"404'1","reported_seq":12,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.204500+0000","last_change":"2024-09-13T15:06:53.898393+0000","last_active":"2024-09-13T15:06:54.204500+0000","last_peered":"2024-09-13T15:06:54.204500+0000","last_clean":"2024-09-13T15:06:54.204500+0000","last_became_active":"2024-09-13T15:06:53.898112+0000","last_became_peered":"2024-09-13T15:06:53.898112+0000","last_unstale":"2024-09-13T15:06:54.204500+0000","last_undegraded":"2024-09-13T15:06:54.204500+0000","last_fullsized":"2024-09-13T15:06:54.204500+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:33.763930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":678,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878740+0000","last_change":"2024-09-13T15:05:52.007634+0000","last_active":"2024-09-13T15:06:53.878740+0000","last_peered":"2024-09-13T15:06:53.878740+0000","last_clean":"2024-09-13T15:06:53.878740+0000","last_became_active":"2024-09-13T15:05:52.007302+0000","last_became_peered":"2024-09-13T15:05:52.007302+0000","last_unstale":"2024-09-13T15:06:53.878740+0000","last_undegraded":"2024-09-13T15:06:53.878740+0000","last_fullsized":"2024-09-13T15:06:53.878740+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:36.203235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":714,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.894465+0000","last_change":"2024-09-13T15:05:52.008086+0000","last_active":"2024-09-13T15:06:53.894465+0000","last_peered":"2024-09-13T15:06:53.894465+0000","last_clean":"2024-09-13T15:06:53.894465+0000","last_became_active":"2024-09-13T15:05:52.007347+0000","last_became_peered":"2024-09-13T15:05:52.007347+0000","last_unstale":"2024-09-13T15:06:53.894465+0000","last_undegraded":"2024-09-13T15:06:53.894465+0000","last_fullsized":"2024-09-13T15:06:53.894465+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:12:09.381403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"82'100","reported_seq":655,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878006+0000","last_change":"2024-09-13T15:03:07.166808+0000","last_active":"2024-09-13T15:06:53.878006+0000","last_peered":"2024-09-13T15:06:53.878006+0000","last_clean":"2024-09-13T15:06:53.878006+0000","last_became_active":"2024-09-13T15:02:23.883387+0000","last_became_peered":"2024-09-13T15:02:23.883387+0000","last_unstale":"2024-09-13T15:06:53.878006+0000","last_undegraded":"2024-09-13T15:06:53.878006+0000","last_fullsized":"2024-09-13T15:06:53.878006+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:07.166034+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:07.166034+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:32:42.605595+0000","scrub_duration":0.01094734,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.11","version":"404'5","reported_seq":21,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987378+0000","last_change":"2024-09-13T15:06:53.894352+0000","last_active":"2024-09-13T15:06:54.987378+0000","last_peered":"2024-09-13T15:06:54.987378+0000","last_clean":"2024-09-13T15:06:54.987378+0000","last_became_active":"2024-09-13T15:06:53.894125+0000","last_became_peered":"2024-09-13T15:06:53.894125+0000","last_unstale":"2024-09-13T15:06:54.987378+0000","last_undegraded":"2024-09-13T15:06:54.987378+0000","last_fullsized":"2024-09-13T15:06:54.987378+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:23:59.895835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":3,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987071+0000","last_change":"2024-09-13T15:05:52.003501+0000","last_active":"2024-09-13T15:06:54.987071+0000","last_peered":"2024-09-13T15:06:54.987071+0000","last_clean":"2024-09-13T15:06:54.987071+0000","last_became_active":"2024-09-13T15:05:52.003200+0000","last_became_peered":"2024-09-13T15:05:52.003200+0000","last_unstale":"2024-09-13T15:06:54.987071+0000","last_undegraded":"2024-09-13T15:06:54.987071+0000","last_fullsized":"2024-09-13T15:06:54.987071+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:46:07.106935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":189,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.883045+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T15:06:53.883045+0000","last_peered":"2024-09-13T15:06:53.883045+0000","last_clean":"2024-09-13T15:06:53.883045+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T15:06:53.883045+0000","last_undegraded":"2024-09-13T15:06:53.883045+0000","last_fullsized":"2024-09-13T15:06:53.883045+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"82'100","reported_seq":73,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.874139+0000","last_change":"2024-09-13T15:02:56.319389+0000","last_active":"2024-09-13T15:06:53.874139+0000","last_peered":"2024-09-13T15:06:53.874139+0000","last_clean":"2024-09-13T15:06:53.874139+0000","last_became_active":"2024-09-13T15:02:23.876739+0000","last_became_peered":"2024-09-13T15:02:23.876739+0000","last_unstale":"2024-09-13T15:06:53.874139+0000","last_undegraded":"2024-09-13T15:06:53.874139+0000","last_fullsized":"2024-09-13T15:06:53.874139+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:56.319302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:56.319302+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:58.230158+0000","scrub_duration":0.0031182190000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.10","version":"404'4","reported_seq":15,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.345325+0000","last_change":"2024-09-13T15:06:53.900041+0000","last_active":"2024-09-13T15:06:54.345325+0000","last_peered":"2024-09-13T15:06:54.345325+0000","last_clean":"2024-09-13T15:06:54.345325+0000","last_became_active":"2024-09-13T15:06:53.899766+0000","last_became_peered":"2024-09-13T15:06:53.899766+0000","last_unstale":"2024-09-13T15:06:54.345325+0000","last_undegraded":"2024-09-13T15:06:54.345325+0000","last_fullsized":"2024-09-13T15:06:54.345325+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:11:00.551314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":5,"num_read_kb":5,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.1f","version":"0'0","reported_seq":14,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987973+0000","last_change":"2024-09-13T15:06:53.903135+0000","last_active":"2024-09-13T15:06:54.987973+0000","last_peered":"2024-09-13T15:06:54.987973+0000","last_clean":"2024-09-13T15:06:54.987973+0000","last_became_active":"2024-09-13T15:06:53.902025+0000","last_became_peered":"2024-09-13T15:06:53.902025+0000","last_unstale":"2024-09-13T15:06:54.987973+0000","last_undegraded":"2024-09-13T15:06:54.987973+0000","last_fullsized":"2024-09-13T15:06:54.987973+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:25.537017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987045+0000","last_change":"2024-09-13T15:05:51.999327+0000","last_active":"2024-09-13T15:06:54.987045+0000","last_peered":"2024-09-13T15:06:54.987045+0000","last_clean":"2024-09-13T15:06:54.987045+0000","last_became_active":"2024-09-13T15:05:51.994485+0000","last_became_peered":"2024-09-13T15:05:51.994485+0000","last_unstale":"2024-09-13T15:06:54.987045+0000","last_undegraded":"2024-09-13T15:06:54.987045+0000","last_fullsized":"2024-09-13T15:06:54.987045+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:55:49.041649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":689,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.886680+0000","last_change":"2024-09-13T15:05:52.006782+0000","last_active":"2024-09-13T15:06:53.886680+0000","last_peered":"2024-09-13T15:06:53.886680+0000","last_clean":"2024-09-13T15:06:53.886680+0000","last_became_active":"2024-09-13T15:05:52.006452+0000","last_became_peered":"2024-09-13T15:05:52.006452+0000","last_unstale":"2024-09-13T15:06:53.886680+0000","last_undegraded":"2024-09-13T15:06:53.886680+0000","last_fullsized":"2024-09-13T15:06:53.886680+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:37:23.855899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"82'100","reported_seq":73,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.875769+0000","last_change":"2024-09-13T15:02:55.870615+0000","last_active":"2024-09-13T15:06:53.875769+0000","last_peered":"2024-09-13T15:06:53.875769+0000","last_clean":"2024-09-13T15:06:53.875769+0000","last_became_active":"2024-09-13T15:02:23.883217+0000","last_became_peered":"2024-09-13T15:02:23.883217+0000","last_unstale":"2024-09-13T15:06:53.875769+0000","last_undegraded":"2024-09-13T15:06:53.875769+0000","last_fullsized":"2024-09-13T15:06:53.875769+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.870528+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.870528+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:10.023787+0000","scrub_duration":0.0047551850000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.1e","version":"404'7","reported_seq":18,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.150788+0000","last_change":"2024-09-13T15:06:53.900986+0000","last_active":"2024-09-13T15:06:54.150788+0000","last_peered":"2024-09-13T15:06:54.150788+0000","last_clean":"2024-09-13T15:06:54.150788+0000","last_became_active":"2024-09-13T15:06:53.900689+0000","last_became_peered":"2024-09-13T15:06:53.900689+0000","last_unstale":"2024-09-13T15:06:54.150788+0000","last_undegraded":"2024-09-13T15:06:54.150788+0000","last_fullsized":"2024-09-13T15:06:54.150788+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:08:17.624502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":4,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987688+0000","last_change":"2024-09-13T15:05:52.010150+0000","last_active":"2024-09-13T15:06:54.987688+0000","last_peered":"2024-09-13T15:06:54.987688+0000","last_clean":"2024-09-13T15:06:54.987688+0000","last_became_active":"2024-09-13T15:05:52.008977+0000","last_became_peered":"2024-09-13T15:05:52.008977+0000","last_unstale":"2024-09-13T15:06:54.987688+0000","last_undegraded":"2024-09-13T15:06:54.987688+0000","last_fullsized":"2024-09-13T15:06:54.987688+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:43.067814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":369,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878816+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T15:06:53.878816+0000","last_peered":"2024-09-13T15:06:53.878816+0000","last_clean":"2024-09-13T15:06:53.878816+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T15:06:53.878816+0000","last_undegraded":"2024-09-13T15:06:53.878816+0000","last_fullsized":"2024-09-13T15:06:53.878816+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":260,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.881726+0000","last_change":"2024-09-13T14:56:01.235655+0000","last_active":"2024-09-13T15:06:53.881726+0000","last_peered":"2024-09-13T15:06:53.881726+0000","last_clean":"2024-09-13T15:06:53.881726+0000","last_became_active":"2024-09-13T14:55:55.195703+0000","last_became_peered":"2024-09-13T14:55:55.195703+0000","last_unstale":"2024-09-13T15:06:53.881726+0000","last_undegraded":"2024-09-13T15:06:53.881726+0000","last_fullsized":"2024-09-13T15:06:53.881726+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.235497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.235497+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:36:33.861613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1d","version":"404'2","reported_seq":12,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.294228+0000","last_change":"2024-09-13T15:06:53.895821+0000","last_active":"2024-09-13T15:06:54.294228+0000","last_peered":"2024-09-13T15:06:54.294228+0000","last_clean":"2024-09-13T15:06:54.294228+0000","last_became_active":"2024-09-13T15:06:53.895611+0000","last_became_peered":"2024-09-13T15:06:53.895611+0000","last_unstale":"2024-09-13T15:06:54.294228+0000","last_undegraded":"2024-09-13T15:06:54.294228+0000","last_fullsized":"2024-09-13T15:06:54.294228+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:25.448450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":411,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.875362+0000","last_change":"2024-09-13T15:05:52.016761+0000","last_active":"2024-09-13T15:06:53.875362+0000","last_peered":"2024-09-13T15:06:53.875362+0000","last_clean":"2024-09-13T15:06:53.875362+0000","last_became_active":"2024-09-13T15:05:52.005623+0000","last_became_peered":"2024-09-13T15:05:52.005623+0000","last_unstale":"2024-09-13T15:06:53.875362+0000","last_undegraded":"2024-09-13T15:06:53.875362+0000","last_fullsized":"2024-09-13T15:06:53.875362+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:58:43.424615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":678,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.880007+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T15:06:53.880007+0000","last_peered":"2024-09-13T15:06:53.880007+0000","last_clean":"2024-09-13T15:06:53.880007+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T15:06:53.880007+0000","last_undegraded":"2024-09-13T15:06:53.880007+0000","last_fullsized":"2024-09-13T15:06:53.880007+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.17","version":"82'100","reported_seq":217,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.879604+0000","last_change":"2024-09-13T15:03:03.108031+0000","last_active":"2024-09-13T15:06:53.879604+0000","last_peered":"2024-09-13T15:06:53.879604+0000","last_clean":"2024-09-13T15:06:53.879604+0000","last_became_active":"2024-09-13T15:02:23.883373+0000","last_became_peered":"2024-09-13T15:02:23.883373+0000","last_unstale":"2024-09-13T15:06:53.879604+0000","last_undegraded":"2024-09-13T15:06:53.879604+0000","last_fullsized":"2024-09-13T15:06:53.879604+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:03.107786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:03.107786+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:17:10.041468+0000","scrub_duration":0.0049024139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1c","version":"0'0","reported_seq":10,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.900626+0000","last_change":"2024-09-13T15:06:53.900304+0000","last_active":"2024-09-13T15:06:53.900626+0000","last_peered":"2024-09-13T15:06:53.900626+0000","last_clean":"2024-09-13T15:06:53.900626+0000","last_became_active":"2024-09-13T15:06:53.900034+0000","last_became_peered":"2024-09-13T15:06:53.900034+0000","last_unstale":"2024-09-13T15:06:53.900626+0000","last_undegraded":"2024-09-13T15:06:53.900626+0000","last_fullsized":"2024-09-13T15:06:53.900626+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:58:38.708814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":696,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.887288+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T15:06:53.887288+0000","last_peered":"2024-09-13T15:06:53.887288+0000","last_clean":"2024-09-13T15:06:53.887288+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T15:06:53.887288+0000","last_undegraded":"2024-09-13T15:06:53.887288+0000","last_fullsized":"2024-09-13T15:06:53.887288+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":389,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.875165+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T15:06:53.875165+0000","last_peered":"2024-09-13T15:06:53.875165+0000","last_clean":"2024-09-13T15:06:53.875165+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T15:06:53.875165+0000","last_undegraded":"2024-09-13T15:06:53.875165+0000","last_fullsized":"2024-09-13T15:06:53.875165+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"82'100","reported_seq":148,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.880278+0000","last_change":"2024-09-13T15:05:52.014434+0000","last_active":"2024-09-13T15:06:53.880278+0000","last_peered":"2024-09-13T15:06:53.880278+0000","last_clean":"2024-09-13T15:06:53.880278+0000","last_became_active":"2024-09-13T15:05:52.014086+0000","last_became_peered":"2024-09-13T15:05:52.014086+0000","last_unstale":"2024-09-13T15:06:53.880278+0000","last_undegraded":"2024-09-13T15:06:53.880278+0000","last_fullsized":"2024-09-13T15:06:53.880278+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:51.192503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:51.192503+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:53.661130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":372,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.874050+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T15:06:53.874050+0000","last_peered":"2024-09-13T15:06:53.874050+0000","last_clean":"2024-09-13T15:06:53.874050+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T15:06:53.874050+0000","last_undegraded":"2024-09-13T15:06:53.874050+0000","last_fullsized":"2024-09-13T15:06:53.874050+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1218,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.883096+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T15:06:53.883096+0000","last_peered":"2024-09-13T15:06:53.883096+0000","last_clean":"2024-09-13T15:06:53.883096+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T15:06:53.883096+0000","last_undegraded":"2024-09-13T15:06:53.883096+0000","last_fullsized":"2024-09-13T15:06:53.883096+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":689,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.894256+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T15:06:53.894256+0000","last_peered":"2024-09-13T15:06:53.894256+0000","last_clean":"2024-09-13T15:06:53.894256+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T15:06:53.894256+0000","last_undegraded":"2024-09-13T15:06:53.894256+0000","last_fullsized":"2024-09-13T15:06:53.894256+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987265+0000","last_change":"2024-09-13T15:05:52.005786+0000","last_active":"2024-09-13T15:06:54.987265+0000","last_peered":"2024-09-13T15:06:54.987265+0000","last_clean":"2024-09-13T15:06:54.987265+0000","last_became_active":"2024-09-13T15:05:52.005453+0000","last_became_peered":"2024-09-13T15:05:52.005453+0000","last_unstale":"2024-09-13T15:06:54.987265+0000","last_undegraded":"2024-09-13T15:06:54.987265+0000","last_fullsized":"2024-09-13T15:06:54.987265+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:54.193156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:54.193156+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:51.141832+0000","scrub_duration":0.0040069900000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"404'9","reported_seq":47,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.405302+0000","last_change":"2024-09-13T15:06:53.884314+0000","last_active":"2024-09-13T15:06:54.405302+0000","last_peered":"2024-09-13T15:06:54.405302+0000","last_clean":"2024-09-13T15:06:54.405302+0000","last_became_active":"2024-09-13T15:06:53.883991+0000","last_became_peered":"2024-09-13T15:06:53.883991+0000","last_unstale":"2024-09-13T15:06:54.405302+0000","last_undegraded":"2024-09-13T15:06:54.405302+0000","last_fullsized":"2024-09-13T15:06:54.405302+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:49.096236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":32,"num_read_kb":52,"num_write":7,"num_write_kb":35,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987173+0000","last_change":"2024-09-13T15:05:52.003257+0000","last_active":"2024-09-13T15:06:54.987173+0000","last_peered":"2024-09-13T15:06:54.987173+0000","last_clean":"2024-09-13T15:06:54.987173+0000","last_became_active":"2024-09-13T15:05:52.002919+0000","last_became_peered":"2024-09-13T15:05:52.002919+0000","last_unstale":"2024-09-13T15:06:54.987173+0000","last_undegraded":"2024-09-13T15:06:54.987173+0000","last_fullsized":"2024-09-13T15:06:54.987173+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:04.833597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":1037,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.880419+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T15:06:53.880419+0000","last_peered":"2024-09-13T15:06:53.880419+0000","last_clean":"2024-09-13T15:06:53.880419+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T15:06:53.880419+0000","last_undegraded":"2024-09-13T15:06:53.880419+0000","last_fullsized":"2024-09-13T15:06:53.880419+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":58,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987928+0000","last_change":"2024-09-13T15:05:52.014941+0000","last_active":"2024-09-13T15:06:54.987928+0000","last_peered":"2024-09-13T15:06:54.987928+0000","last_clean":"2024-09-13T15:06:54.987928+0000","last_became_active":"2024-09-13T15:05:52.002400+0000","last_became_peered":"2024-09-13T15:05:52.002400+0000","last_unstale":"2024-09-13T15:06:54.987928+0000","last_undegraded":"2024-09-13T15:06:54.987928+0000","last_fullsized":"2024-09-13T15:06:54.987928+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:06.227994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":376,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.876412+0000","last_change":"2024-09-13T15:02:52.361444+0000","last_active":"2024-09-13T15:06:53.876412+0000","last_peered":"2024-09-13T15:06:53.876412+0000","last_clean":"2024-09-13T15:06:53.876412+0000","last_became_active":"2024-09-13T14:59:53.220135+0000","last_became_peered":"2024-09-13T14:59:53.220135+0000","last_unstale":"2024-09-13T15:06:53.876412+0000","last_undegraded":"2024-09-13T15:06:53.876412+0000","last_fullsized":"2024-09-13T15:06:53.876412+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.361359+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.361359+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:35.003140+0000","scrub_duration":0.005845154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.f","version":"404'10","reported_seq":25,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.376491+0000","last_change":"2024-09-13T15:06:53.900945+0000","last_active":"2024-09-13T15:06:54.376491+0000","last_peered":"2024-09-13T15:06:54.376491+0000","last_clean":"2024-09-13T15:06:54.376491+0000","last_became_active":"2024-09-13T15:06:53.900475+0000","last_became_peered":"2024-09-13T15:06:53.900475+0000","last_unstale":"2024-09-13T15:06:54.376491+0000","last_undegraded":"2024-09-13T15:06:54.376491+0000","last_fullsized":"2024-09-13T15:06:54.376491+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:54:20.589779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":18,"num_read_kb":8199,"num_write":8,"num_write_kb":4100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":667,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.889461+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T15:06:53.889461+0000","last_peered":"2024-09-13T15:06:53.889461+0000","last_clean":"2024-09-13T15:06:53.889461+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T15:06:53.889461+0000","last_undegraded":"2024-09-13T15:06:53.889461+0000","last_fullsized":"2024-09-13T15:06:53.889461+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":618,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.875854+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T15:06:53.875854+0000","last_peered":"2024-09-13T15:06:53.875854+0000","last_clean":"2024-09-13T15:06:53.875854+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T15:06:53.875854+0000","last_undegraded":"2024-09-13T15:06:53.875854+0000","last_fullsized":"2024-09-13T15:06:53.875854+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":680,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.879868+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T15:06:53.879868+0000","last_peered":"2024-09-13T15:06:53.879868+0000","last_clean":"2024-09-13T15:06:53.879868+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T15:06:53.879868+0000","last_undegraded":"2024-09-13T15:06:53.879868+0000","last_fullsized":"2024-09-13T15:06:53.879868+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":450,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.894140+0000","last_change":"2024-09-13T15:06:32.296671+0000","last_active":"2024-09-13T15:06:53.894140+0000","last_peered":"2024-09-13T15:06:53.894140+0000","last_clean":"2024-09-13T15:06:53.894140+0000","last_became_active":"2024-09-13T15:06:10.446381+0000","last_became_peered":"2024-09-13T15:06:10.446381+0000","last_unstale":"2024-09-13T15:06:53.894140+0000","last_undegraded":"2024-09-13T15:06:53.894140+0000","last_fullsized":"2024-09-13T15:06:53.894140+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:06:32.296605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:06:32.296605+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:27:11.006415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"0'0","reported_seq":11,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.885159+0000","last_change":"2024-09-13T15:06:53.884489+0000","last_active":"2024-09-13T15:06:53.885159+0000","last_peered":"2024-09-13T15:06:53.885159+0000","last_clean":"2024-09-13T15:06:53.885159+0000","last_became_active":"2024-09-13T15:06:53.884215+0000","last_became_peered":"2024-09-13T15:06:53.884215+0000","last_unstale":"2024-09-13T15:06:53.885159+0000","last_undegraded":"2024-09-13T15:06:53.885159+0000","last_fullsized":"2024-09-13T15:06:53.885159+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:57.319513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":655,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.879864+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T15:06:53.879864+0000","last_peered":"2024-09-13T15:06:53.879864+0000","last_clean":"2024-09-13T15:06:53.879864+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T15:06:53.879864+0000","last_undegraded":"2024-09-13T15:06:53.879864+0000","last_fullsized":"2024-09-13T15:06:53.879864+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":1183,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.879964+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T15:06:53.879964+0000","last_peered":"2024-09-13T15:06:53.879964+0000","last_clean":"2024-09-13T15:06:53.879964+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T15:06:53.879964+0000","last_undegraded":"2024-09-13T15:06:53.879964+0000","last_fullsized":"2024-09-13T15:06:53.879964+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":406,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878755+0000","last_change":"2024-09-13T15:05:52.014958+0000","last_active":"2024-09-13T15:06:53.878755+0000","last_peered":"2024-09-13T15:06:53.878755+0000","last_clean":"2024-09-13T15:06:53.878755+0000","last_became_active":"2024-09-13T15:05:52.014471+0000","last_became_peered":"2024-09-13T15:05:52.014471+0000","last_unstale":"2024-09-13T15:06:53.878755+0000","last_undegraded":"2024-09-13T15:06:53.878755+0000","last_fullsized":"2024-09-13T15:06:53.878755+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:38.514748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":291,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.988059+0000","last_change":"2024-09-13T15:06:10.445608+0000","last_active":"2024-09-13T15:06:54.988059+0000","last_peered":"2024-09-13T15:06:54.988059+0000","last_clean":"2024-09-13T15:06:54.988059+0000","last_became_active":"2024-09-13T15:06:10.445259+0000","last_became_peered":"2024-09-13T15:06:10.445259+0000","last_unstale":"2024-09-13T15:06:54.988059+0000","last_undegraded":"2024-09-13T15:06:54.988059+0000","last_fullsized":"2024-09-13T15:06:54.988059+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:49.771572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:49.771572+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:41.953119+0000","scrub_duration":0.0035237329999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"404'30","reported_seq":66,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.370989+0000","last_change":"2024-09-13T15:06:53.894017+0000","last_active":"2024-09-13T15:06:54.370989+0000","last_peered":"2024-09-13T15:06:54.370989+0000","last_clean":"2024-09-13T15:06:54.370989+0000","last_became_active":"2024-09-13T15:06:53.893705+0000","last_became_peered":"2024-09-13T15:06:53.893705+0000","last_unstale":"2024-09-13T15:06:54.370989+0000","last_undegraded":"2024-09-13T15:06:54.370989+0000","last_fullsized":"2024-09-13T15:06:54.370989+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:33:39.116357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":139,"num_read_kb":104,"num_write":29,"num_write_kb":24,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":712,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.879762+0000","last_change":"2024-09-13T15:05:52.008066+0000","last_active":"2024-09-13T15:06:53.879762+0000","last_peered":"2024-09-13T15:06:53.879762+0000","last_clean":"2024-09-13T15:06:53.879762+0000","last_became_active":"2024-09-13T15:05:52.007749+0000","last_became_peered":"2024-09-13T15:05:52.007749+0000","last_unstale":"2024-09-13T15:06:53.879762+0000","last_undegraded":"2024-09-13T15:06:53.879762+0000","last_fullsized":"2024-09-13T15:06:53.879762+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:11.201592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":390,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.875305+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T15:06:53.875305+0000","last_peered":"2024-09-13T15:06:53.875305+0000","last_clean":"2024-09-13T15:06:53.875305+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T15:06:53.875305+0000","last_undegraded":"2024-09-13T15:06:53.875305+0000","last_fullsized":"2024-09-13T15:06:53.875305+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":1031,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878900+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T15:06:53.878900+0000","last_peered":"2024-09-13T15:06:53.878900+0000","last_clean":"2024-09-13T15:06:53.878900+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T15:06:53.878900+0000","last_undegraded":"2024-09-13T15:06:53.878900+0000","last_fullsized":"2024-09-13T15:06:53.878900+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":325,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878046+0000","last_change":"2024-09-13T14:56:01.658240+0000","last_active":"2024-09-13T15:06:53.878046+0000","last_peered":"2024-09-13T15:06:53.878046+0000","last_clean":"2024-09-13T15:06:53.878046+0000","last_became_active":"2024-09-13T14:55:51.160480+0000","last_became_peered":"2024-09-13T14:55:51.160480+0000","last_unstale":"2024-09-13T15:06:53.878046+0000","last_undegraded":"2024-09-13T15:06:53.878046+0000","last_fullsized":"2024-09-13T15:06:53.878046+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.658143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.658143+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:19.262955+0000","scrub_duration":0.0038903079999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"404'37","reported_seq":76,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987494+0000","last_change":"2024-09-13T15:06:53.901134+0000","last_active":"2024-09-13T15:06:54.987494+0000","last_peered":"2024-09-13T15:06:54.987494+0000","last_clean":"2024-09-13T15:06:54.987494+0000","last_became_active":"2024-09-13T15:06:53.900229+0000","last_became_peered":"2024-09-13T15:06:53.900229+0000","last_unstale":"2024-09-13T15:06:54.987494+0000","last_undegraded":"2024-09-13T15:06:54.987494+0000","last_fullsized":"2024-09-13T15:06:54.987494+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":37,"log_dups_size":0,"ondisk_log_size":37,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:05.693974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":215,"num_read_kb":174,"num_write":44,"num_write_kb":41,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":698,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.880591+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T15:06:53.880591+0000","last_peered":"2024-09-13T15:06:53.880591+0000","last_clean":"2024-09-13T15:06:53.880591+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T15:06:53.880591+0000","last_undegraded":"2024-09-13T15:06:53.880591+0000","last_fullsized":"2024-09-13T15:06:53.880591+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":680,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.879099+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T15:06:53.879099+0000","last_peered":"2024-09-13T15:06:53.879099+0000","last_clean":"2024-09-13T15:06:53.879099+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T15:06:53.879099+0000","last_undegraded":"2024-09-13T15:06:53.879099+0000","last_fullsized":"2024-09-13T15:06:53.879099+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":78,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987523+0000","last_change":"2024-09-13T15:05:52.042632+0000","last_active":"2024-09-13T15:06:54.987523+0000","last_peered":"2024-09-13T15:06:54.987523+0000","last_clean":"2024-09-13T15:06:54.987523+0000","last_became_active":"2024-09-13T15:05:52.004324+0000","last_became_peered":"2024-09-13T15:05:52.004324+0000","last_unstale":"2024-09-13T15:06:54.987523+0000","last_undegraded":"2024-09-13T15:06:54.987523+0000","last_fullsized":"2024-09-13T15:06:54.987523+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:44.130757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":182,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.874704+0000","last_change":"2024-09-13T14:55:59.574827+0000","last_active":"2024-09-13T15:06:53.874704+0000","last_peered":"2024-09-13T15:06:53.874704+0000","last_clean":"2024-09-13T15:06:53.874704+0000","last_became_active":"2024-09-13T14:55:51.162903+0000","last_became_peered":"2024-09-13T14:55:51.162903+0000","last_unstale":"2024-09-13T15:06:53.874704+0000","last_undegraded":"2024-09-13T15:06:53.874704+0000","last_fullsized":"2024-09-13T15:06:53.874704+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.574773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.574773+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:39.074072+0000","scrub_duration":0.001553708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.8","version":"404'12","reported_seq":31,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.986989+0000","last_change":"2024-09-13T15:06:53.894444+0000","last_active":"2024-09-13T15:06:54.986989+0000","last_peered":"2024-09-13T15:06:54.986989+0000","last_clean":"2024-09-13T15:06:54.986989+0000","last_became_active":"2024-09-13T15:06:53.894160+0000","last_became_peered":"2024-09-13T15:06:53.894160+0000","last_unstale":"2024-09-13T15:06:54.986989+0000","last_undegraded":"2024-09-13T15:06:54.986989+0000","last_fullsized":"2024-09-13T15:06:54.986989+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:29:22.568550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":18,"num_read_kb":18,"num_write":9,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":703,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.894514+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T15:06:53.894514+0000","last_peered":"2024-09-13T15:06:53.894514+0000","last_clean":"2024-09-13T15:06:53.894514+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T15:06:53.894514+0000","last_undegraded":"2024-09-13T15:06:53.894514+0000","last_fullsized":"2024-09-13T15:06:53.894514+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":713,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.892352+0000","last_change":"2024-09-13T15:05:52.010114+0000","last_active":"2024-09-13T15:06:53.892352+0000","last_peered":"2024-09-13T15:06:53.892352+0000","last_clean":"2024-09-13T15:06:53.892352+0000","last_became_active":"2024-09-13T15:05:52.008634+0000","last_became_peered":"2024-09-13T15:05:52.008634+0000","last_unstale":"2024-09-13T15:06:53.892352+0000","last_undegraded":"2024-09-13T15:06:53.892352+0000","last_fullsized":"2024-09-13T15:06:53.892352+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:25.498296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":616,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.874646+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T15:06:53.874646+0000","last_peered":"2024-09-13T15:06:53.874646+0000","last_clean":"2024-09-13T15:06:53.874646+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T15:06:53.874646+0000","last_undegraded":"2024-09-13T15:06:53.874646+0000","last_fullsized":"2024-09-13T15:06:53.874646+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":832,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878740+0000","last_change":"2024-09-13T14:55:59.245841+0000","last_active":"2024-09-13T15:06:53.878740+0000","last_peered":"2024-09-13T15:06:53.878740+0000","last_clean":"2024-09-13T15:06:53.878740+0000","last_became_active":"2024-09-13T14:55:51.159487+0000","last_became_peered":"2024-09-13T14:55:51.159487+0000","last_unstale":"2024-09-13T15:06:53.878740+0000","last_undegraded":"2024-09-13T15:06:53.878740+0000","last_fullsized":"2024-09-13T15:06:53.878740+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.245213+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.245213+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:28.202646+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.9","version":"404'4","reported_seq":21,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987180+0000","last_change":"2024-09-13T15:06:53.899906+0000","last_active":"2024-09-13T15:06:54.987180+0000","last_peered":"2024-09-13T15:06:54.987180+0000","last_clean":"2024-09-13T15:06:54.987180+0000","last_became_active":"2024-09-13T15:06:53.899565+0000","last_became_peered":"2024-09-13T15:06:53.899565+0000","last_unstale":"2024-09-13T15:06:54.987180+0000","last_undegraded":"2024-09-13T15:06:54.987180+0000","last_fullsized":"2024-09-13T15:06:54.987180+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:45:27.111074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":5,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":189,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.881542+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T15:06:53.881542+0000","last_peered":"2024-09-13T15:06:53.881542+0000","last_clean":"2024-09-13T15:06:53.881542+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T15:06:53.881542+0000","last_undegraded":"2024-09-13T15:06:53.881542+0000","last_fullsized":"2024-09-13T15:06:53.881542+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987699+0000","last_change":"2024-09-13T15:05:52.011318+0000","last_active":"2024-09-13T15:06:54.987699+0000","last_peered":"2024-09-13T15:06:54.987699+0000","last_clean":"2024-09-13T15:06:54.987699+0000","last_became_active":"2024-09-13T15:05:52.011038+0000","last_became_peered":"2024-09-13T15:05:52.011038+0000","last_unstale":"2024-09-13T15:06:54.987699+0000","last_undegraded":"2024-09-13T15:06:54.987699+0000","last_fullsized":"2024-09-13T15:06:54.987699+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:35.033102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":931,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878242+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T15:06:53.878242+0000","last_peered":"2024-09-13T15:06:53.878242+0000","last_clean":"2024-09-13T15:06:53.878242+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T15:06:53.878242+0000","last_undegraded":"2024-09-13T15:06:53.878242+0000","last_fullsized":"2024-09-13T15:06:53.878242+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":553,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.874734+0000","last_change":"2024-09-13T14:55:58.568852+0000","last_active":"2024-09-13T15:06:53.874734+0000","last_peered":"2024-09-13T15:06:53.874734+0000","last_clean":"2024-09-13T15:06:53.874734+0000","last_became_active":"2024-09-13T14:55:51.169987+0000","last_became_peered":"2024-09-13T14:55:51.169987+0000","last_unstale":"2024-09-13T15:06:53.874734+0000","last_undegraded":"2024-09-13T15:06:53.874734+0000","last_fullsized":"2024-09-13T15:06:53.874734+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.568750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.568750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:56.273429+0000","scrub_duration":0.0074886340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.a","version":"404'14","reported_seq":34,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987648+0000","last_change":"2024-09-13T15:06:53.894229+0000","last_active":"2024-09-13T15:06:54.987648+0000","last_peered":"2024-09-13T15:06:54.987648+0000","last_clean":"2024-09-13T15:06:54.987648+0000","last_became_active":"2024-09-13T15:06:53.893833+0000","last_became_peered":"2024-09-13T15:06:53.893833+0000","last_unstale":"2024-09-13T15:06:54.987648+0000","last_undegraded":"2024-09-13T15:06:54.987648+0000","last_fullsized":"2024-09-13T15:06:54.987648+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:03.643220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":44,"num_read_kb":35,"num_write":10,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":679,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.880677+0000","last_change":"2024-09-13T15:05:52.007594+0000","last_active":"2024-09-13T15:06:53.880677+0000","last_peered":"2024-09-13T15:06:53.880677+0000","last_clean":"2024-09-13T15:06:53.880677+0000","last_became_active":"2024-09-13T15:05:52.007146+0000","last_became_peered":"2024-09-13T15:05:52.007146+0000","last_unstale":"2024-09-13T15:06:53.880677+0000","last_undegraded":"2024-09-13T15:06:53.880677+0000","last_fullsized":"2024-09-13T15:06:53.880677+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:45.283696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.986942+0000","last_change":"2024-09-13T15:05:51.991616+0000","last_active":"2024-09-13T15:06:54.986942+0000","last_peered":"2024-09-13T15:06:54.986942+0000","last_clean":"2024-09-13T15:06:54.986942+0000","last_became_active":"2024-09-13T15:05:51.991294+0000","last_became_peered":"2024-09-13T15:05:51.991294+0000","last_unstale":"2024-09-13T15:06:54.986942+0000","last_undegraded":"2024-09-13T15:06:54.986942+0000","last_fullsized":"2024-09-13T15:06:54.986942+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:05.060508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":335,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878683+0000","last_change":"2024-09-13T15:02:58.197427+0000","last_active":"2024-09-13T15:06:53.878683+0000","last_peered":"2024-09-13T15:06:53.878683+0000","last_clean":"2024-09-13T15:06:53.878683+0000","last_became_active":"2024-09-13T14:59:53.232987+0000","last_became_peered":"2024-09-13T14:59:53.232987+0000","last_unstale":"2024-09-13T15:06:53.878683+0000","last_undegraded":"2024-09-13T15:06:53.878683+0000","last_fullsized":"2024-09-13T15:06:53.878683+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:58.195731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:58.195731+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:51.565489+0000","scrub_duration":0.011077536000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.0","version":"0'0","reported_seq":11,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.893820+0000","last_change":"2024-09-13T15:06:53.892499+0000","last_active":"2024-09-13T15:06:53.893820+0000","last_peered":"2024-09-13T15:06:53.893820+0000","last_clean":"2024-09-13T15:06:53.893820+0000","last_became_active":"2024-09-13T15:06:53.892246+0000","last_became_peered":"2024-09-13T15:06:53.892246+0000","last_unstale":"2024-09-13T15:06:53.893820+0000","last_undegraded":"2024-09-13T15:06:53.893820+0000","last_fullsized":"2024-09-13T15:06:53.893820+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:39:06.349624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":389,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.877164+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T15:06:53.877164+0000","last_peered":"2024-09-13T15:06:53.877164+0000","last_clean":"2024-09-13T15:06:53.877164+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T15:06:53.877164+0000","last_undegraded":"2024-09-13T15:06:53.877164+0000","last_fullsized":"2024-09-13T15:06:53.877164+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":664,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.892256+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T15:06:53.892256+0000","last_peered":"2024-09-13T15:06:53.892256+0000","last_clean":"2024-09-13T15:06:53.892256+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T15:06:53.892256+0000","last_undegraded":"2024-09-13T15:06:53.892256+0000","last_fullsized":"2024-09-13T15:06:53.892256+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":860,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.879444+0000","last_change":"2024-09-13T15:00:10.659910+0000","last_active":"2024-09-13T15:06:53.879444+0000","last_peered":"2024-09-13T15:06:53.879444+0000","last_clean":"2024-09-13T15:06:53.879444+0000","last_became_active":"2024-09-13T14:59:53.216739+0000","last_became_peered":"2024-09-13T14:59:53.216739+0000","last_unstale":"2024-09-13T15:06:53.879444+0000","last_undegraded":"2024-09-13T15:06:53.879444+0000","last_fullsized":"2024-09-13T15:06:53.879444+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.659838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.659838+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:23:09.068190+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.1","version":"404'15","reported_seq":25,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.199759+0000","last_change":"2024-09-13T15:06:53.897550+0000","last_active":"2024-09-13T15:06:54.199759+0000","last_peered":"2024-09-13T15:06:54.199759+0000","last_clean":"2024-09-13T15:06:54.199759+0000","last_became_active":"2024-09-13T15:06:53.897345+0000","last_became_peered":"2024-09-13T15:06:53.897345+0000","last_unstale":"2024-09-13T15:06:54.199759+0000","last_undegraded":"2024-09-13T15:06:54.199759+0000","last_fullsized":"2024-09-13T15:06:54.199759+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:46:30.227958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":386,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.876723+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T15:06:53.876723+0000","last_peered":"2024-09-13T15:06:53.876723+0000","last_clean":"2024-09-13T15:06:53.876723+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T15:06:53.876723+0000","last_undegraded":"2024-09-13T15:06:53.876723+0000","last_fullsized":"2024-09-13T15:06:53.876723+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":207,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.874595+0000","last_change":"2024-09-13T15:06:10.447862+0000","last_active":"2024-09-13T15:06:53.874595+0000","last_peered":"2024-09-13T15:06:53.874595+0000","last_clean":"2024-09-13T15:06:53.874595+0000","last_became_active":"2024-09-13T15:06:10.447585+0000","last_became_peered":"2024-09-13T15:06:10.447585+0000","last_unstale":"2024-09-13T15:06:53.874595+0000","last_undegraded":"2024-09-13T15:06:53.874595+0000","last_fullsized":"2024-09-13T15:06:53.874595+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:02:14.708389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":356,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.877570+0000","last_change":"2024-09-13T15:05:52.007408+0000","last_active":"2024-09-13T15:06:53.877570+0000","last_peered":"2024-09-13T15:06:53.877570+0000","last_clean":"2024-09-13T15:06:53.877570+0000","last_became_active":"2024-09-13T15:05:52.007032+0000","last_became_peered":"2024-09-13T15:05:52.007032+0000","last_unstale":"2024-09-13T15:06:53.877570+0000","last_undegraded":"2024-09-13T15:06:53.877570+0000","last_fullsized":"2024-09-13T15:06:53.877570+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:57.138789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:57.138789+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:28:20.106072+0000","scrub_duration":0.0036861670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.2","version":"404'1","reported_seq":16,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987255+0000","last_change":"2024-09-13T15:06:53.900102+0000","last_active":"2024-09-13T15:06:54.987255+0000","last_peered":"2024-09-13T15:06:54.987255+0000","last_clean":"2024-09-13T15:06:54.987255+0000","last_became_active":"2024-09-13T15:06:53.897575+0000","last_became_peered":"2024-09-13T15:06:53.897575+0000","last_unstale":"2024-09-13T15:06:54.987255+0000","last_undegraded":"2024-09-13T15:06:54.987255+0000","last_fullsized":"2024-09-13T15:06:54.987255+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:53.146449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":409,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.875967+0000","last_change":"2024-09-13T15:05:51.998495+0000","last_active":"2024-09-13T15:06:53.875967+0000","last_peered":"2024-09-13T15:06:53.875967+0000","last_clean":"2024-09-13T15:06:53.875967+0000","last_became_active":"2024-09-13T15:05:51.998159+0000","last_became_peered":"2024-09-13T15:05:51.998159+0000","last_unstale":"2024-09-13T15:06:53.875967+0000","last_undegraded":"2024-09-13T15:06:53.875967+0000","last_fullsized":"2024-09-13T15:06:53.875967+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:23:05.948148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":58,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987563+0000","last_change":"2024-09-13T15:05:52.020378+0000","last_active":"2024-09-13T15:06:54.987563+0000","last_peered":"2024-09-13T15:06:54.987563+0000","last_clean":"2024-09-13T15:06:54.987563+0000","last_became_active":"2024-09-13T15:05:52.002686+0000","last_became_peered":"2024-09-13T15:05:52.002686+0000","last_unstale":"2024-09-13T15:06:54.987563+0000","last_undegraded":"2024-09-13T15:06:54.987563+0000","last_fullsized":"2024-09-13T15:06:54.987563+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.430732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":592,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.876174+0000","last_change":"2024-09-13T15:05:52.006880+0000","last_active":"2024-09-13T15:06:53.876174+0000","last_peered":"2024-09-13T15:06:53.876174+0000","last_clean":"2024-09-13T15:06:53.876174+0000","last_became_active":"2024-09-13T15:05:52.005704+0000","last_became_peered":"2024-09-13T15:05:52.005704+0000","last_unstale":"2024-09-13T15:06:53.876174+0000","last_undegraded":"2024-09-13T15:06:53.876174+0000","last_fullsized":"2024-09-13T15:06:53.876174+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:50.810045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:50.810045+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:39:30.729375+0000","scrub_duration":0.0042318529999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.3","version":"0'0","reported_seq":10,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.886223+0000","last_change":"2024-09-13T15:06:53.885331+0000","last_active":"2024-09-13T15:06:53.886223+0000","last_peered":"2024-09-13T15:06:53.886223+0000","last_clean":"2024-09-13T15:06:53.886223+0000","last_became_active":"2024-09-13T15:06:53.885052+0000","last_became_peered":"2024-09-13T15:06:53.885052+0000","last_unstale":"2024-09-13T15:06:53.886223+0000","last_undegraded":"2024-09-13T15:06:53.886223+0000","last_fullsized":"2024-09-13T15:06:53.886223+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:53.667168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":677,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.890895+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T15:06:53.890895+0000","last_peered":"2024-09-13T15:06:53.890895+0000","last_clean":"2024-09-13T15:06:53.890895+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T15:06:53.890895+0000","last_undegraded":"2024-09-13T15:06:53.890895+0000","last_fullsized":"2024-09-13T15:06:53.890895+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":409,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878064+0000","last_change":"2024-09-13T15:05:52.015239+0000","last_active":"2024-09-13T15:06:53.878064+0000","last_peered":"2024-09-13T15:06:53.878064+0000","last_clean":"2024-09-13T15:06:53.878064+0000","last_became_active":"2024-09-13T15:05:52.014825+0000","last_became_peered":"2024-09-13T15:05:52.014825+0000","last_unstale":"2024-09-13T15:06:53.878064+0000","last_undegraded":"2024-09-13T15:06:53.878064+0000","last_fullsized":"2024-09-13T15:06:53.878064+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:47.548807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":175,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.877002+0000","last_change":"2024-09-13T15:05:51.996175+0000","last_active":"2024-09-13T15:06:53.877002+0000","last_peered":"2024-09-13T15:06:53.877002+0000","last_clean":"2024-09-13T15:06:53.877002+0000","last_became_active":"2024-09-13T15:05:51.995536+0000","last_became_peered":"2024-09-13T15:05:51.995536+0000","last_unstale":"2024-09-13T15:06:53.877002+0000","last_undegraded":"2024-09-13T15:06:53.877002+0000","last_fullsized":"2024-09-13T15:06:53.877002+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.358699+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.358699+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:58.331262+0000","scrub_duration":0.0023994049999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.4","version":"0'0","reported_seq":10,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.900610+0000","last_change":"2024-09-13T15:06:53.900244+0000","last_active":"2024-09-13T15:06:53.900610+0000","last_peered":"2024-09-13T15:06:53.900610+0000","last_clean":"2024-09-13T15:06:53.900610+0000","last_became_active":"2024-09-13T15:06:53.899914+0000","last_became_peered":"2024-09-13T15:06:53.899914+0000","last_unstale":"2024-09-13T15:06:53.900610+0000","last_undegraded":"2024-09-13T15:06:53.900610+0000","last_fullsized":"2024-09-13T15:06:53.900610+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:44.591491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":209,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.881884+0000","last_change":"2024-09-13T15:02:38.875727+0000","last_active":"2024-09-13T15:06:53.881884+0000","last_peered":"2024-09-13T15:06:53.881884+0000","last_clean":"2024-09-13T15:06:53.881884+0000","last_became_active":"2024-09-13T15:02:38.875372+0000","last_became_peered":"2024-09-13T15:02:38.875372+0000","last_unstale":"2024-09-13T15:06:53.881884+0000","last_undegraded":"2024-09-13T15:06:53.881884+0000","last_fullsized":"2024-09-13T15:06:53.881884+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:08.504810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":381,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.875674+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T15:06:53.875674+0000","last_peered":"2024-09-13T15:06:53.875674+0000","last_clean":"2024-09-13T15:06:53.875674+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T15:06:53.875674+0000","last_undegraded":"2024-09-13T15:06:53.875674+0000","last_fullsized":"2024-09-13T15:06:53.875674+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":238,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.883009+0000","last_change":"2024-09-13T14:56:08.889974+0000","last_active":"2024-09-13T15:06:53.883009+0000","last_peered":"2024-09-13T15:06:53.883009+0000","last_clean":"2024-09-13T15:06:53.883009+0000","last_became_active":"2024-09-13T14:55:55.185755+0000","last_became_peered":"2024-09-13T14:55:55.185755+0000","last_unstale":"2024-09-13T15:06:53.883009+0000","last_undegraded":"2024-09-13T15:06:53.883009+0000","last_fullsized":"2024-09-13T15:06:53.883009+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:08.889814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:08.889814+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:02.911564+0000","scrub_duration":0.0053731569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.5","version":"0'0","reported_seq":10,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.886895+0000","last_change":"2024-09-13T15:06:53.886414+0000","last_active":"2024-09-13T15:06:53.886895+0000","last_peered":"2024-09-13T15:06:53.886895+0000","last_clean":"2024-09-13T15:06:53.886895+0000","last_became_active":"2024-09-13T15:06:53.885538+0000","last_became_peered":"2024-09-13T15:06:53.885538+0000","last_unstale":"2024-09-13T15:06:53.886895+0000","last_undegraded":"2024-09-13T15:06:53.886895+0000","last_fullsized":"2024-09-13T15:06:53.886895+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:35:11.222397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":446,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.874398+0000","last_change":"2024-09-13T15:05:52.000473+0000","last_active":"2024-09-13T15:06:53.874398+0000","last_peered":"2024-09-13T15:06:53.874398+0000","last_clean":"2024-09-13T15:06:53.874398+0000","last_became_active":"2024-09-13T15:05:52.000152+0000","last_became_peered":"2024-09-13T15:05:52.000152+0000","last_unstale":"2024-09-13T15:06:53.874398+0000","last_undegraded":"2024-09-13T15:06:53.874398+0000","last_fullsized":"2024-09-13T15:06:53.874398+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:31.530855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":371,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.876003+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T15:06:53.876003+0000","last_peered":"2024-09-13T15:06:53.876003+0000","last_clean":"2024-09-13T15:06:53.876003+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T15:06:53.876003+0000","last_undegraded":"2024-09-13T15:06:53.876003+0000","last_fullsized":"2024-09-13T15:06:53.876003+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":58,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987420+0000","last_change":"2024-09-13T15:05:52.036891+0000","last_active":"2024-09-13T15:06:54.987420+0000","last_peered":"2024-09-13T15:06:54.987420+0000","last_clean":"2024-09-13T15:06:54.987420+0000","last_became_active":"2024-09-13T15:05:52.003251+0000","last_became_peered":"2024-09-13T15:05:52.003251+0000","last_unstale":"2024-09-13T15:06:54.987420+0000","last_undegraded":"2024-09-13T15:06:54.987420+0000","last_fullsized":"2024-09-13T15:06:54.987420+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.016260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.016260+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:22.300773+0000","scrub_duration":71,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.6","version":"404'216","reported_seq":276,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.645489+0000","last_change":"2024-09-13T15:06:53.902732+0000","last_active":"2024-09-13T15:06:54.645489+0000","last_peered":"2024-09-13T15:06:54.645489+0000","last_clean":"2024-09-13T15:06:54.645489+0000","last_became_active":"2024-09-13T15:06:53.902347+0000","last_became_peered":"2024-09-13T15:06:53.902347+0000","last_unstale":"2024-09-13T15:06:54.645489+0000","last_undegraded":"2024-09-13T15:06:54.645489+0000","last_fullsized":"2024-09-13T15:06:54.645489+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":216,"log_dups_size":0,"ondisk_log_size":216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:54.547494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":556,"num_read_kb":523,"num_write":291,"num_write_kb":254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":391,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878265+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T15:06:53.878265+0000","last_peered":"2024-09-13T15:06:53.878265+0000","last_clean":"2024-09-13T15:06:53.878265+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T15:06:53.878265+0000","last_undegraded":"2024-09-13T15:06:53.878265+0000","last_fullsized":"2024-09-13T15:06:53.878265+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":667,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.882554+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T15:06:53.882554+0000","last_peered":"2024-09-13T15:06:53.882554+0000","last_clean":"2024-09-13T15:06:53.882554+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T15:06:53.882554+0000","last_undegraded":"2024-09-13T15:06:53.882554+0000","last_fullsized":"2024-09-13T15:06:53.882554+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":155,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.874816+0000","last_change":"2024-09-13T15:02:53.820519+0000","last_active":"2024-09-13T15:06:53.874816+0000","last_peered":"2024-09-13T15:06:53.874816+0000","last_clean":"2024-09-13T15:06:53.874816+0000","last_became_active":"2024-09-13T14:59:53.222306+0000","last_became_peered":"2024-09-13T14:59:53.222306+0000","last_unstale":"2024-09-13T15:06:53.874816+0000","last_undegraded":"2024-09-13T15:06:53.874816+0000","last_fullsized":"2024-09-13T15:06:53.874816+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:53.820435+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:53.820435+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:40.108691+0000","scrub_duration":0.0030873739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.7","version":"404'23","reported_seq":54,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.186840+0000","last_change":"2024-09-13T15:06:53.883824+0000","last_active":"2024-09-13T15:06:54.186840+0000","last_peered":"2024-09-13T15:06:54.186840+0000","last_clean":"2024-09-13T15:06:54.186840+0000","last_became_active":"2024-09-13T15:06:53.883538+0000","last_became_peered":"2024-09-13T15:06:53.883538+0000","last_unstale":"2024-09-13T15:06:54.186840+0000","last_undegraded":"2024-09-13T15:06:54.186840+0000","last_fullsized":"2024-09-13T15:06:54.186840+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:58.619976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":40,"num_read_kb":39,"num_write":17,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.18","version":"404'15","reported_seq":54,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.340785+0000","last_change":"2024-09-13T15:06:53.888796+0000","last_active":"2024-09-13T15:06:54.340785+0000","last_peered":"2024-09-13T15:06:54.340785+0000","last_clean":"2024-09-13T15:06:54.340785+0000","last_became_active":"2024-09-13T15:06:53.888463+0000","last_became_peered":"2024-09-13T15:06:53.888463+0000","last_unstale":"2024-09-13T15:06:54.340785+0000","last_undegraded":"2024-09-13T15:06:54.340785+0000","last_fullsized":"2024-09-13T15:06:54.340785+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:20.776742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":86,"num_read_kb":82,"num_write":12,"num_write_kb":9,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":199,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.881169+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T15:06:53.881169+0000","last_peered":"2024-09-13T15:06:53.881169+0000","last_clean":"2024-09-13T15:06:53.881169+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T15:06:53.881169+0000","last_undegraded":"2024-09-13T15:06:53.881169+0000","last_fullsized":"2024-09-13T15:06:53.881169+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.988055+0000","last_change":"2024-09-13T15:05:51.999191+0000","last_active":"2024-09-13T15:06:54.988055+0000","last_peered":"2024-09-13T15:06:54.988055+0000","last_clean":"2024-09-13T15:06:54.988055+0000","last_became_active":"2024-09-13T15:05:51.998885+0000","last_became_peered":"2024-09-13T15:05:51.998885+0000","last_unstale":"2024-09-13T15:06:54.988055+0000","last_undegraded":"2024-09-13T15:06:54.988055+0000","last_fullsized":"2024-09-13T15:06:54.988055+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:52.548839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":135,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878807+0000","last_change":"2024-09-13T14:56:03.605447+0000","last_active":"2024-09-13T15:06:53.878807+0000","last_peered":"2024-09-13T15:06:53.878807+0000","last_clean":"2024-09-13T15:06:53.878807+0000","last_became_active":"2024-09-13T14:55:55.218939+0000","last_became_peered":"2024-09-13T14:55:55.218939+0000","last_unstale":"2024-09-13T15:06:53.878807+0000","last_undegraded":"2024-09-13T15:06:53.878807+0000","last_fullsized":"2024-09-13T15:06:53.878807+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.605356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.605356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:12.467371+0000","scrub_duration":0.0020269620000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"404'7","reported_seq":20,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.353632+0000","last_change":"2024-09-13T15:06:53.900834+0000","last_active":"2024-09-13T15:06:54.353632+0000","last_peered":"2024-09-13T15:06:54.353632+0000","last_clean":"2024-09-13T15:06:54.353632+0000","last_became_active":"2024-09-13T15:06:53.900506+0000","last_became_peered":"2024-09-13T15:06:53.900506+0000","last_unstale":"2024-09-13T15:06:54.353632+0000","last_undegraded":"2024-09-13T15:06:54.353632+0000","last_fullsized":"2024-09-13T15:06:54.353632+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:11.476388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":13,"num_read_kb":10,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":372,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.876369+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T15:06:53.876369+0000","last_peered":"2024-09-13T15:06:53.876369+0000","last_clean":"2024-09-13T15:06:53.876369+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T15:06:53.876369+0000","last_undegraded":"2024-09-13T15:06:53.876369+0000","last_fullsized":"2024-09-13T15:06:53.876369+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":704,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.894366+0000","last_change":"2024-09-13T15:05:52.008360+0000","last_active":"2024-09-13T15:06:53.894366+0000","last_peered":"2024-09-13T15:06:53.894366+0000","last_clean":"2024-09-13T15:06:53.894366+0000","last_became_active":"2024-09-13T15:05:52.007789+0000","last_became_peered":"2024-09-13T15:05:52.007789+0000","last_unstale":"2024-09-13T15:06:53.894366+0000","last_undegraded":"2024-09-13T15:06:53.894366+0000","last_fullsized":"2024-09-13T15:06:53.894366+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:12:26.443562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":851,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.879288+0000","last_change":"2024-09-13T15:05:52.009899+0000","last_active":"2024-09-13T15:06:53.879288+0000","last_peered":"2024-09-13T15:06:53.879288+0000","last_clean":"2024-09-13T15:06:53.879288+0000","last_became_active":"2024-09-13T15:05:52.009365+0000","last_became_peered":"2024-09-13T15:05:52.009365+0000","last_unstale":"2024-09-13T15:06:53.879288+0000","last_undegraded":"2024-09-13T15:06:53.879288+0000","last_fullsized":"2024-09-13T15:06:53.879288+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.240882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.240882+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:02:12.573230+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.1a","version":"0'0","reported_seq":10,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.888846+0000","last_change":"2024-09-13T15:06:53.887834+0000","last_active":"2024-09-13T15:06:53.888846+0000","last_peered":"2024-09-13T15:06:53.888846+0000","last_clean":"2024-09-13T15:06:53.888846+0000","last_became_active":"2024-09-13T15:06:53.887386+0000","last_became_peered":"2024-09-13T15:06:53.887386+0000","last_unstale":"2024-09-13T15:06:53.888846+0000","last_undegraded":"2024-09-13T15:06:53.888846+0000","last_fullsized":"2024-09-13T15:06:53.888846+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:21.862562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":711,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.890486+0000","last_change":"2024-09-13T15:05:52.011548+0000","last_active":"2024-09-13T15:06:53.890486+0000","last_peered":"2024-09-13T15:06:53.890486+0000","last_clean":"2024-09-13T15:06:53.890486+0000","last_became_active":"2024-09-13T15:05:52.009924+0000","last_became_peered":"2024-09-13T15:05:52.009924+0000","last_unstale":"2024-09-13T15:06:53.890486+0000","last_undegraded":"2024-09-13T15:06:53.890486+0000","last_fullsized":"2024-09-13T15:06:53.890486+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:49.485860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":388,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878559+0000","last_change":"2024-09-13T15:05:52.014781+0000","last_active":"2024-09-13T15:06:53.878559+0000","last_peered":"2024-09-13T15:06:53.878559+0000","last_clean":"2024-09-13T15:06:53.878559+0000","last_became_active":"2024-09-13T15:05:52.014049+0000","last_became_peered":"2024-09-13T15:05:52.014049+0000","last_unstale":"2024-09-13T15:06:53.878559+0000","last_undegraded":"2024-09-13T15:06:53.878559+0000","last_fullsized":"2024-09-13T15:06:53.878559+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:33.528483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":135,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.878172+0000","last_change":"2024-09-13T14:56:04.643672+0000","last_active":"2024-09-13T15:06:53.878172+0000","last_peered":"2024-09-13T15:06:53.878172+0000","last_clean":"2024-09-13T15:06:53.878172+0000","last_became_active":"2024-09-13T14:55:55.185995+0000","last_became_peered":"2024-09-13T14:55:55.185995+0000","last_unstale":"2024-09-13T15:06:53.878172+0000","last_undegraded":"2024-09-13T15:06:53.878172+0000","last_fullsized":"2024-09-13T15:06:53.878172+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.643614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.643614+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:36.495546+0000","scrub_duration":0.001441681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1b","version":"0'0","reported_seq":10,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.889346+0000","last_change":"2024-09-13T15:06:53.888925+0000","last_active":"2024-09-13T15:06:53.889346+0000","last_peered":"2024-09-13T15:06:53.889346+0000","last_clean":"2024-09-13T15:06:53.889346+0000","last_became_active":"2024-09-13T15:06:53.888351+0000","last_became_peered":"2024-09-13T15:06:53.888351+0000","last_unstale":"2024-09-13T15:06:53.889346+0000","last_undegraded":"2024-09-13T15:06:53.889346+0000","last_fullsized":"2024-09-13T15:06:53.889346+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:06:52.865607+0000","last_clean_scrub_stamp":"2024-09-13T15:06:52.865607+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:52:30.489382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":371,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.876041+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T15:06:53.876041+0000","last_peered":"2024-09-13T15:06:53.876041+0000","last_clean":"2024-09-13T15:06:53.876041+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T15:06:53.876041+0000","last_undegraded":"2024-09-13T15:06:53.876041+0000","last_fullsized":"2024-09-13T15:06:53.876041+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-09-13T15:06:54.987516+0000","last_change":"2024-09-13T15:05:52.015281+0000","last_active":"2024-09-13T15:06:54.987516+0000","last_peered":"2024-09-13T15:06:54.987516+0000","last_clean":"2024-09-13T15:06:54.987516+0000","last_became_active":"2024-09-13T15:05:52.014517+0000","last_became_peered":"2024-09-13T15:05:52.014517+0000","last_unstale":"2024-09-13T15:06:54.987516+0000","last_undegraded":"2024-09-13T15:06:54.987516+0000","last_fullsized":"2024-09-13T15:06:54.987516+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:10:37.226490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"82'100","reported_seq":420,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-09-13T15:06:53.892444+0000","last_change":"2024-09-13T15:05:52.008756+0000","last_active":"2024-09-13T15:06:53.892444+0000","last_peered":"2024-09-13T15:06:53.892444+0000","last_clean":"2024-09-13T15:06:53.892444+0000","last_became_active":"2024-09-13T15:05:52.007609+0000","last_became_peered":"2024-09-13T15:05:52.007609+0000","last_unstale":"2024-09-13T15:06:53.892444+0000","last_undegraded":"2024-09-13T15:06:53.892444+0000","last_fullsized":"2024-09-13T15:06:53.892444+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.087994+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.087994+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:33.077541+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":32,"stat_sum":{"num_bytes":332,"num_objects":31,"num_object_clones":0,"num_object_copies":93,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31,"num_whiteouts":0,"num_read":1242,"num_read_kb":9334,"num_write":472,"num_write_kb":4530,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":24,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":73728,"data_stored":1004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":443,"ondisk_log_size":443,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":16262,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24498,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397996,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":887320,"kb_used_data":6200,"kb_used_omap":1,"kb_used_meta":881086,"kb_avail":92857832,"statfs":{"total":95995035648,"available":95086419968,"internally_reserved":0,"allocated":6348800,"data_stored":3504315,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":902232408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594225,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":320200,"kb_used_data":10356,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93424952,"statfs":{"total":95995035648,"available":95667150848,"internally_reserved":0,"allocated":10604544,"data_stored":7763381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790454,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1195712,"kb_used_data":10272,"kb_used_omap":1,"kb_used_meta":1185406,"kb_avail":92549440,"statfs":{"total":95995035648,"available":94770626560,"internally_reserved":0,"allocated":10518528,"data_stored":7694030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1213856088},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986683,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907328,"kb_used_data":5932,"kb_used_omap":1,"kb_used_meta":901374,"kb_avail":92837824,"statfs":{"total":95995035648,"available":95065931776,"internally_reserved":0,"allocated":6074368,"data_stored":3236891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923007320},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":335,"seq":1438814044230,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":123640,"kb_used_data":6232,"kb_used_omap":10,"kb_used_meta":117365,"kb_avail":93621512,"statfs":{"total":95995035648,"available":95868428288,"internally_reserved":0,"allocated":6381568,"data_stored":3564215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10273,"internal_metadata":120182751},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762657,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1055452,"kb_used_data":6160,"kb_used_omap":8,"kb_used_meta":1049271,"kb_avail":92689700,"statfs":{"total":95995035648,"available":94914252800,"internally_reserved":0,"allocated":6307840,"data_stored":3490766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":1074453962},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926183,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1006724,"kb_used_data":9976,"kb_used_omap":7,"kb_used_meta":996728,"kb_avail":92738428,"statfs":{"total":95995035648,"available":94964150272,"internally_reserved":0,"allocated":10215424,"data_stored":7413000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":1020650185},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122423,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1039720,"kb_used_data":6232,"kb_used_omap":7,"kb_used_meta":1033464,"kb_avail":92705432,"statfs":{"total":95995035648,"available":94930362368,"internally_reserved":0,"allocated":6381568,"data_stored":3560712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":1058267727},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7266,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":58,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:07:00.315 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 31 2024-09-13T15:07:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:07:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:07:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:07:00.512 INFO:teuthology.orchestra.run.smithi096.stdout:osd.0: {} 2024-09-13T15:07:00.512 INFO:teuthology.orchestra.run.smithi096.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:07:00.520 INFO:teuthology.orchestra.run.smithi096.stdout:osd.1: {} 2024-09-13T15:07:00.521 INFO:teuthology.orchestra.run.smithi096.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:07:00.537 INFO:teuthology.orchestra.run.smithi096.stdout:osd.2: {} 2024-09-13T15:07:00.537 INFO:teuthology.orchestra.run.smithi096.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:07:00.549 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:00.558 INFO:teuthology.orchestra.run.smithi096.stdout:osd.3: {} 2024-09-13T15:07:00.558 INFO:teuthology.orchestra.run.smithi096.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:07:00.577 INFO:teuthology.orchestra.run.smithi096.stdout:osd.4: {} 2024-09-13T15:07:00.577 INFO:teuthology.orchestra.run.smithi096.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:07:00.588 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:00.594 INFO:teuthology.orchestra.run.smithi096.stdout:osd.5: {} 2024-09-13T15:07:00.594 INFO:teuthology.orchestra.run.smithi096.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:07:00.612 INFO:teuthology.orchestra.run.smithi096.stdout:osd.6: {} 2024-09-13T15:07:00.612 INFO:teuthology.orchestra.run.smithi096.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:07:00.628 INFO:teuthology.orchestra.run.smithi096.stdout:osd.7: {} 2024-09-13T15:07:00.628 INFO:teuthology.orchestra.run.smithi096.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:07:01.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:00 smithi204 ceph-mon[87071]: osdmap e408: 8 total, 8 up, 8 in 2024-09-13T15:07:01.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:00 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1554251467' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:01.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:00 smithi096 ceph-mon[98901]: osdmap e408: 8 total, 8 up, 8 in 2024-09-13T15:07:01.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:00 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1554251467' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:01.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:00 smithi096 ceph-mon[102928]: osdmap e408: 8 total, 8 up, 8 in 2024-09-13T15:07:01.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:00 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1554251467' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:01.956 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:02.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:01 smithi204 ceph-mon[87071]: pgmap v1361: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:02.259 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:01 smithi096 ceph-mon[98901]: pgmap v1361: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:02.260 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:01 smithi096 ceph-mon[102928]: pgmap v1361: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:04.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:03 smithi204 ceph-mon[87071]: pgmap v1362: 103 pgs: 103 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:04.314 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:03 smithi096 ceph-mon[98901]: pgmap v1362: 103 pgs: 103 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:04.314 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:03 smithi096 ceph-mon[102928]: pgmap v1362: 103 pgs: 103 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:05.951 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is set 2024-09-13T15:07:05.951 INFO:teuthology.orchestra.run.smithi096.stderr:set pool 1 pgp_num to 31 2024-09-13T15:07:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:05 smithi096 ceph-mon[102928]: pgmap v1363: 103 pgs: 103 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:05 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3305079976' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:05 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:05 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4264129299' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T15:07:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:05 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T15:07:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:05 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:07:06.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:05 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:06.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:05 smithi096 ceph-mon[98901]: pgmap v1363: 103 pgs: 103 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:06.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:05 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3305079976' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:06.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:05 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:06.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:05 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4264129299' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T15:07:06.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:05 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T15:07:06.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:05 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:07:06.074 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:05 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:06.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:05 smithi204 ceph-mon[87071]: pgmap v1363: 103 pgs: 103 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:06.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:05 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3305079976' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:06.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:05 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:06.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:05 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4264129299' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T15:07:06.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:05 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T15:07:06.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:05 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:07:06.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:05 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:06 smithi204 ceph-mon[87071]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:06 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:06 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-13T15:07:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:06 smithi204 ceph-mon[87071]: osdmap e409: 8 total, 8 up, 8 in 2024-09-13T15:07:07.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:07:07.275 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:06 smithi096 ceph-mon[98901]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:07.275 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:06 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:07.275 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:06 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-13T15:07:07.275 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:06 smithi096 ceph-mon[98901]: osdmap e409: 8 total, 8 up, 8 in 2024-09-13T15:07:07.275 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:07:07.276 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:06 smithi096 ceph-mon[102928]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:07.276 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:06 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:07.276 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:06 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-13T15:07:07.276 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:06 smithi096 ceph-mon[102928]: osdmap e409: 8 total, 8 up, 8 in 2024-09-13T15:07:07.276 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:07:07.972 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:07.973 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:08.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:07 smithi096 ceph-mon[98901]: pgmap v1365: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:08.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:07 smithi096 ceph-mon[102928]: pgmap v1365: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:08.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:07:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:07:07] "GET /metrics HTTP/1.1" 200 34160 "" "Prometheus/2.51.0" 2024-09-13T15:07:08.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:07 smithi204 ceph-mon[87071]: pgmap v1365: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:08.654 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T15:07:08.898 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:09.778 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:07:10.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:09 smithi096 ceph-mon[98901]: pgmap v1366: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:10.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:09 smithi096 ceph-mon[102928]: pgmap v1366: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:10.213 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:09 smithi204 ceph-mon[87071]: pgmap v1366: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:10.218 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:07:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:07:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:07:11.330 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:11.330 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:11.955 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T15:07:12.213 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:12.218 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:11 smithi204 ceph-mon[87071]: pgmap v1367: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:12.305 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2024-09-13T15:07:12.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:11 smithi096 ceph-mon[98901]: pgmap v1367: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:12.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:11 smithi096 ceph-mon[102928]: pgmap v1367: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:12.969 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:13.874 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:13.874 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:13.957 INFO:teuthology.orchestra.run.smithi096.stderr:noscrub is unset 2024-09-13T15:07:14.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:13 smithi096 ceph-mon[98901]: pgmap v1368: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:14.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:13 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3636388628' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:07:14.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:13 smithi096 ceph-mon[102928]: pgmap v1368: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:14.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:13 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3636388628' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:07:14.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:13 smithi204 ceph-mon[87071]: pgmap v1368: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:14.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:13 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3636388628' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:07:14.469 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T15:07:14.718 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:14.811 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:07:15.125 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:14 smithi096 ceph-mon[98901]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:15.126 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:14 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3636388628' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:07:15.126 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:14 smithi096 ceph-mon[98901]: osdmap e410: 8 total, 8 up, 8 in 2024-09-13T15:07:15.126 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:14 smithi096 ceph-mon[102928]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:15.126 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:14 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3636388628' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:07:15.126 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:14 smithi096 ceph-mon[102928]: osdmap e410: 8 total, 8 up, 8 in 2024-09-13T15:07:15.165 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:14 smithi204 ceph-mon[87071]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:15.165 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:14 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3636388628' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:07:15.165 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:14 smithi204 ceph-mon[87071]: osdmap e410: 8 total, 8 up, 8 in 2024-09-13T15:07:15.221 INFO:teuthology.orchestra.run.smithi096.stdout:mon.a: {} 2024-09-13T15:07:15.221 INFO:teuthology.orchestra.run.smithi096.stdout:mon.b: {} 2024-09-13T15:07:15.222 INFO:teuthology.orchestra.run.smithi096.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T15:07:15.222 INFO:teuthology.orchestra.run.smithi096.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T15:07:15.227 INFO:teuthology.orchestra.run.smithi096.stdout:mon.c: {} 2024-09-13T15:07:15.227 INFO:teuthology.orchestra.run.smithi096.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T15:07:15.944 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T15:07:15.967 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:16.041 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:15 smithi204 ceph-mon[87071]: pgmap v1370: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:16.041 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:15 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2805576699' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T15:07:16.222 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:15 smithi096 ceph-mon[98901]: pgmap v1370: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:16.222 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:15 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2805576699' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T15:07:16.223 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:15 smithi096 ceph-mon[102928]: pgmap v1370: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:16.223 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:15 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2805576699' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T15:07:16.280 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:16.280 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:16.934 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T15:07:17.141 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:17.179 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:18.217 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:17 smithi096 ceph-mon[98901]: pgmap v1371: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:18.217 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:17 smithi096 ceph-mon[102928]: pgmap v1371: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:18.217 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:07:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:07:17] "GET /metrics HTTP/1.1" 200 34160 "" "Prometheus/2.51.0" 2024-09-13T15:07:18.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:17 smithi204 ceph-mon[87071]: pgmap v1371: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:19.063 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:19.063 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:19.735 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T15:07:19.979 INFO:teuthology.orchestra.run.smithi096.stderr:nodeep-scrub is unset 2024-09-13T15:07:19.981 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T15:07:19.990 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:20.225 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:19 smithi096 ceph-mon[98901]: pgmap v1372: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:20.225 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:19 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1597217475' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:07:20.225 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:19 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:07:20.225 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:19 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/563627650' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:20.225 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:19 smithi096 ceph-mon[102928]: pgmap v1372: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:20.225 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:19 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1597217475' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:07:20.225 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:19 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:07:20.225 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:19 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/563627650' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:20.236 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:19 smithi204 ceph-mon[87071]: pgmap v1372: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:20.236 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:19 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1597217475' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:07:20.236 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:19 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:07:20.236 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:19 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/563627650' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:20.236 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:07:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:07:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:07:20.878 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 0 1 2024-09-13T15:07:21.067 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:20 smithi204 ceph-mon[87071]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:07:21.067 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:20 smithi204 ceph-mon[87071]: Cluster is now healthy 2024-09-13T15:07:21.067 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:20 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:07:21.067 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:20 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/563627650' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:21.067 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:20 smithi204 ceph-mon[87071]: osdmap e411: 8 total, 8 up, 8 in 2024-09-13T15:07:21.067 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:20 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:07:21.068 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:20 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:21.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:20 smithi096 ceph-mon[98901]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:07:21.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:20 smithi096 ceph-mon[98901]: Cluster is now healthy 2024-09-13T15:07:21.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:20 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:07:21.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:20 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/563627650' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:21.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:20 smithi096 ceph-mon[98901]: osdmap e411: 8 total, 8 up, 8 in 2024-09-13T15:07:21.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:20 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:07:21.324 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:20 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:21.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:20 smithi096 ceph-mon[102928]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:07:21.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:20 smithi096 ceph-mon[102928]: Cluster is now healthy 2024-09-13T15:07:21.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:20 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:07:21.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:20 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/563627650' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:21.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:20 smithi096 ceph-mon[102928]: osdmap e411: 8 total, 8 up, 8 in 2024-09-13T15:07:21.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:20 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:07:21.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:20 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:21.505 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:21.590 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:21.591 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:22.192 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:21 smithi204 ceph-mon[87071]: pgmap v1374: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:22.216 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T15:07:22.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:21 smithi096 ceph-mon[98901]: pgmap v1374: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:22.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:21 smithi096 ceph-mon[102928]: pgmap v1374: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:22.463 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:23.995 INFO:teuthology.orchestra.run.smithi096.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-09-13T15:07:24.214 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:24.214 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:24.318 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:23 smithi096 ceph-mon[98901]: pgmap v1375: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:24.318 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:23 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/729483581' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:07:24.318 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:23 smithi096 ceph-mon[102928]: pgmap v1375: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:24.318 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:23 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/729483581' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:07:24.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:23 smithi204 ceph-mon[87071]: pgmap v1375: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:24.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:23 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/729483581' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:07:24.658 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T15:07:24.902 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T15:07:24.910 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:25.156 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:25.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:25 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/729483581' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:07:25.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:25 smithi204 ceph-mon[87071]: osdmap e412: 8 total, 8 up, 8 in 2024-09-13T15:07:25.259 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:25 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/729483581' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:07:25.259 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:25 smithi096 ceph-mon[98901]: osdmap e412: 8 total, 8 up, 8 in 2024-09-13T15:07:25.259 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:25 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/729483581' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:07:25.259 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:25 smithi096 ceph-mon[102928]: osdmap e412: 8 total, 8 up, 8 in 2024-09-13T15:07:26.304 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:26 smithi096 ceph-mon[98901]: pgmap v1377: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:26.304 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:26 smithi096 ceph-mon[102928]: pgmap v1377: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:26.313 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:26 smithi204 ceph-mon[87071]: pgmap v1377: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:26.761 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:26.761 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:27.023 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T15:07:27.311 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:27 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2120189757' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:07:27.311 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:27 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:07:27.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:27 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2120189757' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:07:27.322 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:27 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:07:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:27 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2120189757' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:07:27.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:27 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:07:27.422 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T15:07:27.670 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:27.701 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 1 1 2024-09-13T15:07:27.943 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:28.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:28 smithi204 ceph-mon[87071]: pgmap v1378: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:28.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:28 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:07:28.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:28 smithi204 ceph-mon[87071]: osdmap e413: 8 total, 8 up, 8 in 2024-09-13T15:07:28.224 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:28 smithi096 ceph-mon[98901]: pgmap v1378: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:28.224 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:28 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:07:28.224 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:28 smithi096 ceph-mon[98901]: osdmap e413: 8 total, 8 up, 8 in 2024-09-13T15:07:28.225 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:28 smithi096 ceph-mon[102928]: pgmap v1378: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:28.225 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:28 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:07:28.225 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:28 smithi096 ceph-mon[102928]: osdmap e413: 8 total, 8 up, 8 in 2024-09-13T15:07:28.225 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:07:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:07:27] "GET /metrics HTTP/1.1" 200 34158 "" "Prometheus/2.51.0" 2024-09-13T15:07:29.203 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:29 smithi096 ceph-mon[98901]: pgmap v1380: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:29.203 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:29 smithi096 ceph-mon[102928]: pgmap v1380: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:29.302 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:29 smithi204 ceph-mon[87071]: pgmap v1380: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:29.580 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:29.581 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:30.032 INFO:teuthology.orchestra.run.smithi096.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-09-13T15:07:30.173 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:30 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1721825187' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:07:30.174 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:30 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:07:30.199 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T15:07:30.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:30 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1721825187' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:07:30.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:30 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:07:30.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:30 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1721825187' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:07:30.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:30 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:07:30.448 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:07:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:07:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:07:30.686 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T15:07:30.933 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:31.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:31 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:07:31.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:31 smithi204 ceph-mon[87071]: osdmap e414: 8 total, 8 up, 8 in 2024-09-13T15:07:31.219 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:31 smithi204 ceph-mon[87071]: pgmap v1382: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:31.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:31 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:07:31.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:31 smithi096 ceph-mon[98901]: osdmap e414: 8 total, 8 up, 8 in 2024-09-13T15:07:31.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:31 smithi096 ceph-mon[98901]: pgmap v1382: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:31.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:31 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:07:31.323 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:31 smithi096 ceph-mon[102928]: osdmap e414: 8 total, 8 up, 8 in 2024-09-13T15:07:31.324 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:31 smithi096 ceph-mon[102928]: pgmap v1382: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:32.034 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:32.034 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:32.655 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T15:07:32.902 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:33.331 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T15:07:33.572 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:33 smithi096 ceph-mon[98901]: pgmap v1383: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:33.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:33 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1162507289' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:07:33.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:33 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:07:33.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:33 smithi096 ceph-mon[102928]: pgmap v1383: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:33.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:33 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1162507289' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:07:33.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:33 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:07:33.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:33 smithi204 ceph-mon[87071]: pgmap v1383: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:33.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:33 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1162507289' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:07:33.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:33 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:07:33.959 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 2 1 2024-09-13T15:07:34.204 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:34.456 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:34 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:07:34.457 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:34 smithi204 ceph-mon[87071]: osdmap e415: 8 total, 8 up, 8 in 2024-09-13T15:07:34.663 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:34 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:07:34.663 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:34 smithi096 ceph-mon[98901]: osdmap e415: 8 total, 8 up, 8 in 2024-09-13T15:07:34.663 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:34 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:07:34.663 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:34 smithi096 ceph-mon[102928]: osdmap e415: 8 total, 8 up, 8 in 2024-09-13T15:07:34.691 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:34.691 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:35.334 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T15:07:35.583 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:35.677 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:35 smithi096 ceph-mon[98901]: pgmap v1385: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:35.678 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:35 smithi096 ceph-mon[102928]: pgmap v1385: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:35.704 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:35 smithi204 ceph-mon[87071]: pgmap v1385: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:36.434 INFO:teuthology.orchestra.run.smithi096.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-09-13T15:07:36.694 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:36.695 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:36 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1219626745' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:07:36.695 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:36 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:07:36.725 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:36.725 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:36 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1219626745' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:07:36.725 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:36 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:07:36.726 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:36.726 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:36 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1219626745' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:07:36.726 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:36 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:07:37.154 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T15:07:37.292 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:37.292 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:37.405 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:37.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:37 smithi204 ceph-mon[87071]: pgmap v1386: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:37.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:37 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:07:37.720 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:37 smithi204 ceph-mon[87071]: osdmap e416: 8 total, 8 up, 8 in 2024-09-13T15:07:37.805 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:37 smithi096 ceph-mon[98901]: pgmap v1386: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:37.805 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:37 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:07:37.805 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:37 smithi096 ceph-mon[98901]: osdmap e416: 8 total, 8 up, 8 in 2024-09-13T15:07:37.806 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:37 smithi096 ceph-mon[102928]: pgmap v1386: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:37.806 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:37 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:07:37.806 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:37 smithi096 ceph-mon[102928]: osdmap e416: 8 total, 8 up, 8 in 2024-09-13T15:07:37.970 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T15:07:38.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:07:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:07:37] "GET /metrics HTTP/1.1" 200 34158 "" "Prometheus/2.51.0" 2024-09-13T15:07:38.216 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:38.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:38 smithi096 ceph-mon[98901]: osdmap e417: 8 total, 8 up, 8 in 2024-09-13T15:07:38.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:38 smithi096 ceph-mon[102928]: osdmap e417: 8 total, 8 up, 8 in 2024-09-13T15:07:38.598 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:38 smithi204 ceph-mon[87071]: osdmap e417: 8 total, 8 up, 8 in 2024-09-13T15:07:39.445 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T15:07:39.676 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:39 smithi204 ceph-mon[87071]: pgmap v1389: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:39.677 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:39 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3235410493' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:07:39.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:39 smithi096 ceph-mon[98901]: pgmap v1389: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:39.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:39 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3235410493' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:07:39.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:39 smithi096 ceph-mon[102928]: pgmap v1389: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:39.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:39 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3235410493' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:07:39.897 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:39.897 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:40.103 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 4 1 2024-09-13T15:07:40.346 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:40 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3235410493' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:07:40.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:40 smithi204 ceph-mon[87071]: osdmap e418: 8 total, 8 up, 8 in 2024-09-13T15:07:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:07:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:07:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:07:40.559 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T15:07:40.714 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:40 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3235410493' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:07:40.714 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:40 smithi096 ceph-mon[98901]: osdmap e418: 8 total, 8 up, 8 in 2024-09-13T15:07:40.714 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:40 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3235410493' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:07:40.715 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:40 smithi096 ceph-mon[102928]: osdmap e418: 8 total, 8 up, 8 in 2024-09-13T15:07:40.804 INFO:teuthology.orchestra.run.smithi204.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:41.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:41 smithi204 ceph-mon[87071]: pgmap v1391: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:41.799 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:41 smithi096 ceph-mon[98901]: pgmap v1391: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:41.799 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:41 smithi096 ceph-mon[102928]: pgmap v1391: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:42.467 INFO:teuthology.orchestra.run.smithi096.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-09-13T15:07:42.537 INFO:teuthology.orchestra.run.smithi204.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:42.538 INFO:teuthology.orchestra.run.smithi204.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:42.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:42 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/579694495' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:07:42.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:42 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:07:42.777 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:42 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/579694495' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:07:42.777 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:42 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:07:42.777 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:42 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/579694495' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:07:42.777 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:42 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:07:43.167 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T15:07:43.180 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T15:07:43.431 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:43.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:43 smithi204 ceph-mon[87071]: pgmap v1392: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:43.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:43 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:07:43.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:43 smithi204 ceph-mon[87071]: osdmap e419: 8 total, 8 up, 8 in 2024-09-13T15:07:43.780 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:43 smithi096 ceph-mon[98901]: pgmap v1392: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:43.781 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:43 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:07:43.781 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:43 smithi096 ceph-mon[98901]: osdmap e419: 8 total, 8 up, 8 in 2024-09-13T15:07:43.781 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:43 smithi096 ceph-mon[102928]: pgmap v1392: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:43.781 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:43 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:07:43.781 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:43 smithi096 ceph-mon[102928]: osdmap e419: 8 total, 8 up, 8 in 2024-09-13T15:07:44.110 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:45.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:45 smithi096 ceph-mon[98901]: pgmap v1394: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:45.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:45 smithi096 ceph-mon[102928]: pgmap v1394: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:45.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:45 smithi204 ceph-mon[87071]: pgmap v1394: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:46.475 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T15:07:46.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:46 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3943535489' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:07:46.795 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:46 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3943535489' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:07:46.795 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:46 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3943535489' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:07:47.136 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 5 1 2024-09-13T15:07:47.378 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:47.378 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:47.718 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:47 smithi096 ceph-mon[98901]: pgmap v1395: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:47.718 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:47 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3943535489' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:07:47.718 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:47 smithi096 ceph-mon[98901]: osdmap e420: 8 total, 8 up, 8 in 2024-09-13T15:07:47.719 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:47 smithi096 ceph-mon[102928]: pgmap v1395: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:47.719 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:47 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3943535489' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:07:47.719 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:47 smithi096 ceph-mon[102928]: osdmap e420: 8 total, 8 up, 8 in 2024-09-13T15:07:47.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:47 smithi204 ceph-mon[87071]: pgmap v1395: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:47.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:47 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3943535489' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:07:47.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:47 smithi204 ceph-mon[87071]: osdmap e420: 8 total, 8 up, 8 in 2024-09-13T15:07:48.073 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:07:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:07:47] "GET /metrics HTTP/1.1" 200 34158 "" "Prometheus/2.51.0" 2024-09-13T15:07:48.100 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:48.105 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T15:07:49.566 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:49.767 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:49 smithi096 ceph-mon[98901]: pgmap v1397: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:49.767 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:49 smithi096 ceph-mon[102928]: pgmap v1397: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:49.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:49 smithi204 ceph-mon[87071]: pgmap v1397: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:07:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:07:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:07:51.487 INFO:teuthology.orchestra.run.smithi096.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-09-13T15:07:51.586 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:51.586 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:51.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:51 smithi096 ceph-mon[98901]: pgmap v1398: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:51.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:51 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3290797815' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:07:51.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:51.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:51 smithi096 ceph-mon[102928]: pgmap v1398: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:51.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:51 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3290797815' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:07:51.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:51.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:51 smithi204 ceph-mon[87071]: pgmap v1398: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:51.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:51 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3290797815' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:07:51.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:52.548 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T15:07:52.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:52 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3290797815' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:07:52.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:52 smithi096 ceph-mon[98901]: osdmap e421: 8 total, 8 up, 8 in 2024-09-13T15:07:52.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:52 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3290797815' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:07:52.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:52 smithi096 ceph-mon[102928]: osdmap e421: 8 total, 8 up, 8 in 2024-09-13T15:07:52.871 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:52 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3290797815' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:07:52.871 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:52 smithi204 ceph-mon[87071]: osdmap e421: 8 total, 8 up, 8 in 2024-09-13T15:07:52.953 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T15:07:53.029 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:53.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:53 smithi096 ceph-mon[98901]: pgmap v1400: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:53.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:53 smithi096 ceph-mon[102928]: pgmap v1400: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:53.701 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:53.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:53 smithi204 ceph-mon[87071]: pgmap v1400: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:55.776 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:55 smithi096 ceph-mon[98901]: pgmap v1401: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:55.777 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:55 smithi096 ceph-mon[102928]: pgmap v1401: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:55.905 INFO:teuthology.orchestra.run.smithi096.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:55.905 INFO:teuthology.orchestra.run.smithi096.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:55 smithi204 ceph-mon[87071]: pgmap v1401: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:56.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:56 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:07:56.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:56 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:07:56.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:56 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:07:57.504 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T15:07:57.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:57 smithi096 ceph-mon[98901]: pgmap v1402: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:57.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:57 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/182160646' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:07:57.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:57 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:07:57.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:57 smithi096 ceph-mon[102928]: pgmap v1402: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:57.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:57 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/182160646' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:07:57.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:57 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:07:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:57 smithi204 ceph-mon[87071]: pgmap v1402: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:57 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/182160646' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:07:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:57 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:07:58.054 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:07:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:07:57] "GET /metrics HTTP/1.1" 200 34160 "" "Prometheus/2.51.0" 2024-09-13T15:07:58.498 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 6 1 2024-09-13T15:07:58.771 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:58.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:58 smithi096 ceph-mon[98901]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:07:58.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:58 smithi096 ceph-mon[98901]: osdmap e422: 8 total, 8 up, 8 in 2024-09-13T15:07:58.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:58 smithi096 ceph-mon[102928]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:07:58.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:58 smithi096 ceph-mon[102928]: osdmap e422: 8 total, 8 up, 8 in 2024-09-13T15:07:58.925 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:58 smithi204 ceph-mon[87071]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:07:58.925 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:58 smithi204 ceph-mon[87071]: osdmap e422: 8 total, 8 up, 8 in 2024-09-13T15:07:59.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:07:59 smithi096 ceph-mon[98901]: pgmap v1404: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:59.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:07:59 smithi096 ceph-mon[102928]: pgmap v1404: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:07:59.896 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:07:59 smithi204 ceph-mon[87071]: pgmap v1404: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:08:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:08:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:08:01.388 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:01 smithi096 ceph-mon[98901]: pgmap v1405: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:01.388 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:01 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:08:01.389 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:01 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:08:01.389 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:01 smithi096 ceph-mon[102928]: pgmap v1405: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:01.389 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:01 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:08:01.389 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:01 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:08:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:01 smithi204 ceph-mon[87071]: pgmap v1405: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:01 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:08:01.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:01 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:08:02.282 INFO:teuthology.orchestra.run.smithi096.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-09-13T15:08:02.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:02 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2964915120' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:08:02.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:02 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2964915120' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:08:02.719 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:02 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2964915120' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:08:03.391 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T15:08:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:03 smithi096 ceph-mon[98901]: pgmap v1406: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:03 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2964915120' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:08:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:03 smithi096 ceph-mon[98901]: osdmap e423: 8 total, 8 up, 8 in 2024-09-13T15:08:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:03 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:08:03.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:03 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:08:03.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:03 smithi096 ceph-mon[102928]: pgmap v1406: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:03.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:03 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2964915120' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:08:03.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:03 smithi096 ceph-mon[102928]: osdmap e423: 8 total, 8 up, 8 in 2024-09-13T15:08:03.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:03 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:08:03.574 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:03 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:08:03.638 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:03 smithi204 ceph-mon[87071]: pgmap v1406: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:03.638 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:03 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2964915120' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:08:03.638 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:03 smithi204 ceph-mon[87071]: osdmap e423: 8 total, 8 up, 8 in 2024-09-13T15:08:03.639 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:03 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:08:03.639 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:03 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:08:03.653 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:04.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:04 smithi204 ceph-mon[87071]: osdmap e424: 8 total, 8 up, 8 in 2024-09-13T15:08:04.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:04 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:08:04.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:04 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:08:04.469 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:04 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:08:04.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:04 smithi096 ceph-mon[98901]: osdmap e424: 8 total, 8 up, 8 in 2024-09-13T15:08:04.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:04 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:08:04.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:04 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:08:04.573 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:04 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:08:04.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:04 smithi096 ceph-mon[102928]: osdmap e424: 8 total, 8 up, 8 in 2024-09-13T15:08:04.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:04 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:08:04.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:04 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:08:04.573 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:04 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' 2024-09-13T15:08:05.554 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T15:08:05.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:05 smithi096 ceph-mon[98901]: pgmap v1409: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:05.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:05 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2614166772' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:08:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:05 smithi096 ceph-mon[102928]: pgmap v1409: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:05.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:05 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2614166772' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:08:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:05 smithi204 ceph-mon[87071]: pgmap v1409: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:05.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:05 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2614166772' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:08:06.390 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 7 1 2024-09-13T15:08:06.640 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:06.759 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:06 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2614166772' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:08:06.759 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:06 smithi096 ceph-mon[98901]: osdmap e425: 8 total, 8 up, 8 in 2024-09-13T15:08:06.759 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:06 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:06.759 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:06 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2614166772' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:08:06.760 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:06 smithi096 ceph-mon[102928]: osdmap e425: 8 total, 8 up, 8 in 2024-09-13T15:08:06.760 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:06 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:06.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:06 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2614166772' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:08:06.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:06 smithi204 ceph-mon[87071]: osdmap e425: 8 total, 8 up, 8 in 2024-09-13T15:08:06.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:06 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:07.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:07 smithi096 ceph-mon[98901]: pgmap v1411: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:07.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:07 smithi096 ceph-mon[102928]: pgmap v1411: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:07.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:07 smithi204 ceph-mon[87071]: pgmap v1411: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:08.110 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:08:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:08:07] "GET /metrics HTTP/1.1" 200 34159 "" "Prometheus/2.51.0" 2024-09-13T15:08:08.571 INFO:teuthology.orchestra.run.smithi096.stderr:set osd.7 primary-affinity to 1 (8655362) 2024-09-13T15:08:08.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:08 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3310481011' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:08:08.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:08 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3310481011' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:08:08.972 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:08 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3310481011' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:08:09.390 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T15:08:09.634 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:09.755 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:09 smithi096 ceph-mon[98901]: pgmap v1412: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:09.755 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:09 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3310481011' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:08:09.755 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:09 smithi096 ceph-mon[98901]: osdmap e426: 8 total, 8 up, 8 in 2024-09-13T15:08:09.756 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:09 smithi096 ceph-mon[102928]: pgmap v1412: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:09.756 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:09 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3310481011' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:08:09.756 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:09 smithi096 ceph-mon[102928]: osdmap e426: 8 total, 8 up, 8 in 2024-09-13T15:08:09.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:09 smithi204 ceph-mon[87071]: pgmap v1412: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:09.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:09 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3310481011' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:08:09.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:09 smithi204 ceph-mon[87071]: osdmap e426: 8 total, 8 up, 8 in 2024-09-13T15:08:10.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:08:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:08:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:08:11.575 INFO:teuthology.orchestra.run.smithi096.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T15:08:11.938 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:11 smithi096 ceph-mon[98901]: pgmap v1414: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:11.938 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:11 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1482203078' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:08:11.939 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:11 smithi096 ceph-mon[102928]: pgmap v1414: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:11.939 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:11 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1482203078' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:08:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:11 smithi204 ceph-mon[87071]: pgmap v1414: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:11.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:11 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1482203078' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:08:12.374 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 3 1 2024-09-13T15:08:12.625 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:12.745 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:12 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1482203078' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:08:12.746 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:12 smithi096 ceph-mon[98901]: osdmap e427: 8 total, 8 up, 8 in 2024-09-13T15:08:12.746 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:12 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1482203078' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:08:12.746 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:12 smithi096 ceph-mon[102928]: osdmap e427: 8 total, 8 up, 8 in 2024-09-13T15:08:12.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:12 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1482203078' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:08:12.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:12 smithi204 ceph-mon[87071]: osdmap e427: 8 total, 8 up, 8 in 2024-09-13T15:08:13.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:13 smithi204 ceph-mon[87071]: pgmap v1416: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:13.987 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:13 smithi096 ceph-mon[98901]: pgmap v1416: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:13.987 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:13 smithi096 ceph-mon[102928]: pgmap v1416: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:14.590 INFO:teuthology.orchestra.run.smithi096.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-09-13T15:08:14.920 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:14 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4192057765' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:08:14.920 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:14 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4192057765' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:08:14.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:14 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4192057765' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:08:15.395 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2024-09-13T15:08:15.395 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2024-09-13T15:08:15.395 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2024-09-13T15:08:15.395 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2024-09-13T15:08:15.396 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T15:08:15.396 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T15:08:15.641 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:15.763 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:15 smithi096 ceph-mon[98901]: pgmap v1417: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:15.763 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:15 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4192057765' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:08:15.763 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:15 smithi096 ceph-mon[98901]: osdmap e428: 8 total, 8 up, 8 in 2024-09-13T15:08:15.763 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:15 smithi096 ceph-mon[102928]: pgmap v1417: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:15.763 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:15 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4192057765' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:08:15.763 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:15 smithi096 ceph-mon[102928]: osdmap e428: 8 total, 8 up, 8 in 2024-09-13T15:08:15.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:15 smithi204 ceph-mon[87071]: pgmap v1417: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:15.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:15 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4192057765' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:08:15.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:15 smithi204 ceph-mon[87071]: osdmap e428: 8 total, 8 up, 8 in 2024-09-13T15:08:17.518 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T15:08:17.518 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":428,"fsid":"51e50ed6-71db-11ef-bceb-c7b262605968","created":"2024-09-13T14:21:16.367777+0000","modified":"2024-09-13T15:08:14.587875+0000","last_up_change":"2024-09-13T15:01:12.901422+0000","last_in_change":"2024-09-13T15:05:50.962594+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:24:20.514144+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"409","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"325","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:26:54.629267+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":78,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:28:09.271608+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:28:09.734465+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":88,"up_thru":403,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6802","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6803","nonce":2849588101}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6804","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6805","nonce":2849588101}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6808","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6809","nonce":2849588101}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6806","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6807","nonce":2849588101}]},"public_addr":"172.21.15.96:6803/2849588101","cluster_addr":"172.21.15.96:6805/2849588101","heartbeat_back_addr":"172.21.15.96:6809/2849588101","heartbeat_front_addr":"172.21.15.96:6807/2849588101","state":["exists","up"]},{"osd":1,"uuid":"1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":90,"up_from":94,"up_thru":403,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6810","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6811","nonce":289441923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6812","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6813","nonce":289441923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6816","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6817","nonce":289441923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6814","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6815","nonce":289441923}]},"public_addr":"172.21.15.96:6811/289441923","cluster_addr":"172.21.15.96:6813/289441923","heartbeat_back_addr":"172.21.15.96:6817/289441923","heartbeat_front_addr":"172.21.15.96:6815/289441923","state":["exists","up"]},{"osd":2,"uuid":"6a92a5b3-bdc7-499a-8b0f-2b0936fd9598","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":95,"up_from":99,"up_thru":416,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6818","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6819","nonce":4202655894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6820","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6821","nonce":4202655894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6824","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6825","nonce":4202655894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6822","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6823","nonce":4202655894}]},"public_addr":"172.21.15.96:6819/4202655894","cluster_addr":"172.21.15.96:6821/4202655894","heartbeat_back_addr":"172.21.15.96:6825/4202655894","heartbeat_front_addr":"172.21.15.96:6823/4202655894","state":["exists","up"]},{"osd":3,"uuid":"1230646c-934e-4eb4-b4b9-16be52ffba66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":104,"last_clean_end":317,"up_from":335,"up_thru":403,"down_at":318,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6826","nonce":3715117729},{"type":"v1","addr":"172.21.15.96:6827","nonce":3715117729}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6828","nonce":3715117729},{"type":"v1","addr":"172.21.15.96:6829","nonce":3715117729}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6832","nonce":3715117729},{"type":"v1","addr":"172.21.15.96:6833","nonce":3715117729}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6830","nonce":3715117729},{"type":"v1","addr":"172.21.15.96:6831","nonce":3715117729}]},"public_addr":"172.21.15.96:6827/3715117729","cluster_addr":"172.21.15.96:6829/3715117729","heartbeat_back_addr":"172.21.15.96:6833/3715117729","heartbeat_front_addr":"172.21.15.96:6831/3715117729","state":["exists","up"]},{"osd":4,"uuid":"14115269-3d41-4ebf-8613-71ed59ec0f29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":403,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6800","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6801","nonce":1188704632}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6802","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6803","nonce":1188704632}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6806","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6807","nonce":1188704632}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6804","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6805","nonce":1188704632}]},"public_addr":"172.21.15.204:6801/1188704632","cluster_addr":"172.21.15.204:6803/1188704632","heartbeat_back_addr":"172.21.15.204:6807/1188704632","heartbeat_front_addr":"172.21.15.204:6805/1188704632","state":["exists","up"]},{"osd":5,"uuid":"15148a0c-df44-419e-b0f0-2b1600c207f4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":403,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6808","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6809","nonce":580729752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6810","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6811","nonce":580729752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6814","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6815","nonce":580729752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6812","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6813","nonce":580729752}]},"public_addr":"172.21.15.204:6809/580729752","cluster_addr":"172.21.15.204:6811/580729752","heartbeat_back_addr":"172.21.15.204:6815/580729752","heartbeat_front_addr":"172.21.15.204:6813/580729752","state":["exists","up"]},{"osd":6,"uuid":"8404e088-3523-4c21-b7c2-0a31b22e0279","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":423,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6816","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6817","nonce":118365497}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6818","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6819","nonce":118365497}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6822","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6823","nonce":118365497}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6820","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6821","nonce":118365497}]},"public_addr":"172.21.15.204:6817/118365497","cluster_addr":"172.21.15.204:6819/118365497","heartbeat_back_addr":"172.21.15.204:6823/118365497","heartbeat_front_addr":"172.21.15.204:6821/118365497","state":["exists","up"]},{"osd":7,"uuid":"93b23a35-0059-4e03-a2ba-28fee6431892","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":403,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6824","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6825","nonce":3529639424}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6826","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6827","nonce":3529639424}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6830","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6831","nonce":3529639424}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6828","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6829","nonce":3529639424}]},"public_addr":"172.21.15.204:6825/3529639424","cluster_addr":"172.21.15.204:6827/3529639424","heartbeat_back_addr":"172.21.15.204:6831/3529639424","heartbeat_front_addr":"172.21.15.204:6829/3529639424","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:28.841233+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:53.021149+0000","dead_epoch":90},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:17.120825+0000","dead_epoch":95},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:41.339674+0000","dead_epoch":317},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:02.571454+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:26.608523+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:51.119331+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:26:16.050687+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.1","mappings":[{"from":4,"to":2}]},{"pgid":"3.6","mappings":[{"from":4,"to":2}]},{"pgid":"3.9","mappings":[{"from":4,"to":0}]},{"pgid":"3.f","mappings":[{"from":7,"to":2}]},{"pgid":"3.14","mappings":[{"from":4,"to":0}]},{"pgid":"3.1d","mappings":[{"from":4,"to":0}]},{"pgid":"4.9","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":0}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":7}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.96:0/2744909732":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/2862942348":"2024-09-14T14:32:35.504695+0000","172.21.15.96:6800/2724610080":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/1761759058":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/3329894409":"2024-09-14T14:32:35.504695+0000","172.21.15.96:6800/776211071":"2024-09-14T14:30:47.049718+0000","172.21.15.96:6801/776211071":"2024-09-14T14:30:47.049718+0000","172.21.15.96:6800/2749644659":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/902309444":"2024-09-14T14:30:47.049718+0000","172.21.15.96:0/752011643":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/1494034928":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/1652886381":"2024-09-14T14:30:47.049718+0000","172.21.15.204:0/3779551551":"2024-09-14T14:31:56.307623+0000","172.21.15.96:0/19119516":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6801/2724610080":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/3050864675":"2024-09-14T14:30:47.049718+0000","172.21.15.96:6801/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.204:0/4067568101":"2024-09-14T14:31:56.307623+0000","172.21.15.96:0/2292899363":"2024-09-14T14:21:45.592521+0000","172.21.15.204:0/4099954218":"2024-09-14T14:31:56.307623+0000","172.21.15.96:0/960238088":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/260172418":"2024-09-14T14:26:50.514276+0000","172.21.15.96:6800/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/1644412166":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6801/2749644659":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/3782790019":"2024-09-14T14:30:47.049718+0000","172.21.15.204:0/3780556706":"2024-09-14T14:31:56.307623+0000","172.21.15.96:0/376655558":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/378263291":"2024-09-14T14:26:50.514276+0000","172.21.15.204:6834/2474101951":"2024-09-14T14:31:56.307623+0000","172.21.15.204:6835/2474101951":"2024-09-14T14:31:56.307623+0000","172.21.15.204:0/1453713144":"2024-09-14T14:31:56.307623+0000","172.21.15.204:0/2380512971":"2024-09-14T14:31:56.307623+0000","172.21.15.96:6801/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/3047961062":"2024-09-14T14:32:35.504695+0000","172.21.15.96:6800/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/667264267":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/1790922234":"2024-09-14T14:21:45.592521+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T15:08:17.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:17 smithi096 ceph-mon[102928]: pgmap v1419: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:17.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:17 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2200887063' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:08:17.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:17 smithi096 ceph-mon[98901]: pgmap v1419: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:17.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:17 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2200887063' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:08:17.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:17 smithi204 ceph-mon[87071]: pgmap v1419: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:17.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:17 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2200887063' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:08:18.160 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T15:08:18.161 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T15:08:18.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:08:17 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:08:17] "GET /metrics HTTP/1.1" 200 34159 "" "Prometheus/2.51.0" 2024-09-13T15:08:18.409 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:19.778 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:19 smithi096 ceph-mon[98901]: pgmap v1420: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:19.778 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:19 smithi096 ceph-mon[102928]: pgmap v1420: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:19.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:19 smithi204 ceph-mon[87071]: pgmap v1420: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:20.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:08:20 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:08:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:08:20.470 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T15:08:20.470 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":428,"fsid":"51e50ed6-71db-11ef-bceb-c7b262605968","created":"2024-09-13T14:21:16.367777+0000","modified":"2024-09-13T15:08:14.587875+0000","last_up_change":"2024-09-13T15:01:12.901422+0000","last_in_change":"2024-09-13T15:05:50.962594+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:24:20.514144+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"409","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"325","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:26:54.629267+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":78,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:28:09.271608+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:28:09.734465+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"eba0f65f-a3a7-4b30-a9a2-2dc91ccc35ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":88,"up_thru":403,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6802","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6803","nonce":2849588101}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6804","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6805","nonce":2849588101}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6808","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6809","nonce":2849588101}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6806","nonce":2849588101},{"type":"v1","addr":"172.21.15.96:6807","nonce":2849588101}]},"public_addr":"172.21.15.96:6803/2849588101","cluster_addr":"172.21.15.96:6805/2849588101","heartbeat_back_addr":"172.21.15.96:6809/2849588101","heartbeat_front_addr":"172.21.15.96:6807/2849588101","state":["exists","up"]},{"osd":1,"uuid":"1c8efa5f-1cf5-4dc0-a350-b78a24d9f0d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":90,"up_from":94,"up_thru":403,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6810","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6811","nonce":289441923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6812","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6813","nonce":289441923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6816","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6817","nonce":289441923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6814","nonce":289441923},{"type":"v1","addr":"172.21.15.96:6815","nonce":289441923}]},"public_addr":"172.21.15.96:6811/289441923","cluster_addr":"172.21.15.96:6813/289441923","heartbeat_back_addr":"172.21.15.96:6817/289441923","heartbeat_front_addr":"172.21.15.96:6815/289441923","state":["exists","up"]},{"osd":2,"uuid":"6a92a5b3-bdc7-499a-8b0f-2b0936fd9598","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":95,"up_from":99,"up_thru":416,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6818","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6819","nonce":4202655894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6820","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6821","nonce":4202655894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6824","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6825","nonce":4202655894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6822","nonce":4202655894},{"type":"v1","addr":"172.21.15.96:6823","nonce":4202655894}]},"public_addr":"172.21.15.96:6819/4202655894","cluster_addr":"172.21.15.96:6821/4202655894","heartbeat_back_addr":"172.21.15.96:6825/4202655894","heartbeat_front_addr":"172.21.15.96:6823/4202655894","state":["exists","up"]},{"osd":3,"uuid":"1230646c-934e-4eb4-b4b9-16be52ffba66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":104,"last_clean_end":317,"up_from":335,"up_thru":403,"down_at":318,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6826","nonce":3715117729},{"type":"v1","addr":"172.21.15.96:6827","nonce":3715117729}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6828","nonce":3715117729},{"type":"v1","addr":"172.21.15.96:6829","nonce":3715117729}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6832","nonce":3715117729},{"type":"v1","addr":"172.21.15.96:6833","nonce":3715117729}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6830","nonce":3715117729},{"type":"v1","addr":"172.21.15.96:6831","nonce":3715117729}]},"public_addr":"172.21.15.96:6827/3715117729","cluster_addr":"172.21.15.96:6829/3715117729","heartbeat_back_addr":"172.21.15.96:6833/3715117729","heartbeat_front_addr":"172.21.15.96:6831/3715117729","state":["exists","up"]},{"osd":4,"uuid":"14115269-3d41-4ebf-8613-71ed59ec0f29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":403,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6800","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6801","nonce":1188704632}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6802","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6803","nonce":1188704632}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6806","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6807","nonce":1188704632}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6804","nonce":1188704632},{"type":"v1","addr":"172.21.15.204:6805","nonce":1188704632}]},"public_addr":"172.21.15.204:6801/1188704632","cluster_addr":"172.21.15.204:6803/1188704632","heartbeat_back_addr":"172.21.15.204:6807/1188704632","heartbeat_front_addr":"172.21.15.204:6805/1188704632","state":["exists","up"]},{"osd":5,"uuid":"15148a0c-df44-419e-b0f0-2b1600c207f4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":403,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6808","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6809","nonce":580729752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6810","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6811","nonce":580729752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6814","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6815","nonce":580729752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6812","nonce":580729752},{"type":"v1","addr":"172.21.15.204:6813","nonce":580729752}]},"public_addr":"172.21.15.204:6809/580729752","cluster_addr":"172.21.15.204:6811/580729752","heartbeat_back_addr":"172.21.15.204:6815/580729752","heartbeat_front_addr":"172.21.15.204:6813/580729752","state":["exists","up"]},{"osd":6,"uuid":"8404e088-3523-4c21-b7c2-0a31b22e0279","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":423,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6816","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6817","nonce":118365497}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6818","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6819","nonce":118365497}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6822","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6823","nonce":118365497}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6820","nonce":118365497},{"type":"v1","addr":"172.21.15.204:6821","nonce":118365497}]},"public_addr":"172.21.15.204:6817/118365497","cluster_addr":"172.21.15.204:6819/118365497","heartbeat_back_addr":"172.21.15.204:6823/118365497","heartbeat_front_addr":"172.21.15.204:6821/118365497","state":["exists","up"]},{"osd":7,"uuid":"93b23a35-0059-4e03-a2ba-28fee6431892","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":403,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6824","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6825","nonce":3529639424}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6826","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6827","nonce":3529639424}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6830","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6831","nonce":3529639424}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6828","nonce":3529639424},{"type":"v1","addr":"172.21.15.204:6829","nonce":3529639424}]},"public_addr":"172.21.15.204:6825/3529639424","cluster_addr":"172.21.15.204:6827/3529639424","heartbeat_back_addr":"172.21.15.204:6831/3529639424","heartbeat_front_addr":"172.21.15.204:6829/3529639424","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:28.841233+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:23:53.021149+0000","dead_epoch":90},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:17.120825+0000","dead_epoch":95},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:24:41.339674+0000","dead_epoch":317},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:02.571454+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:26.608523+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:25:51.119331+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:26:16.050687+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.1","mappings":[{"from":4,"to":2}]},{"pgid":"3.6","mappings":[{"from":4,"to":2}]},{"pgid":"3.9","mappings":[{"from":4,"to":0}]},{"pgid":"3.f","mappings":[{"from":7,"to":2}]},{"pgid":"3.14","mappings":[{"from":4,"to":0}]},{"pgid":"3.1d","mappings":[{"from":4,"to":0}]},{"pgid":"4.9","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":0}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":7}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.96:0/378263291":"2024-09-14T14:26:50.514276+0000","172.21.15.204:0/1453713144":"2024-09-14T14:31:56.307623+0000","172.21.15.96:6801/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/1494034928":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6800/2875623667":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/3047961062":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/2862942348":"2024-09-14T14:32:35.504695+0000","172.21.15.96:6800/2749644659":"2024-09-14T14:26:50.514276+0000","172.21.15.204:0/4099954218":"2024-09-14T14:31:56.307623+0000","172.21.15.96:0/960238088":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/260172418":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/1652886381":"2024-09-14T14:30:47.049718+0000","172.21.15.204:0/3779551551":"2024-09-14T14:31:56.307623+0000","172.21.15.96:0/19119516":"2024-09-14T14:22:08.207226+0000","172.21.15.96:6801/2724610080":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/3050864675":"2024-09-14T14:30:47.049718+0000","172.21.15.96:6800/2724610080":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/902309444":"2024-09-14T14:30:47.049718+0000","172.21.15.96:0/1790922234":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/667264267":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/1761759058":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/3329894409":"2024-09-14T14:32:35.504695+0000","172.21.15.204:6834/2474101951":"2024-09-14T14:31:56.307623+0000","172.21.15.96:6800/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/1644412166":"2024-09-14T14:22:08.207226+0000","172.21.15.96:0/2744909732":"2024-09-14T14:32:35.504695+0000","172.21.15.96:0/752011643":"2024-09-14T14:26:50.514276+0000","172.21.15.96:6800/776211071":"2024-09-14T14:30:47.049718+0000","172.21.15.96:6801/3509914111":"2024-09-14T14:21:45.592521+0000","172.21.15.96:0/2292899363":"2024-09-14T14:21:45.592521+0000","172.21.15.204:0/4067568101":"2024-09-14T14:31:56.307623+0000","172.21.15.96:6801/776211071":"2024-09-14T14:30:47.049718+0000","172.21.15.96:6801/2749644659":"2024-09-14T14:26:50.514276+0000","172.21.15.96:0/3782790019":"2024-09-14T14:30:47.049718+0000","172.21.15.204:0/3780556706":"2024-09-14T14:31:56.307623+0000","172.21.15.96:0/376655558":"2024-09-14T14:21:45.592521+0000","172.21.15.204:6835/2474101951":"2024-09-14T14:31:56.307623+0000","172.21.15.204:0/2380512971":"2024-09-14T14:31:56.307623+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T15:08:20.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:20 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4193320492' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:08:20.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:20 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4193320492' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:08:20.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:20 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4193320492' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:08:21.253 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T15:08:21.254 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T15:08:21.254 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T15:08:21.255 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T15:08:21.255 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T15:08:21.255 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T15:08:21.256 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T15:08:21.256 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T15:08:21.811 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:21.812 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:21.816 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:21.816 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:21.817 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:21.820 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:21.821 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:21.821 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:21.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:21 smithi096 ceph-mon[98901]: pgmap v1421: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:21.946 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:21 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:21.946 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:21 smithi096 ceph-mon[102928]: pgmap v1421: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:21.947 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:21 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:21.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:21 smithi204 ceph-mon[87071]: pgmap v1421: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:21.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:21 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:23.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:23 smithi096 ceph-mon[98901]: pgmap v1422: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:23.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:23 smithi096 ceph-mon[102928]: pgmap v1422: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:23.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:23 smithi204 ceph-mon[87071]: pgmap v1422: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:25.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:25 smithi204 ceph-mon[87071]: pgmap v1423: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:26.072 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:25 smithi096 ceph-mon[98901]: pgmap v1423: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:26.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:25 smithi096 ceph-mon[102928]: pgmap v1423: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:27.948 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:27 smithi096 ceph-mon[98901]: pgmap v1424: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:27.948 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:27 smithi096 ceph-mon[102928]: pgmap v1424: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:27.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:27 smithi204 ceph-mon[87071]: pgmap v1424: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:28.302 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:08:27 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:08:27] "GET /metrics HTTP/1.1" 200 34159 "" "Prometheus/2.51.0" 2024-09-13T15:08:28.985 INFO:teuthology.orchestra.run.smithi096.stdout:158913790472 2024-09-13T15:08:28.986 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T15:08:29.785 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:29 smithi096 ceph-mon[98901]: pgmap v1425: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:29.786 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:29 smithi096 ceph-mon[102928]: pgmap v1425: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:29.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:29 smithi204 ceph-mon[87071]: pgmap v1425: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:30.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:08:30 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:08:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:08:30.626 INFO:teuthology.orchestra.run.smithi096.stdout:425201762677 2024-09-13T15:08:30.626 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T15:08:31.944 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:31 smithi096 ceph-mon[98901]: pgmap v1426: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:31.944 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:31 smithi096 ceph-mon[102928]: pgmap v1426: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:31.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:31 smithi204 ceph-mon[87071]: pgmap v1426: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:33.708 INFO:teuthology.orchestra.run.smithi096.stdout:377957122443 2024-09-13T15:08:33.708 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T15:08:33.915 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:33 smithi096 ceph-mon[98901]: pgmap v1427: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:33.916 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:33 smithi096 ceph-mon[102928]: pgmap v1427: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:33.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:33 smithi204 ceph-mon[87071]: pgmap v1427: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:35.536 INFO:teuthology.orchestra.run.smithi096.stdout:1438814044250 2024-09-13T15:08:35.536 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T15:08:35.762 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:35 smithi096 ceph-mon[98901]: pgmap v1428: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:35.762 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:35 smithi096 ceph-mon[102928]: pgmap v1428: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:35.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:35 smithi204 ceph-mon[87071]: pgmap v1428: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:36.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:36 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:37.019 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:36 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:37.019 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:36 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:37.439 INFO:teuthology.orchestra.run.smithi096.stdout:133143986703 2024-09-13T15:08:37.440 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T15:08:37.681 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:37 smithi096 ceph-mon[98901]: pgmap v1429: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:37.682 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:37 smithi096 ceph-mon[102928]: pgmap v1429: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:37.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:37 smithi204 ceph-mon[87071]: pgmap v1429: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:38.294 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:08:37 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:08:37] "GET /metrics HTTP/1.1" 200 34160 "" "Prometheus/2.51.0" 2024-09-13T15:08:38.325 INFO:teuthology.orchestra.run.smithi096.stdout:210453398017 2024-09-13T15:08:38.326 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T15:08:38.326 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:38.797 INFO:teuthology.orchestra.run.smithi096.stdout:403726926204 2024-09-13T15:08:38.797 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T15:08:39.224 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:39.259 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:39.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:39 smithi096 ceph-mon[98901]: pgmap v1430: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:39.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:39 smithi096 ceph-mon[102928]: pgmap v1430: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:39.968 INFO:teuthology.orchestra.run.smithi096.stdout:184683594246 2024-09-13T15:08:39.969 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T15:08:39.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:39 smithi204 ceph-mon[87071]: pgmap v1430: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:39.998 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:40.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:08:40 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:08:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:08:41.751 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:41 smithi096 ceph-mon[98901]: pgmap v1431: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:41.751 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:41 smithi096 ceph-mon[102928]: pgmap v1431: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:41.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:41 smithi204 ceph-mon[87071]: pgmap v1431: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:42.206 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:42.207 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:43.578 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:43.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:43 smithi096 ceph-mon[98901]: pgmap v1432: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:43.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:43 smithi096 ceph-mon[102928]: pgmap v1432: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:43.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:43 smithi204 ceph-mon[87071]: pgmap v1432: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:44.751 INFO:teuthology.orchestra.run.smithi096.stdout:425201762679 2024-09-13T15:08:45.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:45 smithi096 ceph-mon[98901]: pgmap v1433: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:45.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:45 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/4035230283' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T15:08:45.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:45 smithi096 ceph-mon[102928]: pgmap v1433: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:45.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:45 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/4035230283' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T15:08:45.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:45 smithi204 ceph-mon[87071]: pgmap v1433: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:45.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:45 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/4035230283' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T15:08:46.736 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:47.547 INFO:teuthology.orchestra.run.smithi096.stdout:403726926206 2024-09-13T15:08:47.947 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:47 smithi096 ceph-mon[98901]: pgmap v1434: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:47.947 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:47 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1076378016' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T15:08:47.948 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:47 smithi096 ceph-mon[102928]: pgmap v1434: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:47.948 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:47 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1076378016' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T15:08:47.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:47 smithi204 ceph-mon[87071]: pgmap v1434: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:47.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:47 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1076378016' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T15:08:48.232 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:08:47 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:08:47] "GET /metrics HTTP/1.1" 200 34160 "" "Prometheus/2.51.0" 2024-09-13T15:08:48.256 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726926204 got 403726926206 for osd.1 2024-09-13T15:08:48.256 DEBUG:teuthology.parallel:result is None 2024-09-13T15:08:48.610 INFO:teuthology.orchestra.run.smithi096.stdout:210453398018 2024-09-13T15:08:48.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:48 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2097030449' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T15:08:48.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:48 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2097030449' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T15:08:48.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:48 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2097030449' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T15:08:49.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:49 smithi096 ceph-mon[98901]: pgmap v1435: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:49.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:49 smithi096 ceph-mon[102928]: pgmap v1435: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:49.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:49 smithi204 ceph-mon[87071]: pgmap v1435: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:50.327 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453398017 got 210453398018 for osd.7 2024-09-13T15:08:50.327 DEBUG:teuthology.parallel:result is None 2024-09-13T15:08:50.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:08:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:08:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:08:50.933 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:50 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2030280' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T15:08:50.934 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:50 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2030280' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T15:08:50.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:50 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2030280' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T15:08:51.033 INFO:teuthology.orchestra.run.smithi096.stdout:158913790477 2024-09-13T15:08:51.758 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913790472 got 158913790477 for osd.5 2024-09-13T15:08:51.759 DEBUG:teuthology.parallel:result is None 2024-09-13T15:08:51.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:51 smithi204 ceph-mon[87071]: pgmap v1436: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:51.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:51 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:51.981 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:51 smithi096 ceph-mon[98901]: pgmap v1436: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:51.981 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:51 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:51.981 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:51 smithi096 ceph-mon[102928]: pgmap v1436: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:51.982 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:51 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:52.491 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762677 got 425201762679 for osd.2 2024-09-13T15:08:52.491 DEBUG:teuthology.parallel:result is None 2024-09-13T15:08:53.098 INFO:teuthology.orchestra.run.smithi096.stdout:377957122447 2024-09-13T15:08:53.259 INFO:teuthology.orchestra.run.smithi096.stdout:133143986707 2024-09-13T15:08:53.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:53 smithi096 ceph-mon[98901]: pgmap v1437: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:53.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:53 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2320447317' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T15:08:53.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:53 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/1054749504' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T15:08:53.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:53 smithi096 ceph-mon[102928]: pgmap v1437: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:53.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:53 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2320447317' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T15:08:53.824 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:53 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/1054749504' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T15:08:53.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:53 smithi204 ceph-mon[87071]: pgmap v1437: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:53.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:53 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2320447317' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T15:08:53.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:53 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/1054749504' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T15:08:54.249 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986703 got 133143986707 for osd.4 2024-09-13T15:08:54.249 DEBUG:teuthology.parallel:result is None 2024-09-13T15:08:55.582 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122443 got 377957122447 for osd.0 2024-09-13T15:08:55.582 DEBUG:teuthology.parallel:result is None 2024-09-13T15:08:55.822 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:55 smithi096 ceph-mon[98901]: pgmap v1438: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:55.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:55 smithi096 ceph-mon[102928]: pgmap v1438: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:55.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:55 smithi204 ceph-mon[87071]: pgmap v1438: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:56.470 INFO:teuthology.orchestra.run.smithi096.stdout:1438814044254 2024-09-13T15:08:56.749 INFO:teuthology.orchestra.run.smithi096.stdout:184683594249 2024-09-13T15:08:56.750 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:56 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/2834946124' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T15:08:56.750 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:56 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/2834946124' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T15:08:56.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:56 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/2834946124' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T15:08:57.342 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1438814044250 got 1438814044254 for osd.3 2024-09-13T15:08:57.342 DEBUG:teuthology.parallel:result is None 2024-09-13T15:08:57.930 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:57 smithi096 ceph-mon[102928]: pgmap v1439: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:57.931 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:57 smithi096 ceph-mon[102928]: from='client.? 172.21.15.96:0/3258893226' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T15:08:57.931 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:57 smithi096 ceph-mon[98901]: pgmap v1439: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:57.931 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:57 smithi096 ceph-mon[98901]: from='client.? 172.21.15.96:0/3258893226' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T15:08:57.956 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683594246 got 184683594249 for osd.6 2024-09-13T15:08:57.956 DEBUG:teuthology.parallel:result is None 2024-09-13T15:08:57.956 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T15:08:57.957 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:08:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:57 smithi204 ceph-mon[87071]: pgmap v1439: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:08:57.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:57 smithi204 ceph-mon[87071]: from='client.? 172.21.15.96:0/3258893226' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T15:08:58.202 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:58.322 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:08:57 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:08:57] "GET /metrics HTTP/1.1" 200 34162 "" "Prometheus/2.51.0" 2024-09-13T15:08:59.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:08:59 smithi204 ceph-mon[87071]: pgmap v1440: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:09:00.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:08:59 smithi096 ceph-mon[98901]: pgmap v1440: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:09:00.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:08:59 smithi096 ceph-mon[102928]: pgmap v1440: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:09:00.110 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T15:09:00.111 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T15:09:00.469 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:09:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:09:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:09:00.821 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":1440,"stamp":"2024-09-13T15:08:58.274024+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":907,"num_write_kb":12623,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8732588,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4132,"ondisk_log_size":4132,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6542508,"kb_used_data":63688,"kb_used_omap":39,"kb_used_meta":6478680,"kb_avail":743418708,"statfs":{"total":767960285184,"available":761260756992,"internally_reserved":0,"allocated":65216512,"data_stored":41801272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40927,"internal_metadata":6634168353},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002587"},"pg_stats":[{"pgid":"3.1f","version":"64'4","reported_seq":768,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433423+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T15:08:15.433423+0000","last_peered":"2024-09-13T15:08:15.433423+0000","last_clean":"2024-09-13T15:08:15.433423+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T15:08:15.433423+0000","last_undegraded":"2024-09-13T15:08:15.433423+0000","last_fullsized":"2024-09-13T15:08:15.433423+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":415,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.417737+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T15:08:12.417737+0000","last_peered":"2024-09-13T15:08:12.417737+0000","last_clean":"2024-09-13T15:08:12.417737+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T15:08:12.417737+0000","last_undegraded":"2024-09-13T15:08:12.417737+0000","last_fullsized":"2024-09-13T15:08:12.417737+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"82'100","reported_seq":200,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.261092+0000","last_change":"2024-09-13T15:05:52.013759+0000","last_active":"2024-09-13T15:08:15.261092+0000","last_peered":"2024-09-13T15:08:15.261092+0000","last_clean":"2024-09-13T15:08:15.261092+0000","last_became_active":"2024-09-13T15:05:52.013210+0000","last_became_peered":"2024-09-13T15:05:52.013210+0000","last_unstale":"2024-09-13T15:08:15.261092+0000","last_undegraded":"2024-09-13T15:08:15.261092+0000","last_fullsized":"2024-09-13T15:08:15.261092+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.149290+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.149290+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:10.545939+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982560+0000","last_change":"2024-09-13T15:05:51.995035+0000","last_active":"2024-09-13T15:08:14.982560+0000","last_peered":"2024-09-13T15:08:14.982560+0000","last_clean":"2024-09-13T15:08:14.982560+0000","last_became_active":"2024-09-13T15:05:51.994496+0000","last_became_peered":"2024-09-13T15:05:51.994496+0000","last_unstale":"2024-09-13T15:08:14.982560+0000","last_undegraded":"2024-09-13T15:08:14.982560+0000","last_fullsized":"2024-09-13T15:08:14.982560+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:11:51.103755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982168+0000","last_change":"2024-09-13T15:05:52.004131+0000","last_active":"2024-09-13T15:08:14.982168+0000","last_peered":"2024-09-13T15:08:14.982168+0000","last_clean":"2024-09-13T15:08:14.982168+0000","last_became_active":"2024-09-13T15:05:52.003845+0000","last_became_peered":"2024-09-13T15:05:52.003845+0000","last_unstale":"2024-09-13T15:08:14.982168+0000","last_undegraded":"2024-09-13T15:08:14.982168+0000","last_fullsized":"2024-09-13T15:08:14.982168+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:01.268036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"82'100","reported_seq":86,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578220+0000","last_change":"2024-09-13T15:02:23.879862+0000","last_active":"2024-09-13T15:08:11.578220+0000","last_peered":"2024-09-13T15:08:11.578220+0000","last_clean":"2024-09-13T15:08:11.578220+0000","last_became_active":"2024-09-13T15:02:23.879170+0000","last_became_peered":"2024-09-13T15:02:23.879170+0000","last_unstale":"2024-09-13T15:08:11.578220+0000","last_undegraded":"2024-09-13T15:08:11.578220+0000","last_fullsized":"2024-09-13T15:08:11.578220+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:06.945525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:06.945525+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:48.496118+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":433,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578967+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T15:08:11.578967+0000","last_peered":"2024-09-13T15:08:11.578967+0000","last_clean":"2024-09-13T15:08:11.578967+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T15:08:11.578967+0000","last_undegraded":"2024-09-13T15:08:11.578967+0000","last_fullsized":"2024-09-13T15:08:11.578967+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":429,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418029+0000","last_change":"2024-09-13T15:05:51.995355+0000","last_active":"2024-09-13T15:08:12.418029+0000","last_peered":"2024-09-13T15:08:12.418029+0000","last_clean":"2024-09-13T15:08:12.418029+0000","last_became_active":"2024-09-13T15:05:51.995038+0000","last_became_peered":"2024-09-13T15:05:51.995038+0000","last_unstale":"2024-09-13T15:08:12.418029+0000","last_undegraded":"2024-09-13T15:08:12.418029+0000","last_fullsized":"2024-09-13T15:08:12.418029+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:24:14.413713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":394,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579092+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T15:08:11.579092+0000","last_peered":"2024-09-13T15:08:11.579092+0000","last_clean":"2024-09-13T15:08:11.579092+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T15:08:11.579092+0000","last_undegraded":"2024-09-13T15:08:11.579092+0000","last_fullsized":"2024-09-13T15:08:11.579092+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":429,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418190+0000","last_change":"2024-09-13T15:05:51.998541+0000","last_active":"2024-09-13T15:08:12.418190+0000","last_peered":"2024-09-13T15:08:12.418190+0000","last_clean":"2024-09-13T15:08:12.418190+0000","last_became_active":"2024-09-13T15:05:51.998234+0000","last_became_peered":"2024-09-13T15:05:51.998234+0000","last_unstale":"2024-09-13T15:08:12.418190+0000","last_undegraded":"2024-09-13T15:08:12.418190+0000","last_fullsized":"2024-09-13T15:08:12.418190+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:25.705700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"82'100","reported_seq":110,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982545+0000","last_change":"2024-09-13T15:05:52.535449+0000","last_active":"2024-09-13T15:08:14.982545+0000","last_peered":"2024-09-13T15:08:14.982545+0000","last_clean":"2024-09-13T15:08:14.982545+0000","last_became_active":"2024-09-13T15:05:51.991505+0000","last_became_peered":"2024-09-13T15:05:51.991505+0000","last_unstale":"2024-09-13T15:08:14.982545+0000","last_undegraded":"2024-09-13T15:08:14.982545+0000","last_fullsized":"2024-09-13T15:08:14.982545+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:05:52.535365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:05:52.535365+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:13:01.193300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":716,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433356+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T15:08:15.433356+0000","last_peered":"2024-09-13T15:08:15.433356+0000","last_clean":"2024-09-13T15:08:15.433356+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T15:08:15.433356+0000","last_undegraded":"2024-09-13T15:08:15.433356+0000","last_fullsized":"2024-09-13T15:08:15.433356+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":496,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.591873+0000","last_change":"2024-09-13T15:07:37.914968+0000","last_active":"2024-09-13T15:08:14.591873+0000","last_peered":"2024-09-13T15:08:14.591873+0000","last_clean":"2024-09-13T15:08:14.591873+0000","last_became_active":"2024-09-13T15:07:37.914657+0000","last_became_peered":"2024-09-13T15:07:37.914657+0000","last_unstale":"2024-09-13T15:08:14.591873+0000","last_undegraded":"2024-09-13T15:08:14.591873+0000","last_fullsized":"2024-09-13T15:08:14.591873+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:55.882895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"82'100","reported_seq":102,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982075+0000","last_change":"2024-09-13T15:05:52.002778+0000","last_active":"2024-09-13T15:08:14.982075+0000","last_peered":"2024-09-13T15:08:14.982075+0000","last_clean":"2024-09-13T15:08:14.982075+0000","last_became_active":"2024-09-13T15:05:52.002405+0000","last_became_peered":"2024-09-13T15:05:52.002405+0000","last_unstale":"2024-09-13T15:08:14.982075+0000","last_undegraded":"2024-09-13T15:08:14.982075+0000","last_fullsized":"2024-09-13T15:08:14.982075+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.429543+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.429543+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:21:59.195306+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":411,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418004+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T15:08:12.418004+0000","last_peered":"2024-09-13T15:08:12.418004+0000","last_clean":"2024-09-13T15:08:12.418004+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T15:08:12.418004+0000","last_undegraded":"2024-09-13T15:08:12.418004+0000","last_fullsized":"2024-09-13T15:08:12.418004+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982467+0000","last_change":"2024-09-13T15:05:52.001307+0000","last_active":"2024-09-13T15:08:14.982467+0000","last_peered":"2024-09-13T15:08:14.982467+0000","last_clean":"2024-09-13T15:08:14.982467+0000","last_became_active":"2024-09-13T15:05:52.001040+0000","last_became_peered":"2024-09-13T15:05:52.001040+0000","last_unstale":"2024-09-13T15:08:14.982467+0000","last_undegraded":"2024-09-13T15:08:14.982467+0000","last_fullsized":"2024-09-13T15:08:14.982467+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:49:48.758670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"82'100","reported_seq":383,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579377+0000","last_change":"2024-09-13T15:03:04.101898+0000","last_active":"2024-09-13T15:08:11.579377+0000","last_peered":"2024-09-13T15:08:11.579377+0000","last_clean":"2024-09-13T15:08:11.579377+0000","last_became_active":"2024-09-13T15:02:23.884406+0000","last_became_peered":"2024-09-13T15:02:23.884406+0000","last_unstale":"2024-09-13T15:08:11.579377+0000","last_undegraded":"2024-09-13T15:08:11.579377+0000","last_fullsized":"2024-09-13T15:08:11.579377+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:04.101802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:04.101802+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:34.459623+0000","scrub_duration":0.003875819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":727,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260773+0000","last_change":"2024-09-13T15:05:52.007634+0000","last_active":"2024-09-13T15:08:15.260773+0000","last_peered":"2024-09-13T15:08:15.260773+0000","last_clean":"2024-09-13T15:08:15.260773+0000","last_became_active":"2024-09-13T15:05:52.007302+0000","last_became_peered":"2024-09-13T15:05:52.007302+0000","last_unstale":"2024-09-13T15:08:15.260773+0000","last_undegraded":"2024-09-13T15:08:15.260773+0000","last_fullsized":"2024-09-13T15:08:15.260773+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:36.203235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":762,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433566+0000","last_change":"2024-09-13T15:05:52.008086+0000","last_active":"2024-09-13T15:08:15.433566+0000","last_peered":"2024-09-13T15:08:15.433566+0000","last_clean":"2024-09-13T15:08:15.433566+0000","last_became_active":"2024-09-13T15:05:52.007347+0000","last_became_peered":"2024-09-13T15:05:52.007347+0000","last_unstale":"2024-09-13T15:08:15.433566+0000","last_undegraded":"2024-09-13T15:08:15.433566+0000","last_fullsized":"2024-09-13T15:08:15.433566+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:12:09.381403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"82'100","reported_seq":678,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578095+0000","last_change":"2024-09-13T15:03:07.166808+0000","last_active":"2024-09-13T15:08:11.578095+0000","last_peered":"2024-09-13T15:08:11.578095+0000","last_clean":"2024-09-13T15:08:11.578095+0000","last_became_active":"2024-09-13T15:02:23.883387+0000","last_became_peered":"2024-09-13T15:02:23.883387+0000","last_unstale":"2024-09-13T15:08:11.578095+0000","last_undegraded":"2024-09-13T15:08:11.578095+0000","last_fullsized":"2024-09-13T15:08:11.578095+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:07.166034+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:07.166034+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:32:42.605595+0000","scrub_duration":0.01094734,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982061+0000","last_change":"2024-09-13T15:05:52.003501+0000","last_active":"2024-09-13T15:08:14.982061+0000","last_peered":"2024-09-13T15:08:14.982061+0000","last_clean":"2024-09-13T15:08:14.982061+0000","last_became_active":"2024-09-13T15:05:52.003200+0000","last_became_peered":"2024-09-13T15:05:52.003200+0000","last_unstale":"2024-09-13T15:08:14.982061+0000","last_undegraded":"2024-09-13T15:08:14.982061+0000","last_fullsized":"2024-09-13T15:08:14.982061+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:46:07.106935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":212,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600577+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T15:08:11.600577+0000","last_peered":"2024-09-13T15:08:11.600577+0000","last_clean":"2024-09-13T15:08:11.600577+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T15:08:11.600577+0000","last_undegraded":"2024-09-13T15:08:11.600577+0000","last_fullsized":"2024-09-13T15:08:11.600577+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"82'100","reported_seq":96,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.417783+0000","last_change":"2024-09-13T15:02:56.319389+0000","last_active":"2024-09-13T15:08:12.417783+0000","last_peered":"2024-09-13T15:08:12.417783+0000","last_clean":"2024-09-13T15:08:12.417783+0000","last_became_active":"2024-09-13T15:02:23.876739+0000","last_became_peered":"2024-09-13T15:02:23.876739+0000","last_unstale":"2024-09-13T15:08:12.417783+0000","last_undegraded":"2024-09-13T15:08:12.417783+0000","last_fullsized":"2024-09-13T15:08:12.417783+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:56.319302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:56.319302+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:58.230158+0000","scrub_duration":0.0031182190000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.981928+0000","last_change":"2024-09-13T15:05:51.999327+0000","last_active":"2024-09-13T15:08:14.981928+0000","last_peered":"2024-09-13T15:08:14.981928+0000","last_clean":"2024-09-13T15:08:14.981928+0000","last_became_active":"2024-09-13T15:05:51.994485+0000","last_became_peered":"2024-09-13T15:05:51.994485+0000","last_unstale":"2024-09-13T15:08:14.981928+0000","last_undegraded":"2024-09-13T15:08:14.981928+0000","last_fullsized":"2024-09-13T15:08:14.981928+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:55:49.041649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":737,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433505+0000","last_change":"2024-09-13T15:05:52.006782+0000","last_active":"2024-09-13T15:08:15.433505+0000","last_peered":"2024-09-13T15:08:15.433505+0000","last_clean":"2024-09-13T15:08:15.433505+0000","last_became_active":"2024-09-13T15:05:52.006452+0000","last_became_peered":"2024-09-13T15:05:52.006452+0000","last_unstale":"2024-09-13T15:08:15.433505+0000","last_undegraded":"2024-09-13T15:08:15.433505+0000","last_fullsized":"2024-09-13T15:08:15.433505+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:37:23.855899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"82'100","reported_seq":96,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578641+0000","last_change":"2024-09-13T15:02:55.870615+0000","last_active":"2024-09-13T15:08:11.578641+0000","last_peered":"2024-09-13T15:08:11.578641+0000","last_clean":"2024-09-13T15:08:11.578641+0000","last_became_active":"2024-09-13T15:02:23.883217+0000","last_became_peered":"2024-09-13T15:02:23.883217+0000","last_unstale":"2024-09-13T15:08:11.578641+0000","last_undegraded":"2024-09-13T15:08:11.578641+0000","last_fullsized":"2024-09-13T15:08:11.578641+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.870528+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.870528+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:10.023787+0000","scrub_duration":0.0047551850000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982226+0000","last_change":"2024-09-13T15:05:52.010150+0000","last_active":"2024-09-13T15:08:14.982226+0000","last_peered":"2024-09-13T15:08:14.982226+0000","last_clean":"2024-09-13T15:08:14.982226+0000","last_became_active":"2024-09-13T15:05:52.008977+0000","last_became_peered":"2024-09-13T15:05:52.008977+0000","last_unstale":"2024-09-13T15:08:14.982226+0000","last_undegraded":"2024-09-13T15:08:14.982226+0000","last_fullsized":"2024-09-13T15:08:14.982226+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:43.067814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":392,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578892+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T15:08:11.578892+0000","last_peered":"2024-09-13T15:08:11.578892+0000","last_clean":"2024-09-13T15:08:11.578892+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T15:08:11.578892+0000","last_undegraded":"2024-09-13T15:08:11.578892+0000","last_fullsized":"2024-09-13T15:08:11.578892+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":310,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.591761+0000","last_change":"2024-09-13T14:56:01.235655+0000","last_active":"2024-09-13T15:08:14.591761+0000","last_peered":"2024-09-13T15:08:14.591761+0000","last_clean":"2024-09-13T15:08:14.591761+0000","last_became_active":"2024-09-13T14:55:55.195703+0000","last_became_peered":"2024-09-13T14:55:55.195703+0000","last_unstale":"2024-09-13T15:08:14.591761+0000","last_undegraded":"2024-09-13T15:08:14.591761+0000","last_fullsized":"2024-09-13T15:08:14.591761+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.235497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.235497+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:39:43.495759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":434,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578402+0000","last_change":"2024-09-13T15:05:52.016761+0000","last_active":"2024-09-13T15:08:11.578402+0000","last_peered":"2024-09-13T15:08:11.578402+0000","last_clean":"2024-09-13T15:08:11.578402+0000","last_became_active":"2024-09-13T15:05:52.005623+0000","last_became_peered":"2024-09-13T15:05:52.005623+0000","last_unstale":"2024-09-13T15:08:11.578402+0000","last_undegraded":"2024-09-13T15:08:11.578402+0000","last_fullsized":"2024-09-13T15:08:11.578402+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:58:43.424615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":727,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260818+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T15:08:15.260818+0000","last_peered":"2024-09-13T15:08:15.260818+0000","last_clean":"2024-09-13T15:08:15.260818+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T15:08:15.260818+0000","last_undegraded":"2024-09-13T15:08:15.260818+0000","last_fullsized":"2024-09-13T15:08:15.260818+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.17","version":"82'100","reported_seq":240,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578845+0000","last_change":"2024-09-13T15:03:03.108031+0000","last_active":"2024-09-13T15:08:11.578845+0000","last_peered":"2024-09-13T15:08:11.578845+0000","last_clean":"2024-09-13T15:08:11.578845+0000","last_became_active":"2024-09-13T15:02:23.883373+0000","last_became_peered":"2024-09-13T15:02:23.883373+0000","last_unstale":"2024-09-13T15:08:11.578845+0000","last_undegraded":"2024-09-13T15:08:11.578845+0000","last_fullsized":"2024-09-13T15:08:11.578845+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:03.107786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:03.107786+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:17:10.041468+0000","scrub_duration":0.0049024139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":744,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433330+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T15:08:15.433330+0000","last_peered":"2024-09-13T15:08:15.433330+0000","last_clean":"2024-09-13T15:08:15.433330+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T15:08:15.433330+0000","last_undegraded":"2024-09-13T15:08:15.433330+0000","last_fullsized":"2024-09-13T15:08:15.433330+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":412,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418241+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T15:08:12.418241+0000","last_peered":"2024-09-13T15:08:12.418241+0000","last_clean":"2024-09-13T15:08:12.418241+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T15:08:12.418241+0000","last_undegraded":"2024-09-13T15:08:12.418241+0000","last_fullsized":"2024-09-13T15:08:12.418241+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"82'100","reported_seq":198,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.261262+0000","last_change":"2024-09-13T15:05:52.014434+0000","last_active":"2024-09-13T15:08:15.261262+0000","last_peered":"2024-09-13T15:08:15.261262+0000","last_clean":"2024-09-13T15:08:15.261262+0000","last_became_active":"2024-09-13T15:05:52.014086+0000","last_became_peered":"2024-09-13T15:05:52.014086+0000","last_unstale":"2024-09-13T15:08:15.261262+0000","last_undegraded":"2024-09-13T15:08:15.261262+0000","last_fullsized":"2024-09-13T15:08:15.261262+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:51.192503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:51.192503+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:23.864648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":395,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.417729+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T15:08:12.417729+0000","last_peered":"2024-09-13T15:08:12.417729+0000","last_clean":"2024-09-13T15:08:12.417729+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T15:08:12.417729+0000","last_undegraded":"2024-09-13T15:08:12.417729+0000","last_fullsized":"2024-09-13T15:08:12.417729+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1241,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600586+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T15:08:11.600586+0000","last_peered":"2024-09-13T15:08:11.600586+0000","last_clean":"2024-09-13T15:08:11.600586+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T15:08:11.600586+0000","last_undegraded":"2024-09-13T15:08:11.600586+0000","last_fullsized":"2024-09-13T15:08:11.600586+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":737,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433489+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T15:08:15.433489+0000","last_peered":"2024-09-13T15:08:15.433489+0000","last_clean":"2024-09-13T15:08:15.433489+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T15:08:15.433489+0000","last_undegraded":"2024-09-13T15:08:15.433489+0000","last_fullsized":"2024-09-13T15:08:15.433489+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":102,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982276+0000","last_change":"2024-09-13T15:05:52.005786+0000","last_active":"2024-09-13T15:08:14.982276+0000","last_peered":"2024-09-13T15:08:14.982276+0000","last_clean":"2024-09-13T15:08:14.982276+0000","last_became_active":"2024-09-13T15:05:52.005453+0000","last_became_peered":"2024-09-13T15:05:52.005453+0000","last_unstale":"2024-09-13T15:08:14.982276+0000","last_undegraded":"2024-09-13T15:08:14.982276+0000","last_fullsized":"2024-09-13T15:08:14.982276+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:54.193156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:54.193156+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:23:40.816688+0000","scrub_duration":0.0040069900000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982019+0000","last_change":"2024-09-13T15:05:52.003257+0000","last_active":"2024-09-13T15:08:14.982019+0000","last_peered":"2024-09-13T15:08:14.982019+0000","last_clean":"2024-09-13T15:08:14.982019+0000","last_became_active":"2024-09-13T15:05:52.002919+0000","last_became_peered":"2024-09-13T15:05:52.002919+0000","last_unstale":"2024-09-13T15:08:14.982019+0000","last_undegraded":"2024-09-13T15:08:14.982019+0000","last_fullsized":"2024-09-13T15:08:14.982019+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:04.833597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":1086,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.261378+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T15:08:15.261378+0000","last_peered":"2024-09-13T15:08:15.261378+0000","last_clean":"2024-09-13T15:08:15.261378+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T15:08:15.261378+0000","last_undegraded":"2024-09-13T15:08:15.261378+0000","last_fullsized":"2024-09-13T15:08:15.261378+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":105,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982488+0000","last_change":"2024-09-13T15:05:52.014941+0000","last_active":"2024-09-13T15:08:14.982488+0000","last_peered":"2024-09-13T15:08:14.982488+0000","last_clean":"2024-09-13T15:08:14.982488+0000","last_became_active":"2024-09-13T15:05:52.002400+0000","last_became_peered":"2024-09-13T15:05:52.002400+0000","last_unstale":"2024-09-13T15:08:14.982488+0000","last_undegraded":"2024-09-13T15:08:14.982488+0000","last_fullsized":"2024-09-13T15:08:14.982488+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:06.227994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":399,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418081+0000","last_change":"2024-09-13T15:02:52.361444+0000","last_active":"2024-09-13T15:08:12.418081+0000","last_peered":"2024-09-13T15:08:12.418081+0000","last_clean":"2024-09-13T15:08:12.418081+0000","last_became_active":"2024-09-13T14:59:53.220135+0000","last_became_peered":"2024-09-13T14:59:53.220135+0000","last_unstale":"2024-09-13T15:08:12.418081+0000","last_undegraded":"2024-09-13T15:08:12.418081+0000","last_fullsized":"2024-09-13T15:08:12.418081+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.361359+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.361359+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:35.003140+0000","scrub_duration":0.005845154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":715,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433404+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T15:08:15.433404+0000","last_peered":"2024-09-13T15:08:15.433404+0000","last_clean":"2024-09-13T15:08:15.433404+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T15:08:15.433404+0000","last_undegraded":"2024-09-13T15:08:15.433404+0000","last_fullsized":"2024-09-13T15:08:15.433404+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":641,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578634+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T15:08:11.578634+0000","last_peered":"2024-09-13T15:08:11.578634+0000","last_clean":"2024-09-13T15:08:11.578634+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T15:08:11.578634+0000","last_undegraded":"2024-09-13T15:08:11.578634+0000","last_fullsized":"2024-09-13T15:08:11.578634+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":729,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.261009+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T15:08:15.261009+0000","last_peered":"2024-09-13T15:08:15.261009+0000","last_clean":"2024-09-13T15:08:15.261009+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T15:08:15.261009+0000","last_undegraded":"2024-09-13T15:08:15.261009+0000","last_fullsized":"2024-09-13T15:08:15.261009+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":267,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600523+0000","last_change":"2024-09-13T15:08:03.291737+0000","last_active":"2024-09-13T15:08:11.600523+0000","last_peered":"2024-09-13T15:08:11.600523+0000","last_clean":"2024-09-13T15:08:11.600523+0000","last_became_active":"2024-09-13T15:08:03.291318+0000","last_became_peered":"2024-09-13T15:08:03.291318+0000","last_unstale":"2024-09-13T15:08:11.600523+0000","last_undegraded":"2024-09-13T15:08:11.600523+0000","last_fullsized":"2024-09-13T15:08:11.600523+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:06:32.296605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:06:32.296605+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:01:20.023210+0000","scrub_duration":0.0026525849999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":704,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260967+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T15:08:15.260967+0000","last_peered":"2024-09-13T15:08:15.260967+0000","last_clean":"2024-09-13T15:08:15.260967+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T15:08:15.260967+0000","last_undegraded":"2024-09-13T15:08:15.260967+0000","last_fullsized":"2024-09-13T15:08:15.260967+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":1232,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.261062+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T15:08:15.261062+0000","last_peered":"2024-09-13T15:08:15.261062+0000","last_clean":"2024-09-13T15:08:15.261062+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T15:08:15.261062+0000","last_undegraded":"2024-09-13T15:08:15.261062+0000","last_fullsized":"2024-09-13T15:08:15.261062+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":429,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579083+0000","last_change":"2024-09-13T15:05:52.014958+0000","last_active":"2024-09-13T15:08:11.579083+0000","last_peered":"2024-09-13T15:08:11.579083+0000","last_clean":"2024-09-13T15:08:11.579083+0000","last_became_active":"2024-09-13T15:05:52.014471+0000","last_became_peered":"2024-09-13T15:05:52.014471+0000","last_unstale":"2024-09-13T15:08:11.579083+0000","last_undegraded":"2024-09-13T15:08:11.579083+0000","last_fullsized":"2024-09-13T15:08:11.579083+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:38.514748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":277,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600500+0000","last_change":"2024-09-13T15:08:03.292358+0000","last_active":"2024-09-13T15:08:11.600500+0000","last_peered":"2024-09-13T15:08:11.600500+0000","last_clean":"2024-09-13T15:08:11.600500+0000","last_became_active":"2024-09-13T15:08:03.292006+0000","last_became_peered":"2024-09-13T15:08:03.292006+0000","last_unstale":"2024-09-13T15:08:11.600500+0000","last_undegraded":"2024-09-13T15:08:11.600500+0000","last_fullsized":"2024-09-13T15:08:11.600500+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:49.771572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:49.771572+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:21.124308+0000","scrub_duration":0.0035237329999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":761,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260959+0000","last_change":"2024-09-13T15:05:52.008066+0000","last_active":"2024-09-13T15:08:15.260959+0000","last_peered":"2024-09-13T15:08:15.260959+0000","last_clean":"2024-09-13T15:08:15.260959+0000","last_became_active":"2024-09-13T15:05:52.007749+0000","last_became_peered":"2024-09-13T15:05:52.007749+0000","last_unstale":"2024-09-13T15:08:15.260959+0000","last_undegraded":"2024-09-13T15:08:15.260959+0000","last_fullsized":"2024-09-13T15:08:15.260959+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:11.201592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":413,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418367+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T15:08:12.418367+0000","last_peered":"2024-09-13T15:08:12.418367+0000","last_clean":"2024-09-13T15:08:12.418367+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T15:08:12.418367+0000","last_undegraded":"2024-09-13T15:08:12.418367+0000","last_fullsized":"2024-09-13T15:08:12.418367+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":1054,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579205+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T15:08:11.579205+0000","last_peered":"2024-09-13T15:08:11.579205+0000","last_clean":"2024-09-13T15:08:11.579205+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T15:08:11.579205+0000","last_undegraded":"2024-09-13T15:08:11.579205+0000","last_fullsized":"2024-09-13T15:08:11.579205+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":348,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578416+0000","last_change":"2024-09-13T14:56:01.658240+0000","last_active":"2024-09-13T15:08:11.578416+0000","last_peered":"2024-09-13T15:08:11.578416+0000","last_clean":"2024-09-13T15:08:11.578416+0000","last_became_active":"2024-09-13T14:55:51.160480+0000","last_became_peered":"2024-09-13T14:55:51.160480+0000","last_unstale":"2024-09-13T15:08:11.578416+0000","last_undegraded":"2024-09-13T15:08:11.578416+0000","last_fullsized":"2024-09-13T15:08:11.578416+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.658143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.658143+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:19.262955+0000","scrub_duration":0.0038903079999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":747,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.261254+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T15:08:15.261254+0000","last_peered":"2024-09-13T15:08:15.261254+0000","last_clean":"2024-09-13T15:08:15.261254+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T15:08:15.261254+0000","last_undegraded":"2024-09-13T15:08:15.261254+0000","last_fullsized":"2024-09-13T15:08:15.261254+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":729,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260779+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T15:08:15.260779+0000","last_peered":"2024-09-13T15:08:15.260779+0000","last_clean":"2024-09-13T15:08:15.260779+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T15:08:15.260779+0000","last_undegraded":"2024-09-13T15:08:15.260779+0000","last_fullsized":"2024-09-13T15:08:15.260779+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":959,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.592060+0000","last_change":"2024-09-13T15:07:37.444321+0000","last_active":"2024-09-13T15:08:14.592060+0000","last_peered":"2024-09-13T15:08:14.592060+0000","last_clean":"2024-09-13T15:08:14.592060+0000","last_became_active":"2024-09-13T15:07:37.443790+0000","last_became_peered":"2024-09-13T15:07:37.443790+0000","last_unstale":"2024-09-13T15:08:14.592060+0000","last_undegraded":"2024-09-13T15:08:14.592060+0000","last_fullsized":"2024-09-13T15:08:14.592060+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:27:35.063400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":205,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578164+0000","last_change":"2024-09-13T14:55:59.574827+0000","last_active":"2024-09-13T15:08:11.578164+0000","last_peered":"2024-09-13T15:08:11.578164+0000","last_clean":"2024-09-13T15:08:11.578164+0000","last_became_active":"2024-09-13T14:55:51.162903+0000","last_became_peered":"2024-09-13T14:55:51.162903+0000","last_unstale":"2024-09-13T15:08:11.578164+0000","last_undegraded":"2024-09-13T15:08:11.578164+0000","last_fullsized":"2024-09-13T15:08:11.578164+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.574773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.574773+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:39.074072+0000","scrub_duration":0.001553708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":751,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433626+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T15:08:15.433626+0000","last_peered":"2024-09-13T15:08:15.433626+0000","last_clean":"2024-09-13T15:08:15.433626+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T15:08:15.433626+0000","last_undegraded":"2024-09-13T15:08:15.433626+0000","last_fullsized":"2024-09-13T15:08:15.433626+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":761,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433106+0000","last_change":"2024-09-13T15:05:52.010114+0000","last_active":"2024-09-13T15:08:15.433106+0000","last_peered":"2024-09-13T15:08:15.433106+0000","last_clean":"2024-09-13T15:08:15.433106+0000","last_became_active":"2024-09-13T15:05:52.008634+0000","last_became_peered":"2024-09-13T15:05:52.008634+0000","last_unstale":"2024-09-13T15:08:15.433106+0000","last_undegraded":"2024-09-13T15:08:15.433106+0000","last_fullsized":"2024-09-13T15:08:15.433106+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:25.498296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":639,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.577939+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T15:08:11.577939+0000","last_peered":"2024-09-13T15:08:11.577939+0000","last_clean":"2024-09-13T15:08:11.577939+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T15:08:11.577939+0000","last_undegraded":"2024-09-13T15:08:11.577939+0000","last_fullsized":"2024-09-13T15:08:11.577939+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":882,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260623+0000","last_change":"2024-09-13T14:55:59.245841+0000","last_active":"2024-09-13T15:08:15.260623+0000","last_peered":"2024-09-13T15:08:15.260623+0000","last_clean":"2024-09-13T15:08:15.260623+0000","last_became_active":"2024-09-13T14:55:51.159487+0000","last_became_peered":"2024-09-13T14:55:51.159487+0000","last_unstale":"2024-09-13T15:08:15.260623+0000","last_undegraded":"2024-09-13T15:08:15.260623+0000","last_fullsized":"2024-09-13T15:08:15.260623+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.245213+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.245213+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:16.152016+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":212,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600434+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T15:08:11.600434+0000","last_peered":"2024-09-13T15:08:11.600434+0000","last_clean":"2024-09-13T15:08:11.600434+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T15:08:11.600434+0000","last_undegraded":"2024-09-13T15:08:11.600434+0000","last_fullsized":"2024-09-13T15:08:11.600434+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982110+0000","last_change":"2024-09-13T15:05:52.011318+0000","last_active":"2024-09-13T15:08:14.982110+0000","last_peered":"2024-09-13T15:08:14.982110+0000","last_clean":"2024-09-13T15:08:14.982110+0000","last_became_active":"2024-09-13T15:05:52.011038+0000","last_became_peered":"2024-09-13T15:05:52.011038+0000","last_unstale":"2024-09-13T15:08:14.982110+0000","last_undegraded":"2024-09-13T15:08:14.982110+0000","last_fullsized":"2024-09-13T15:08:14.982110+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:35.033102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":954,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578308+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T15:08:11.578308+0000","last_peered":"2024-09-13T15:08:11.578308+0000","last_clean":"2024-09-13T15:08:11.578308+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T15:08:11.578308+0000","last_undegraded":"2024-09-13T15:08:11.578308+0000","last_fullsized":"2024-09-13T15:08:11.578308+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":576,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578023+0000","last_change":"2024-09-13T14:55:58.568852+0000","last_active":"2024-09-13T15:08:11.578023+0000","last_peered":"2024-09-13T15:08:11.578023+0000","last_clean":"2024-09-13T15:08:11.578023+0000","last_became_active":"2024-09-13T14:55:51.169987+0000","last_became_peered":"2024-09-13T14:55:51.169987+0000","last_unstale":"2024-09-13T15:08:11.578023+0000","last_undegraded":"2024-09-13T15:08:11.578023+0000","last_fullsized":"2024-09-13T15:08:11.578023+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.568750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.568750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:56.273429+0000","scrub_duration":0.0074886340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":728,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.261294+0000","last_change":"2024-09-13T15:05:52.007594+0000","last_active":"2024-09-13T15:08:15.261294+0000","last_peered":"2024-09-13T15:08:15.261294+0000","last_clean":"2024-09-13T15:08:15.261294+0000","last_became_active":"2024-09-13T15:05:52.007146+0000","last_became_peered":"2024-09-13T15:05:52.007146+0000","last_unstale":"2024-09-13T15:08:15.261294+0000","last_undegraded":"2024-09-13T15:08:15.261294+0000","last_fullsized":"2024-09-13T15:08:15.261294+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:45.283696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.981900+0000","last_change":"2024-09-13T15:05:51.991616+0000","last_active":"2024-09-13T15:08:14.981900+0000","last_peered":"2024-09-13T15:08:14.981900+0000","last_clean":"2024-09-13T15:08:14.981900+0000","last_became_active":"2024-09-13T15:05:51.991294+0000","last_became_peered":"2024-09-13T15:05:51.991294+0000","last_unstale":"2024-09-13T15:08:14.981900+0000","last_undegraded":"2024-09-13T15:08:14.981900+0000","last_fullsized":"2024-09-13T15:08:14.981900+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:05.060508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":358,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578640+0000","last_change":"2024-09-13T15:02:58.197427+0000","last_active":"2024-09-13T15:08:11.578640+0000","last_peered":"2024-09-13T15:08:11.578640+0000","last_clean":"2024-09-13T15:08:11.578640+0000","last_became_active":"2024-09-13T14:59:53.232987+0000","last_became_peered":"2024-09-13T14:59:53.232987+0000","last_unstale":"2024-09-13T15:08:11.578640+0000","last_undegraded":"2024-09-13T15:08:11.578640+0000","last_fullsized":"2024-09-13T15:08:11.578640+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:58.195731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:58.195731+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:51.565489+0000","scrub_duration":0.011077536000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":412,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418326+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T15:08:12.418326+0000","last_peered":"2024-09-13T15:08:12.418326+0000","last_clean":"2024-09-13T15:08:12.418326+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T15:08:12.418326+0000","last_undegraded":"2024-09-13T15:08:12.418326+0000","last_fullsized":"2024-09-13T15:08:12.418326+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":712,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433214+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T15:08:15.433214+0000","last_peered":"2024-09-13T15:08:15.433214+0000","last_clean":"2024-09-13T15:08:15.433214+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T15:08:15.433214+0000","last_undegraded":"2024-09-13T15:08:15.433214+0000","last_fullsized":"2024-09-13T15:08:15.433214+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":910,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260977+0000","last_change":"2024-09-13T15:00:10.659910+0000","last_active":"2024-09-13T15:08:15.260977+0000","last_peered":"2024-09-13T15:08:15.260977+0000","last_clean":"2024-09-13T15:08:15.260977+0000","last_became_active":"2024-09-13T14:59:53.216739+0000","last_became_peered":"2024-09-13T14:59:53.216739+0000","last_unstale":"2024-09-13T15:08:15.260977+0000","last_undegraded":"2024-09-13T15:08:15.260977+0000","last_fullsized":"2024-09-13T15:08:15.260977+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.659838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.659838+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:16.890653+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":409,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418396+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T15:08:12.418396+0000","last_peered":"2024-09-13T15:08:12.418396+0000","last_clean":"2024-09-13T15:08:12.418396+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T15:08:12.418396+0000","last_undegraded":"2024-09-13T15:08:12.418396+0000","last_fullsized":"2024-09-13T15:08:12.418396+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":200,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600227+0000","last_change":"2024-09-13T15:08:03.399981+0000","last_active":"2024-09-13T15:08:11.600227+0000","last_peered":"2024-09-13T15:08:11.600227+0000","last_clean":"2024-09-13T15:08:11.600227+0000","last_became_active":"2024-09-13T15:08:03.399588+0000","last_became_peered":"2024-09-13T15:08:03.399588+0000","last_unstale":"2024-09-13T15:08:11.600227+0000","last_undegraded":"2024-09-13T15:08:11.600227+0000","last_fullsized":"2024-09-13T15:08:11.600227+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:49:52.771236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":379,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579244+0000","last_change":"2024-09-13T15:05:52.007408+0000","last_active":"2024-09-13T15:08:11.579244+0000","last_peered":"2024-09-13T15:08:11.579244+0000","last_clean":"2024-09-13T15:08:11.579244+0000","last_became_active":"2024-09-13T15:05:52.007032+0000","last_became_peered":"2024-09-13T15:05:52.007032+0000","last_unstale":"2024-09-13T15:08:11.579244+0000","last_undegraded":"2024-09-13T15:08:11.579244+0000","last_fullsized":"2024-09-13T15:08:11.579244+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:57.138789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:57.138789+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:28:20.106072+0000","scrub_duration":0.0036861670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":432,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418077+0000","last_change":"2024-09-13T15:05:51.998495+0000","last_active":"2024-09-13T15:08:12.418077+0000","last_peered":"2024-09-13T15:08:12.418077+0000","last_clean":"2024-09-13T15:08:12.418077+0000","last_became_active":"2024-09-13T15:05:51.998159+0000","last_became_peered":"2024-09-13T15:05:51.998159+0000","last_unstale":"2024-09-13T15:08:12.418077+0000","last_undegraded":"2024-09-13T15:08:12.418077+0000","last_fullsized":"2024-09-13T15:08:12.418077+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:23:05.948148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":105,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982168+0000","last_change":"2024-09-13T15:05:52.020378+0000","last_active":"2024-09-13T15:08:14.982168+0000","last_peered":"2024-09-13T15:08:14.982168+0000","last_clean":"2024-09-13T15:08:14.982168+0000","last_became_active":"2024-09-13T15:05:52.002686+0000","last_became_peered":"2024-09-13T15:05:52.002686+0000","last_unstale":"2024-09-13T15:08:14.982168+0000","last_undegraded":"2024-09-13T15:08:14.982168+0000","last_fullsized":"2024-09-13T15:08:14.982168+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.430732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":615,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.577870+0000","last_change":"2024-09-13T15:05:52.006880+0000","last_active":"2024-09-13T15:08:11.577870+0000","last_peered":"2024-09-13T15:08:11.577870+0000","last_clean":"2024-09-13T15:08:11.577870+0000","last_became_active":"2024-09-13T15:05:52.005704+0000","last_became_peered":"2024-09-13T15:05:52.005704+0000","last_unstale":"2024-09-13T15:08:11.577870+0000","last_undegraded":"2024-09-13T15:08:11.577870+0000","last_fullsized":"2024-09-13T15:08:11.577870+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:50.810045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:50.810045+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:39:30.729375+0000","scrub_duration":0.0042318529999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":725,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433543+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T15:08:15.433543+0000","last_peered":"2024-09-13T15:08:15.433543+0000","last_clean":"2024-09-13T15:08:15.433543+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T15:08:15.433543+0000","last_undegraded":"2024-09-13T15:08:15.433543+0000","last_fullsized":"2024-09-13T15:08:15.433543+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":432,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579130+0000","last_change":"2024-09-13T15:05:52.015239+0000","last_active":"2024-09-13T15:08:11.579130+0000","last_peered":"2024-09-13T15:08:11.579130+0000","last_clean":"2024-09-13T15:08:11.579130+0000","last_became_active":"2024-09-13T15:05:52.014825+0000","last_became_peered":"2024-09-13T15:05:52.014825+0000","last_unstale":"2024-09-13T15:08:11.579130+0000","last_undegraded":"2024-09-13T15:08:11.579130+0000","last_fullsized":"2024-09-13T15:08:11.579130+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:47.548807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":198,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418187+0000","last_change":"2024-09-13T15:05:51.996175+0000","last_active":"2024-09-13T15:08:12.418187+0000","last_peered":"2024-09-13T15:08:12.418187+0000","last_clean":"2024-09-13T15:08:12.418187+0000","last_became_active":"2024-09-13T15:05:51.995536+0000","last_became_peered":"2024-09-13T15:05:51.995536+0000","last_unstale":"2024-09-13T15:08:12.418187+0000","last_undegraded":"2024-09-13T15:08:12.418187+0000","last_fullsized":"2024-09-13T15:08:12.418187+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.358699+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.358699+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:58.331262+0000","scrub_duration":0.0023994049999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":232,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600153+0000","last_change":"2024-09-13T15:02:38.875727+0000","last_active":"2024-09-13T15:08:11.600153+0000","last_peered":"2024-09-13T15:08:11.600153+0000","last_clean":"2024-09-13T15:08:11.600153+0000","last_became_active":"2024-09-13T15:02:38.875372+0000","last_became_peered":"2024-09-13T15:02:38.875372+0000","last_unstale":"2024-09-13T15:08:11.600153+0000","last_undegraded":"2024-09-13T15:08:11.600153+0000","last_fullsized":"2024-09-13T15:08:11.600153+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:08.504810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":404,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578499+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T15:08:11.578499+0000","last_peered":"2024-09-13T15:08:11.578499+0000","last_clean":"2024-09-13T15:08:11.578499+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T15:08:11.578499+0000","last_undegraded":"2024-09-13T15:08:11.578499+0000","last_fullsized":"2024-09-13T15:08:11.578499+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":261,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600524+0000","last_change":"2024-09-13T14:56:08.889974+0000","last_active":"2024-09-13T15:08:11.600524+0000","last_peered":"2024-09-13T15:08:11.600524+0000","last_clean":"2024-09-13T15:08:11.600524+0000","last_became_active":"2024-09-13T14:55:55.185755+0000","last_became_peered":"2024-09-13T14:55:55.185755+0000","last_unstale":"2024-09-13T15:08:11.600524+0000","last_undegraded":"2024-09-13T15:08:11.600524+0000","last_fullsized":"2024-09-13T15:08:11.600524+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:08.889814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:08.889814+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:02.911564+0000","scrub_duration":0.0053731569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":469,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.417839+0000","last_change":"2024-09-13T15:05:52.000473+0000","last_active":"2024-09-13T15:08:12.417839+0000","last_peered":"2024-09-13T15:08:12.417839+0000","last_clean":"2024-09-13T15:08:12.417839+0000","last_became_active":"2024-09-13T15:05:52.000152+0000","last_became_peered":"2024-09-13T15:05:52.000152+0000","last_unstale":"2024-09-13T15:08:12.417839+0000","last_undegraded":"2024-09-13T15:08:12.417839+0000","last_fullsized":"2024-09-13T15:08:12.417839+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:31.530855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":394,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578698+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T15:08:11.578698+0000","last_peered":"2024-09-13T15:08:11.578698+0000","last_clean":"2024-09-13T15:08:11.578698+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T15:08:11.578698+0000","last_undegraded":"2024-09-13T15:08:11.578698+0000","last_fullsized":"2024-09-13T15:08:11.578698+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":106,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982342+0000","last_change":"2024-09-13T15:05:52.036891+0000","last_active":"2024-09-13T15:08:14.982342+0000","last_peered":"2024-09-13T15:08:14.982342+0000","last_clean":"2024-09-13T15:08:14.982342+0000","last_became_active":"2024-09-13T15:05:52.003251+0000","last_became_peered":"2024-09-13T15:05:52.003251+0000","last_unstale":"2024-09-13T15:08:14.982342+0000","last_undegraded":"2024-09-13T15:08:14.982342+0000","last_fullsized":"2024-09-13T15:08:14.982342+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.016260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.016260+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:42.653316+0000","scrub_duration":71,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":414,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579391+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T15:08:11.579391+0000","last_peered":"2024-09-13T15:08:11.579391+0000","last_clean":"2024-09-13T15:08:11.579391+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T15:08:11.579391+0000","last_undegraded":"2024-09-13T15:08:11.579391+0000","last_fullsized":"2024-09-13T15:08:11.579391+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":716,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.592072+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T15:08:14.592072+0000","last_peered":"2024-09-13T15:08:14.592072+0000","last_clean":"2024-09-13T15:08:14.592072+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T15:08:14.592072+0000","last_undegraded":"2024-09-13T15:08:14.592072+0000","last_fullsized":"2024-09-13T15:08:14.592072+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":178,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578278+0000","last_change":"2024-09-13T15:02:53.820519+0000","last_active":"2024-09-13T15:08:11.578278+0000","last_peered":"2024-09-13T15:08:11.578278+0000","last_clean":"2024-09-13T15:08:11.578278+0000","last_became_active":"2024-09-13T14:59:53.222306+0000","last_became_peered":"2024-09-13T14:59:53.222306+0000","last_unstale":"2024-09-13T15:08:11.578278+0000","last_undegraded":"2024-09-13T15:08:11.578278+0000","last_fullsized":"2024-09-13T15:08:11.578278+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:53.820435+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:53.820435+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:40.108691+0000","scrub_duration":0.0030873739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":222,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600183+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T15:08:11.600183+0000","last_peered":"2024-09-13T15:08:11.600183+0000","last_clean":"2024-09-13T15:08:11.600183+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T15:08:11.600183+0000","last_undegraded":"2024-09-13T15:08:11.600183+0000","last_fullsized":"2024-09-13T15:08:11.600183+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982570+0000","last_change":"2024-09-13T15:05:51.999191+0000","last_active":"2024-09-13T15:08:14.982570+0000","last_peered":"2024-09-13T15:08:14.982570+0000","last_clean":"2024-09-13T15:08:14.982570+0000","last_became_active":"2024-09-13T15:05:51.998885+0000","last_became_peered":"2024-09-13T15:05:51.998885+0000","last_unstale":"2024-09-13T15:08:14.982570+0000","last_undegraded":"2024-09-13T15:08:14.982570+0000","last_fullsized":"2024-09-13T15:08:14.982570+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:52.548839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":158,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578818+0000","last_change":"2024-09-13T14:56:03.605447+0000","last_active":"2024-09-13T15:08:11.578818+0000","last_peered":"2024-09-13T15:08:11.578818+0000","last_clean":"2024-09-13T15:08:11.578818+0000","last_became_active":"2024-09-13T14:55:55.218939+0000","last_became_peered":"2024-09-13T14:55:55.218939+0000","last_unstale":"2024-09-13T15:08:11.578818+0000","last_undegraded":"2024-09-13T15:08:11.578818+0000","last_fullsized":"2024-09-13T15:08:11.578818+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.605356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.605356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:12.467371+0000","scrub_duration":0.0020269620000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":395,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578112+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T15:08:11.578112+0000","last_peered":"2024-09-13T15:08:11.578112+0000","last_clean":"2024-09-13T15:08:11.578112+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T15:08:11.578112+0000","last_undegraded":"2024-09-13T15:08:11.578112+0000","last_fullsized":"2024-09-13T15:08:11.578112+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":752,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433470+0000","last_change":"2024-09-13T15:05:52.008360+0000","last_active":"2024-09-13T15:08:15.433470+0000","last_peered":"2024-09-13T15:08:15.433470+0000","last_clean":"2024-09-13T15:08:15.433470+0000","last_became_active":"2024-09-13T15:05:52.007789+0000","last_became_peered":"2024-09-13T15:05:52.007789+0000","last_unstale":"2024-09-13T15:08:15.433470+0000","last_undegraded":"2024-09-13T15:08:15.433470+0000","last_fullsized":"2024-09-13T15:08:15.433470+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:12:26.443562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":901,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260857+0000","last_change":"2024-09-13T15:05:52.009899+0000","last_active":"2024-09-13T15:08:15.260857+0000","last_peered":"2024-09-13T15:08:15.260857+0000","last_clean":"2024-09-13T15:08:15.260857+0000","last_became_active":"2024-09-13T15:05:52.009365+0000","last_became_peered":"2024-09-13T15:05:52.009365+0000","last_unstale":"2024-09-13T15:08:15.260857+0000","last_undegraded":"2024-09-13T15:08:15.260857+0000","last_fullsized":"2024-09-13T15:08:15.260857+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.240882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.240882+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:05:45.534365+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":759,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433208+0000","last_change":"2024-09-13T15:05:52.011548+0000","last_active":"2024-09-13T15:08:15.433208+0000","last_peered":"2024-09-13T15:08:15.433208+0000","last_clean":"2024-09-13T15:08:15.433208+0000","last_became_active":"2024-09-13T15:05:52.009924+0000","last_became_peered":"2024-09-13T15:05:52.009924+0000","last_unstale":"2024-09-13T15:08:15.433208+0000","last_undegraded":"2024-09-13T15:08:15.433208+0000","last_fullsized":"2024-09-13T15:08:15.433208+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:49.485860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":411,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578954+0000","last_change":"2024-09-13T15:05:52.014781+0000","last_active":"2024-09-13T15:08:11.578954+0000","last_peered":"2024-09-13T15:08:11.578954+0000","last_clean":"2024-09-13T15:08:11.578954+0000","last_became_active":"2024-09-13T15:05:52.014049+0000","last_became_peered":"2024-09-13T15:05:52.014049+0000","last_unstale":"2024-09-13T15:08:11.578954+0000","last_undegraded":"2024-09-13T15:08:11.578954+0000","last_fullsized":"2024-09-13T15:08:11.578954+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:33.528483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":158,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579157+0000","last_change":"2024-09-13T14:56:04.643672+0000","last_active":"2024-09-13T15:08:11.579157+0000","last_peered":"2024-09-13T15:08:11.579157+0000","last_clean":"2024-09-13T15:08:11.579157+0000","last_became_active":"2024-09-13T14:55:55.185995+0000","last_became_peered":"2024-09-13T14:55:55.185995+0000","last_unstale":"2024-09-13T15:08:11.579157+0000","last_undegraded":"2024-09-13T15:08:11.579157+0000","last_fullsized":"2024-09-13T15:08:11.579157+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.643614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.643614+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:36.495546+0000","scrub_duration":0.001441681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":394,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578269+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T15:08:11.578269+0000","last_peered":"2024-09-13T15:08:11.578269+0000","last_clean":"2024-09-13T15:08:11.578269+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T15:08:11.578269+0000","last_undegraded":"2024-09-13T15:08:11.578269+0000","last_fullsized":"2024-09-13T15:08:11.578269+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982372+0000","last_change":"2024-09-13T15:05:52.015281+0000","last_active":"2024-09-13T15:08:14.982372+0000","last_peered":"2024-09-13T15:08:14.982372+0000","last_clean":"2024-09-13T15:08:14.982372+0000","last_became_active":"2024-09-13T15:05:52.014517+0000","last_became_peered":"2024-09-13T15:05:52.014517+0000","last_unstale":"2024-09-13T15:08:14.982372+0000","last_undegraded":"2024-09-13T15:08:14.982372+0000","last_fullsized":"2024-09-13T15:08:14.982372+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:10:37.226490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"82'100","reported_seq":469,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433226+0000","last_change":"2024-09-13T15:05:52.008756+0000","last_active":"2024-09-13T15:08:15.433226+0000","last_peered":"2024-09-13T15:08:15.433226+0000","last_clean":"2024-09-13T15:08:15.433226+0000","last_became_active":"2024-09-13T15:05:52.007609+0000","last_became_peered":"2024-09-13T15:05:52.007609+0000","last_unstale":"2024-09-13T15:08:15.433226+0000","last_undegraded":"2024-09-13T15:08:15.433226+0000","last_fullsized":"2024-09-13T15:08:15.433226+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.087994+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.087994+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:33:22.602291+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":16262,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24498,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398021,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":887580,"kb_used_data":6480,"kb_used_omap":1,"kb_used_meta":881086,"kb_avail":92857572,"statfs":{"total":95995035648,"available":95086153728,"internally_reserved":0,"allocated":6635520,"data_stored":3696576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":902232408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594250,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":324624,"kb_used_data":10628,"kb_used_omap":1,"kb_used_meta":313982,"kb_avail":93420528,"statfs":{"total":95995035648,"available":95662620672,"internally_reserved":0,"allocated":10883072,"data_stored":7955444,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":321517912},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790479,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1195980,"kb_used_data":10560,"kb_used_omap":1,"kb_used_meta":1185406,"kb_avail":92549172,"statfs":{"total":95995035648,"available":94770352128,"internally_reserved":0,"allocated":10813440,"data_stored":7886349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1213856088},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986708,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907596,"kb_used_data":6208,"kb_used_omap":1,"kb_used_meta":901374,"kb_avail":92837556,"statfs":{"total":95995035648,"available":95065657344,"internally_reserved":0,"allocated":6356992,"data_stored":3429088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923007320},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":335,"seq":1438814044255,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":123904,"kb_used_data":6516,"kb_used_omap":10,"kb_used_meta":117365,"kb_avail":93621248,"statfs":{"total":95995035648,"available":95868157952,"internally_reserved":0,"allocated":6672384,"data_stored":3756496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10273,"internal_metadata":120182751},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762682,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1051600,"kb_used_data":6464,"kb_used_omap":8,"kb_used_meta":1045111,"kb_avail":92693552,"statfs":{"total":95995035648,"available":94918197248,"internally_reserved":0,"allocated":6619136,"data_stored":3692040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":1070194122},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926209,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1015364,"kb_used_data":10296,"kb_used_omap":7,"kb_used_meta":1005048,"kb_avail":92729788,"statfs":{"total":95995035648,"available":94955302912,"internally_reserved":0,"allocated":10543104,"data_stored":7623293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":1029169865},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122448,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1035860,"kb_used_data":6536,"kb_used_omap":7,"kb_used_meta":1029304,"kb_avail":92709292,"statfs":{"total":95995035648,"available":94934315008,"internally_reserved":0,"allocated":6692864,"data_stored":3761986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":1054007887},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7266,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:09:00.826 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:09:01.072 INFO:teuthology.orchestra.run.smithi096.stderr:Inferring config /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:01.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:09:01 smithi096 ceph-mon[98901]: from='client.46940 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:09:01.823 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:09:01 smithi096 ceph-mon[98901]: pgmap v1441: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:09:01.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:09:01 smithi096 ceph-mon[102928]: from='client.46940 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:09:01.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:09:01 smithi096 ceph-mon[102928]: pgmap v1441: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:09:01.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:09:01 smithi204 ceph-mon[87071]: from='client.46940 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:09:01.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:09:01 smithi204 ceph-mon[87071]: pgmap v1441: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:09:02.958 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-13T15:09:02.959 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2024-09-13T15:09:03.765 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":1442,"stamp":"2024-09-13T15:09:02.275010+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":907,"num_write_kb":12623,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8732588,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4132,"ondisk_log_size":4132,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6542508,"kb_used_data":63688,"kb_used_omap":39,"kb_used_meta":6478680,"kb_avail":743418708,"statfs":{"total":767960285184,"available":761260756992,"internally_reserved":0,"allocated":65216512,"data_stored":41801272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40927,"internal_metadata":6634168353},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002630"},"pg_stats":[{"pgid":"3.1f","version":"64'4","reported_seq":768,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433423+0000","last_change":"2024-09-13T14:51:02.038847+0000","last_active":"2024-09-13T15:08:15.433423+0000","last_peered":"2024-09-13T15:08:15.433423+0000","last_clean":"2024-09-13T15:08:15.433423+0000","last_became_active":"2024-09-13T14:51:02.007889+0000","last_became_peered":"2024-09-13T14:51:02.007889+0000","last_unstale":"2024-09-13T15:08:15.433423+0000","last_undegraded":"2024-09-13T15:08:15.433423+0000","last_fullsized":"2024-09-13T15:08:15.433423+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:44.636477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":415,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.417737+0000","last_change":"2024-09-13T14:48:32.197450+0000","last_active":"2024-09-13T15:08:12.417737+0000","last_peered":"2024-09-13T15:08:12.417737+0000","last_clean":"2024-09-13T15:08:12.417737+0000","last_became_active":"2024-09-13T14:48:32.197171+0000","last_became_peered":"2024-09-13T14:48:32.197171+0000","last_unstale":"2024-09-13T15:08:12.417737+0000","last_undegraded":"2024-09-13T15:08:12.417737+0000","last_fullsized":"2024-09-13T15:08:12.417737+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:36.544087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"82'100","reported_seq":200,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.261092+0000","last_change":"2024-09-13T15:05:52.013759+0000","last_active":"2024-09-13T15:08:15.261092+0000","last_peered":"2024-09-13T15:08:15.261092+0000","last_clean":"2024-09-13T15:08:15.261092+0000","last_became_active":"2024-09-13T15:05:52.013210+0000","last_became_peered":"2024-09-13T15:05:52.013210+0000","last_unstale":"2024-09-13T15:08:15.261092+0000","last_undegraded":"2024-09-13T15:08:15.261092+0000","last_fullsized":"2024-09-13T15:08:15.261092+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.149290+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.149290+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:10.545939+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982560+0000","last_change":"2024-09-13T15:05:51.995035+0000","last_active":"2024-09-13T15:08:14.982560+0000","last_peered":"2024-09-13T15:08:14.982560+0000","last_clean":"2024-09-13T15:08:14.982560+0000","last_became_active":"2024-09-13T15:05:51.994496+0000","last_became_peered":"2024-09-13T15:05:51.994496+0000","last_unstale":"2024-09-13T15:08:14.982560+0000","last_undegraded":"2024-09-13T15:08:14.982560+0000","last_fullsized":"2024-09-13T15:08:14.982560+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:11:51.103755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982168+0000","last_change":"2024-09-13T15:05:52.004131+0000","last_active":"2024-09-13T15:08:14.982168+0000","last_peered":"2024-09-13T15:08:14.982168+0000","last_clean":"2024-09-13T15:08:14.982168+0000","last_became_active":"2024-09-13T15:05:52.003845+0000","last_became_peered":"2024-09-13T15:05:52.003845+0000","last_unstale":"2024-09-13T15:08:14.982168+0000","last_undegraded":"2024-09-13T15:08:14.982168+0000","last_fullsized":"2024-09-13T15:08:14.982168+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:01.268036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"82'100","reported_seq":86,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578220+0000","last_change":"2024-09-13T15:02:23.879862+0000","last_active":"2024-09-13T15:08:11.578220+0000","last_peered":"2024-09-13T15:08:11.578220+0000","last_clean":"2024-09-13T15:08:11.578220+0000","last_became_active":"2024-09-13T15:02:23.879170+0000","last_became_peered":"2024-09-13T15:02:23.879170+0000","last_unstale":"2024-09-13T15:08:11.578220+0000","last_undegraded":"2024-09-13T15:08:11.578220+0000","last_fullsized":"2024-09-13T15:08:11.578220+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:06.945525+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:06.945525+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:48.496118+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":433,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578967+0000","last_change":"2024-09-13T14:48:32.211446+0000","last_active":"2024-09-13T15:08:11.578967+0000","last_peered":"2024-09-13T15:08:11.578967+0000","last_clean":"2024-09-13T15:08:11.578967+0000","last_became_active":"2024-09-13T14:48:32.205156+0000","last_became_peered":"2024-09-13T14:48:32.205156+0000","last_unstale":"2024-09-13T15:08:11.578967+0000","last_undegraded":"2024-09-13T15:08:11.578967+0000","last_fullsized":"2024-09-13T15:08:11.578967+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:30.192198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":429,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418029+0000","last_change":"2024-09-13T15:05:51.995355+0000","last_active":"2024-09-13T15:08:12.418029+0000","last_peered":"2024-09-13T15:08:12.418029+0000","last_clean":"2024-09-13T15:08:12.418029+0000","last_became_active":"2024-09-13T15:05:51.995038+0000","last_became_peered":"2024-09-13T15:05:51.995038+0000","last_unstale":"2024-09-13T15:08:12.418029+0000","last_undegraded":"2024-09-13T15:08:12.418029+0000","last_fullsized":"2024-09-13T15:08:12.418029+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:24:14.413713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":394,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579092+0000","last_change":"2024-09-13T14:37:09.292555+0000","last_active":"2024-09-13T15:08:11.579092+0000","last_peered":"2024-09-13T15:08:11.579092+0000","last_clean":"2024-09-13T15:08:11.579092+0000","last_became_active":"2024-09-13T14:37:09.292214+0000","last_became_peered":"2024-09-13T14:37:09.292214+0000","last_unstale":"2024-09-13T15:08:11.579092+0000","last_undegraded":"2024-09-13T15:08:11.579092+0000","last_fullsized":"2024-09-13T15:08:11.579092+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.373391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":429,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418190+0000","last_change":"2024-09-13T15:05:51.998541+0000","last_active":"2024-09-13T15:08:12.418190+0000","last_peered":"2024-09-13T15:08:12.418190+0000","last_clean":"2024-09-13T15:08:12.418190+0000","last_became_active":"2024-09-13T15:05:51.998234+0000","last_became_peered":"2024-09-13T15:05:51.998234+0000","last_unstale":"2024-09-13T15:08:12.418190+0000","last_undegraded":"2024-09-13T15:08:12.418190+0000","last_fullsized":"2024-09-13T15:08:12.418190+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:25.705700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"82'100","reported_seq":110,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982545+0000","last_change":"2024-09-13T15:05:52.535449+0000","last_active":"2024-09-13T15:08:14.982545+0000","last_peered":"2024-09-13T15:08:14.982545+0000","last_clean":"2024-09-13T15:08:14.982545+0000","last_became_active":"2024-09-13T15:05:51.991505+0000","last_became_peered":"2024-09-13T15:05:51.991505+0000","last_unstale":"2024-09-13T15:08:14.982545+0000","last_undegraded":"2024-09-13T15:08:14.982545+0000","last_fullsized":"2024-09-13T15:08:14.982545+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:05:52.535365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:05:52.535365+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:13:01.193300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":716,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433356+0000","last_change":"2024-09-13T14:35:49.989800+0000","last_active":"2024-09-13T15:08:15.433356+0000","last_peered":"2024-09-13T15:08:15.433356+0000","last_clean":"2024-09-13T15:08:15.433356+0000","last_became_active":"2024-09-13T14:35:49.989131+0000","last_became_peered":"2024-09-13T14:35:49.989131+0000","last_unstale":"2024-09-13T15:08:15.433356+0000","last_undegraded":"2024-09-13T15:08:15.433356+0000","last_fullsized":"2024-09-13T15:08:15.433356+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:02.503604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":496,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.591873+0000","last_change":"2024-09-13T15:07:37.914968+0000","last_active":"2024-09-13T15:08:14.591873+0000","last_peered":"2024-09-13T15:08:14.591873+0000","last_clean":"2024-09-13T15:08:14.591873+0000","last_became_active":"2024-09-13T15:07:37.914657+0000","last_became_peered":"2024-09-13T15:07:37.914657+0000","last_unstale":"2024-09-13T15:08:14.591873+0000","last_undegraded":"2024-09-13T15:08:14.591873+0000","last_fullsized":"2024-09-13T15:08:14.591873+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:55.882895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"82'100","reported_seq":102,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982075+0000","last_change":"2024-09-13T15:05:52.002778+0000","last_active":"2024-09-13T15:08:14.982075+0000","last_peered":"2024-09-13T15:08:14.982075+0000","last_clean":"2024-09-13T15:08:14.982075+0000","last_became_active":"2024-09-13T15:05:52.002405+0000","last_became_peered":"2024-09-13T15:05:52.002405+0000","last_unstale":"2024-09-13T15:08:14.982075+0000","last_undegraded":"2024-09-13T15:08:14.982075+0000","last_fullsized":"2024-09-13T15:08:14.982075+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.429543+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.429543+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:21:59.195306+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":411,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418004+0000","last_change":"2024-09-13T14:37:40.926838+0000","last_active":"2024-09-13T15:08:12.418004+0000","last_peered":"2024-09-13T15:08:12.418004+0000","last_clean":"2024-09-13T15:08:12.418004+0000","last_became_active":"2024-09-13T14:37:40.926149+0000","last_became_peered":"2024-09-13T14:37:40.926149+0000","last_unstale":"2024-09-13T15:08:12.418004+0000","last_undegraded":"2024-09-13T15:08:12.418004+0000","last_fullsized":"2024-09-13T15:08:12.418004+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:47.725636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982467+0000","last_change":"2024-09-13T15:05:52.001307+0000","last_active":"2024-09-13T15:08:14.982467+0000","last_peered":"2024-09-13T15:08:14.982467+0000","last_clean":"2024-09-13T15:08:14.982467+0000","last_became_active":"2024-09-13T15:05:52.001040+0000","last_became_peered":"2024-09-13T15:05:52.001040+0000","last_unstale":"2024-09-13T15:08:14.982467+0000","last_undegraded":"2024-09-13T15:08:14.982467+0000","last_fullsized":"2024-09-13T15:08:14.982467+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:49:48.758670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"82'100","reported_seq":383,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579377+0000","last_change":"2024-09-13T15:03:04.101898+0000","last_active":"2024-09-13T15:08:11.579377+0000","last_peered":"2024-09-13T15:08:11.579377+0000","last_clean":"2024-09-13T15:08:11.579377+0000","last_became_active":"2024-09-13T15:02:23.884406+0000","last_became_peered":"2024-09-13T15:02:23.884406+0000","last_unstale":"2024-09-13T15:08:11.579377+0000","last_undegraded":"2024-09-13T15:08:11.579377+0000","last_fullsized":"2024-09-13T15:08:11.579377+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:04.101802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:04.101802+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:34.459623+0000","scrub_duration":0.003875819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":727,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260773+0000","last_change":"2024-09-13T15:05:52.007634+0000","last_active":"2024-09-13T15:08:15.260773+0000","last_peered":"2024-09-13T15:08:15.260773+0000","last_clean":"2024-09-13T15:08:15.260773+0000","last_became_active":"2024-09-13T15:05:52.007302+0000","last_became_peered":"2024-09-13T15:05:52.007302+0000","last_unstale":"2024-09-13T15:08:15.260773+0000","last_undegraded":"2024-09-13T15:08:15.260773+0000","last_fullsized":"2024-09-13T15:08:15.260773+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:36.203235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":762,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433566+0000","last_change":"2024-09-13T15:05:52.008086+0000","last_active":"2024-09-13T15:08:15.433566+0000","last_peered":"2024-09-13T15:08:15.433566+0000","last_clean":"2024-09-13T15:08:15.433566+0000","last_became_active":"2024-09-13T15:05:52.007347+0000","last_became_peered":"2024-09-13T15:05:52.007347+0000","last_unstale":"2024-09-13T15:08:15.433566+0000","last_undegraded":"2024-09-13T15:08:15.433566+0000","last_fullsized":"2024-09-13T15:08:15.433566+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:12:09.381403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"82'100","reported_seq":678,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578095+0000","last_change":"2024-09-13T15:03:07.166808+0000","last_active":"2024-09-13T15:08:11.578095+0000","last_peered":"2024-09-13T15:08:11.578095+0000","last_clean":"2024-09-13T15:08:11.578095+0000","last_became_active":"2024-09-13T15:02:23.883387+0000","last_became_peered":"2024-09-13T15:02:23.883387+0000","last_unstale":"2024-09-13T15:08:11.578095+0000","last_undegraded":"2024-09-13T15:08:11.578095+0000","last_fullsized":"2024-09-13T15:08:11.578095+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:07.166034+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:07.166034+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:32:42.605595+0000","scrub_duration":0.01094734,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982061+0000","last_change":"2024-09-13T15:05:52.003501+0000","last_active":"2024-09-13T15:08:14.982061+0000","last_peered":"2024-09-13T15:08:14.982061+0000","last_clean":"2024-09-13T15:08:14.982061+0000","last_became_active":"2024-09-13T15:05:52.003200+0000","last_became_peered":"2024-09-13T15:05:52.003200+0000","last_unstale":"2024-09-13T15:08:14.982061+0000","last_undegraded":"2024-09-13T15:08:14.982061+0000","last_fullsized":"2024-09-13T15:08:14.982061+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:46:07.106935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":212,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600577+0000","last_change":"2024-09-13T14:48:32.205555+0000","last_active":"2024-09-13T15:08:11.600577+0000","last_peered":"2024-09-13T15:08:11.600577+0000","last_clean":"2024-09-13T15:08:11.600577+0000","last_became_active":"2024-09-13T14:48:32.204794+0000","last_became_peered":"2024-09-13T14:48:32.204794+0000","last_unstale":"2024-09-13T15:08:11.600577+0000","last_undegraded":"2024-09-13T15:08:11.600577+0000","last_fullsized":"2024-09-13T15:08:11.600577+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:21.939663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"82'100","reported_seq":96,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.417783+0000","last_change":"2024-09-13T15:02:56.319389+0000","last_active":"2024-09-13T15:08:12.417783+0000","last_peered":"2024-09-13T15:08:12.417783+0000","last_clean":"2024-09-13T15:08:12.417783+0000","last_became_active":"2024-09-13T15:02:23.876739+0000","last_became_peered":"2024-09-13T15:02:23.876739+0000","last_unstale":"2024-09-13T15:08:12.417783+0000","last_undegraded":"2024-09-13T15:08:12.417783+0000","last_fullsized":"2024-09-13T15:08:12.417783+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:56.319302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:56.319302+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:58.230158+0000","scrub_duration":0.0031182190000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.981928+0000","last_change":"2024-09-13T15:05:51.999327+0000","last_active":"2024-09-13T15:08:14.981928+0000","last_peered":"2024-09-13T15:08:14.981928+0000","last_clean":"2024-09-13T15:08:14.981928+0000","last_became_active":"2024-09-13T15:05:51.994485+0000","last_became_peered":"2024-09-13T15:05:51.994485+0000","last_unstale":"2024-09-13T15:08:14.981928+0000","last_undegraded":"2024-09-13T15:08:14.981928+0000","last_fullsized":"2024-09-13T15:08:14.981928+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:55:49.041649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":737,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433505+0000","last_change":"2024-09-13T15:05:52.006782+0000","last_active":"2024-09-13T15:08:15.433505+0000","last_peered":"2024-09-13T15:08:15.433505+0000","last_clean":"2024-09-13T15:08:15.433505+0000","last_became_active":"2024-09-13T15:05:52.006452+0000","last_became_peered":"2024-09-13T15:05:52.006452+0000","last_unstale":"2024-09-13T15:08:15.433505+0000","last_undegraded":"2024-09-13T15:08:15.433505+0000","last_fullsized":"2024-09-13T15:08:15.433505+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:37:23.855899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"82'100","reported_seq":96,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578641+0000","last_change":"2024-09-13T15:02:55.870615+0000","last_active":"2024-09-13T15:08:11.578641+0000","last_peered":"2024-09-13T15:08:11.578641+0000","last_clean":"2024-09-13T15:08:11.578641+0000","last_became_active":"2024-09-13T15:02:23.883217+0000","last_became_peered":"2024-09-13T15:02:23.883217+0000","last_unstale":"2024-09-13T15:08:11.578641+0000","last_undegraded":"2024-09-13T15:08:11.578641+0000","last_fullsized":"2024-09-13T15:08:11.578641+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.870528+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.870528+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:10.023787+0000","scrub_duration":0.0047551850000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982226+0000","last_change":"2024-09-13T15:05:52.010150+0000","last_active":"2024-09-13T15:08:14.982226+0000","last_peered":"2024-09-13T15:08:14.982226+0000","last_clean":"2024-09-13T15:08:14.982226+0000","last_became_active":"2024-09-13T15:05:52.008977+0000","last_became_peered":"2024-09-13T15:05:52.008977+0000","last_unstale":"2024-09-13T15:08:14.982226+0000","last_undegraded":"2024-09-13T15:08:14.982226+0000","last_fullsized":"2024-09-13T15:08:14.982226+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:43.067814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":392,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578892+0000","last_change":"2024-09-13T14:37:09.288985+0000","last_active":"2024-09-13T15:08:11.578892+0000","last_peered":"2024-09-13T15:08:11.578892+0000","last_clean":"2024-09-13T15:08:11.578892+0000","last_became_active":"2024-09-13T14:37:09.288612+0000","last_became_peered":"2024-09-13T14:37:09.288612+0000","last_unstale":"2024-09-13T15:08:11.578892+0000","last_undegraded":"2024-09-13T15:08:11.578892+0000","last_fullsized":"2024-09-13T15:08:11.578892+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:26.880593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":310,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.591761+0000","last_change":"2024-09-13T14:56:01.235655+0000","last_active":"2024-09-13T15:08:14.591761+0000","last_peered":"2024-09-13T15:08:14.591761+0000","last_clean":"2024-09-13T15:08:14.591761+0000","last_became_active":"2024-09-13T14:55:55.195703+0000","last_became_peered":"2024-09-13T14:55:55.195703+0000","last_unstale":"2024-09-13T15:08:14.591761+0000","last_undegraded":"2024-09-13T15:08:14.591761+0000","last_fullsized":"2024-09-13T15:08:14.591761+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.235497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.235497+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:39:43.495759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":434,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578402+0000","last_change":"2024-09-13T15:05:52.016761+0000","last_active":"2024-09-13T15:08:11.578402+0000","last_peered":"2024-09-13T15:08:11.578402+0000","last_clean":"2024-09-13T15:08:11.578402+0000","last_became_active":"2024-09-13T15:05:52.005623+0000","last_became_peered":"2024-09-13T15:05:52.005623+0000","last_unstale":"2024-09-13T15:08:11.578402+0000","last_undegraded":"2024-09-13T15:08:11.578402+0000","last_fullsized":"2024-09-13T15:08:11.578402+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:58:43.424615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":727,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260818+0000","last_change":"2024-09-13T14:48:32.205983+0000","last_active":"2024-09-13T15:08:15.260818+0000","last_peered":"2024-09-13T15:08:15.260818+0000","last_clean":"2024-09-13T15:08:15.260818+0000","last_became_active":"2024-09-13T14:48:32.205574+0000","last_became_peered":"2024-09-13T14:48:32.205574+0000","last_unstale":"2024-09-13T15:08:15.260818+0000","last_undegraded":"2024-09-13T15:08:15.260818+0000","last_fullsized":"2024-09-13T15:08:15.260818+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:22.072710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.17","version":"82'100","reported_seq":240,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578845+0000","last_change":"2024-09-13T15:03:03.108031+0000","last_active":"2024-09-13T15:08:11.578845+0000","last_peered":"2024-09-13T15:08:11.578845+0000","last_clean":"2024-09-13T15:08:11.578845+0000","last_became_active":"2024-09-13T15:02:23.883373+0000","last_became_peered":"2024-09-13T15:02:23.883373+0000","last_unstale":"2024-09-13T15:08:11.578845+0000","last_undegraded":"2024-09-13T15:08:11.578845+0000","last_fullsized":"2024-09-13T15:08:11.578845+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:03:03.107786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:03:03.107786+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:17:10.041468+0000","scrub_duration":0.0049024139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":744,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433330+0000","last_change":"2024-09-13T14:48:32.219851+0000","last_active":"2024-09-13T15:08:15.433330+0000","last_peered":"2024-09-13T15:08:15.433330+0000","last_clean":"2024-09-13T15:08:15.433330+0000","last_became_active":"2024-09-13T14:48:32.208703+0000","last_became_peered":"2024-09-13T14:48:32.208703+0000","last_unstale":"2024-09-13T15:08:15.433330+0000","last_undegraded":"2024-09-13T15:08:15.433330+0000","last_fullsized":"2024-09-13T15:08:15.433330+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.611212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":412,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418241+0000","last_change":"2024-09-13T14:48:32.203397+0000","last_active":"2024-09-13T15:08:12.418241+0000","last_peered":"2024-09-13T15:08:12.418241+0000","last_clean":"2024-09-13T15:08:12.418241+0000","last_became_active":"2024-09-13T14:48:32.203119+0000","last_became_peered":"2024-09-13T14:48:32.203119+0000","last_unstale":"2024-09-13T15:08:12.418241+0000","last_undegraded":"2024-09-13T15:08:12.418241+0000","last_fullsized":"2024-09-13T15:08:12.418241+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:12.309300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"82'100","reported_seq":198,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.261262+0000","last_change":"2024-09-13T15:05:52.014434+0000","last_active":"2024-09-13T15:08:15.261262+0000","last_peered":"2024-09-13T15:08:15.261262+0000","last_clean":"2024-09-13T15:08:15.261262+0000","last_became_active":"2024-09-13T15:05:52.014086+0000","last_became_peered":"2024-09-13T15:05:52.014086+0000","last_unstale":"2024-09-13T15:08:15.261262+0000","last_undegraded":"2024-09-13T15:08:15.261262+0000","last_fullsized":"2024-09-13T15:08:15.261262+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":325,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:51.192503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:51.192503+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:23.864648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":395,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.417729+0000","last_change":"2024-09-13T14:48:32.198210+0000","last_active":"2024-09-13T15:08:12.417729+0000","last_peered":"2024-09-13T15:08:12.417729+0000","last_clean":"2024-09-13T15:08:12.417729+0000","last_became_active":"2024-09-13T14:48:32.197841+0000","last_became_peered":"2024-09-13T14:48:32.197841+0000","last_unstale":"2024-09-13T15:08:12.417729+0000","last_undegraded":"2024-09-13T15:08:12.417729+0000","last_fullsized":"2024-09-13T15:08:12.417729+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:39.672599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"78'195","reported_seq":1241,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600586+0000","last_change":"2024-09-13T14:51:47.215777+0000","last_active":"2024-09-13T15:08:11.600586+0000","last_peered":"2024-09-13T15:08:11.600586+0000","last_clean":"2024-09-13T15:08:11.600586+0000","last_became_active":"2024-09-13T14:51:47.215241+0000","last_became_peered":"2024-09-13T14:51:47.215241+0000","last_unstale":"2024-09-13T15:08:11.600586+0000","last_undegraded":"2024-09-13T15:08:11.600586+0000","last_fullsized":"2024-09-13T15:08:11.600586+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:46.078919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092384900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1891,"num_read_kb":2363,"num_write":145,"num_write_kb":419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":737,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433489+0000","last_change":"2024-09-13T14:37:40.926507+0000","last_active":"2024-09-13T15:08:15.433489+0000","last_peered":"2024-09-13T15:08:15.433489+0000","last_clean":"2024-09-13T15:08:15.433489+0000","last_became_active":"2024-09-13T14:37:40.926213+0000","last_became_peered":"2024-09-13T14:37:40.926213+0000","last_unstale":"2024-09-13T15:08:15.433489+0000","last_undegraded":"2024-09-13T15:08:15.433489+0000","last_fullsized":"2024-09-13T15:08:15.433489+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:41.696195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":102,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982276+0000","last_change":"2024-09-13T15:05:52.005786+0000","last_active":"2024-09-13T15:08:14.982276+0000","last_peered":"2024-09-13T15:08:14.982276+0000","last_clean":"2024-09-13T15:08:14.982276+0000","last_became_active":"2024-09-13T15:05:52.005453+0000","last_became_peered":"2024-09-13T15:05:52.005453+0000","last_unstale":"2024-09-13T15:08:14.982276+0000","last_undegraded":"2024-09-13T15:08:14.982276+0000","last_fullsized":"2024-09-13T15:08:14.982276+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:54.193156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:54.193156+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:23:40.816688+0000","scrub_duration":0.0040069900000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982019+0000","last_change":"2024-09-13T15:05:52.003257+0000","last_active":"2024-09-13T15:08:14.982019+0000","last_peered":"2024-09-13T15:08:14.982019+0000","last_clean":"2024-09-13T15:08:14.982019+0000","last_became_active":"2024-09-13T15:05:52.002919+0000","last_became_peered":"2024-09-13T15:05:52.002919+0000","last_unstale":"2024-09-13T15:08:14.982019+0000","last_undegraded":"2024-09-13T15:08:14.982019+0000","last_fullsized":"2024-09-13T15:08:14.982019+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:04.833597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'126","reported_seq":1086,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.261378+0000","last_change":"2024-09-13T14:48:35.123678+0000","last_active":"2024-09-13T15:08:15.261378+0000","last_peered":"2024-09-13T15:08:15.261378+0000","last_clean":"2024-09-13T15:08:15.261378+0000","last_became_active":"2024-09-13T14:48:35.123120+0000","last_became_peered":"2024-09-13T14:48:35.123120+0000","last_unstale":"2024-09-13T15:08:15.261378+0000","last_undegraded":"2024-09-13T15:08:15.261378+0000","last_fullsized":"2024-09-13T15:08:15.261378+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:23.578146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019299040000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":387,"num_read_kb":3654,"num_write":107,"num_write_kb":3267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":105,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982488+0000","last_change":"2024-09-13T15:05:52.014941+0000","last_active":"2024-09-13T15:08:14.982488+0000","last_peered":"2024-09-13T15:08:14.982488+0000","last_clean":"2024-09-13T15:08:14.982488+0000","last_became_active":"2024-09-13T15:05:52.002400+0000","last_became_peered":"2024-09-13T15:05:52.002400+0000","last_unstale":"2024-09-13T15:08:14.982488+0000","last_undegraded":"2024-09-13T15:08:14.982488+0000","last_fullsized":"2024-09-13T15:08:14.982488+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:06.227994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":399,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418081+0000","last_change":"2024-09-13T15:02:52.361444+0000","last_active":"2024-09-13T15:08:12.418081+0000","last_peered":"2024-09-13T15:08:12.418081+0000","last_clean":"2024-09-13T15:08:12.418081+0000","last_became_active":"2024-09-13T14:59:53.220135+0000","last_became_peered":"2024-09-13T14:59:53.220135+0000","last_unstale":"2024-09-13T15:08:12.418081+0000","last_undegraded":"2024-09-13T15:08:12.418081+0000","last_fullsized":"2024-09-13T15:08:12.418081+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:52.361359+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:52.361359+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:35.003140+0000","scrub_duration":0.005845154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":715,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433404+0000","last_change":"2024-09-13T14:35:49.989482+0000","last_active":"2024-09-13T15:08:15.433404+0000","last_peered":"2024-09-13T15:08:15.433404+0000","last_clean":"2024-09-13T15:08:15.433404+0000","last_became_active":"2024-09-13T14:35:49.985023+0000","last_became_peered":"2024-09-13T14:35:49.985023+0000","last_unstale":"2024-09-13T15:08:15.433404+0000","last_undegraded":"2024-09-13T15:08:15.433404+0000","last_fullsized":"2024-09-13T15:08:15.433404+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:50.366066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"78'54","reported_seq":641,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578634+0000","last_change":"2024-09-13T14:35:49.985539+0000","last_active":"2024-09-13T15:08:11.578634+0000","last_peered":"2024-09-13T15:08:11.578634+0000","last_clean":"2024-09-13T15:08:11.578634+0000","last_became_active":"2024-09-13T14:35:49.985193+0000","last_became_peered":"2024-09-13T14:35:49.985193+0000","last_unstale":"2024-09-13T15:08:11.578634+0000","last_undegraded":"2024-09-13T15:08:11.578634+0000","last_fullsized":"2024-09-13T15:08:11.578634+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:06.101808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038031499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":382,"num_read_kb":321,"num_write":47,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":729,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.261009+0000","last_change":"2024-09-13T14:51:02.037184+0000","last_active":"2024-09-13T15:08:15.261009+0000","last_peered":"2024-09-13T15:08:15.261009+0000","last_clean":"2024-09-13T15:08:15.261009+0000","last_became_active":"2024-09-13T14:51:02.036644+0000","last_became_peered":"2024-09-13T14:51:02.036644+0000","last_unstale":"2024-09-13T15:08:15.261009+0000","last_undegraded":"2024-09-13T15:08:15.261009+0000","last_fullsized":"2024-09-13T15:08:15.261009+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:09.990760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":267,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600523+0000","last_change":"2024-09-13T15:08:03.291737+0000","last_active":"2024-09-13T15:08:11.600523+0000","last_peered":"2024-09-13T15:08:11.600523+0000","last_clean":"2024-09-13T15:08:11.600523+0000","last_became_active":"2024-09-13T15:08:03.291318+0000","last_became_peered":"2024-09-13T15:08:03.291318+0000","last_unstale":"2024-09-13T15:08:11.600523+0000","last_undegraded":"2024-09-13T15:08:11.600523+0000","last_fullsized":"2024-09-13T15:08:11.600523+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:06:32.296605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:06:32.296605+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:01:20.023210+0000","scrub_duration":0.0026525849999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":704,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260967+0000","last_change":"2024-09-13T14:37:09.289603+0000","last_active":"2024-09-13T15:08:15.260967+0000","last_peered":"2024-09-13T15:08:15.260967+0000","last_clean":"2024-09-13T15:08:15.260967+0000","last_became_active":"2024-09-13T14:37:09.289037+0000","last_became_peered":"2024-09-13T14:37:09.289037+0000","last_unstale":"2024-09-13T15:08:15.260967+0000","last_undegraded":"2024-09-13T15:08:15.260967+0000","last_fullsized":"2024-09-13T15:08:15.260967+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:55.710533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'165","reported_seq":1232,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.261062+0000","last_change":"2024-09-13T14:37:09.292533+0000","last_active":"2024-09-13T15:08:15.261062+0000","last_peered":"2024-09-13T15:08:15.261062+0000","last_clean":"2024-09-13T15:08:15.261062+0000","last_became_active":"2024-09-13T14:37:09.292079+0000","last_became_peered":"2024-09-13T14:37:09.292079+0000","last_unstale":"2024-09-13T15:08:15.261062+0000","last_undegraded":"2024-09-13T15:08:15.261062+0000","last_fullsized":"2024-09-13T15:08:15.261062+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":165,"log_dups_size":0,"ondisk_log_size":165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:11.123918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001763634,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":747,"num_read_kb":1679,"num_write":178,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":429,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579083+0000","last_change":"2024-09-13T15:05:52.014958+0000","last_active":"2024-09-13T15:08:11.579083+0000","last_peered":"2024-09-13T15:08:11.579083+0000","last_clean":"2024-09-13T15:08:11.579083+0000","last_became_active":"2024-09-13T15:05:52.014471+0000","last_became_peered":"2024-09-13T15:05:52.014471+0000","last_unstale":"2024-09-13T15:08:11.579083+0000","last_undegraded":"2024-09-13T15:08:11.579083+0000","last_fullsized":"2024-09-13T15:08:11.579083+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:38.514748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":277,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600500+0000","last_change":"2024-09-13T15:08:03.292358+0000","last_active":"2024-09-13T15:08:11.600500+0000","last_peered":"2024-09-13T15:08:11.600500+0000","last_clean":"2024-09-13T15:08:11.600500+0000","last_became_active":"2024-09-13T15:08:03.292006+0000","last_became_peered":"2024-09-13T15:08:03.292006+0000","last_unstale":"2024-09-13T15:08:11.600500+0000","last_undegraded":"2024-09-13T15:08:11.600500+0000","last_fullsized":"2024-09-13T15:08:11.600500+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:49.771572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:49.771572+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:21.124308+0000","scrub_duration":0.0035237329999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":761,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260959+0000","last_change":"2024-09-13T15:05:52.008066+0000","last_active":"2024-09-13T15:08:15.260959+0000","last_peered":"2024-09-13T15:08:15.260959+0000","last_clean":"2024-09-13T15:08:15.260959+0000","last_became_active":"2024-09-13T15:05:52.007749+0000","last_became_peered":"2024-09-13T15:05:52.007749+0000","last_unstale":"2024-09-13T15:08:15.260959+0000","last_undegraded":"2024-09-13T15:08:15.260959+0000","last_fullsized":"2024-09-13T15:08:15.260959+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:11.201592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":413,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418367+0000","last_change":"2024-09-13T14:48:32.198801+0000","last_active":"2024-09-13T15:08:12.418367+0000","last_peered":"2024-09-13T15:08:12.418367+0000","last_clean":"2024-09-13T15:08:12.418367+0000","last_became_active":"2024-09-13T14:48:32.198524+0000","last_became_peered":"2024-09-13T14:48:32.198524+0000","last_unstale":"2024-09-13T15:08:12.418367+0000","last_undegraded":"2024-09-13T15:08:12.418367+0000","last_fullsized":"2024-09-13T15:08:12.418367+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:25:28.108961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"78'79","reported_seq":1054,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579205+0000","last_change":"2024-09-13T14:48:32.237012+0000","last_active":"2024-09-13T15:08:11.579205+0000","last_peered":"2024-09-13T15:08:11.579205+0000","last_clean":"2024-09-13T15:08:11.579205+0000","last_became_active":"2024-09-13T14:48:32.206592+0000","last_became_peered":"2024-09-13T14:48:32.206592+0000","last_unstale":"2024-09-13T15:08:11.579205+0000","last_undegraded":"2024-09-13T15:08:11.579205+0000","last_fullsized":"2024-09-13T15:08:11.579205+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:19:27.261734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050177800000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":807,"num_read_kb":898,"num_write":52,"num_write_kb":4196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":348,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578416+0000","last_change":"2024-09-13T14:56:01.658240+0000","last_active":"2024-09-13T15:08:11.578416+0000","last_peered":"2024-09-13T15:08:11.578416+0000","last_clean":"2024-09-13T15:08:11.578416+0000","last_became_active":"2024-09-13T14:55:51.160480+0000","last_became_peered":"2024-09-13T14:55:51.160480+0000","last_unstale":"2024-09-13T15:08:11.578416+0000","last_undegraded":"2024-09-13T15:08:11.578416+0000","last_fullsized":"2024-09-13T15:08:11.578416+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.658143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.658143+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:19.262955+0000","scrub_duration":0.0038903079999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":747,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.261254+0000","last_change":"2024-09-13T14:37:09.290194+0000","last_active":"2024-09-13T15:08:15.261254+0000","last_peered":"2024-09-13T15:08:15.261254+0000","last_clean":"2024-09-13T15:08:15.261254+0000","last_became_active":"2024-09-13T14:37:09.289747+0000","last_became_peered":"2024-09-13T14:37:09.289747+0000","last_unstale":"2024-09-13T15:08:15.261254+0000","last_undegraded":"2024-09-13T15:08:15.261254+0000","last_fullsized":"2024-09-13T15:08:15.261254+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:13.221687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":729,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260779+0000","last_change":"2024-09-13T14:48:32.204943+0000","last_active":"2024-09-13T15:08:15.260779+0000","last_peered":"2024-09-13T15:08:15.260779+0000","last_clean":"2024-09-13T15:08:15.260779+0000","last_became_active":"2024-09-13T14:48:32.204076+0000","last_became_peered":"2024-09-13T14:48:32.204076+0000","last_unstale":"2024-09-13T15:08:15.260779+0000","last_undegraded":"2024-09-13T15:08:15.260779+0000","last_fullsized":"2024-09-13T15:08:15.260779+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:46.076963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"78'100","reported_seq":959,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.592060+0000","last_change":"2024-09-13T15:07:37.444321+0000","last_active":"2024-09-13T15:08:14.592060+0000","last_peered":"2024-09-13T15:08:14.592060+0000","last_clean":"2024-09-13T15:08:14.592060+0000","last_became_active":"2024-09-13T15:07:37.443790+0000","last_became_peered":"2024-09-13T15:07:37.443790+0000","last_unstale":"2024-09-13T15:08:14.592060+0000","last_undegraded":"2024-09-13T15:08:14.592060+0000","last_fullsized":"2024-09-13T15:08:14.592060+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:27:35.063400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032475400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":1717,"num_write":85,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":205,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578164+0000","last_change":"2024-09-13T14:55:59.574827+0000","last_active":"2024-09-13T15:08:11.578164+0000","last_peered":"2024-09-13T15:08:11.578164+0000","last_clean":"2024-09-13T15:08:11.578164+0000","last_became_active":"2024-09-13T14:55:51.162903+0000","last_became_peered":"2024-09-13T14:55:51.162903+0000","last_unstale":"2024-09-13T15:08:11.578164+0000","last_undegraded":"2024-09-13T15:08:11.578164+0000","last_fullsized":"2024-09-13T15:08:11.578164+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.574773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.574773+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:39.074072+0000","scrub_duration":0.001553708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":751,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433626+0000","last_change":"2024-09-13T14:51:02.015853+0000","last_active":"2024-09-13T15:08:15.433626+0000","last_peered":"2024-09-13T15:08:15.433626+0000","last_clean":"2024-09-13T15:08:15.433626+0000","last_became_active":"2024-09-13T14:51:02.015457+0000","last_became_peered":"2024-09-13T14:51:02.015457+0000","last_unstale":"2024-09-13T15:08:15.433626+0000","last_undegraded":"2024-09-13T15:08:15.433626+0000","last_fullsized":"2024-09-13T15:08:15.433626+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:29.365520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":761,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433106+0000","last_change":"2024-09-13T15:05:52.010114+0000","last_active":"2024-09-13T15:08:15.433106+0000","last_peered":"2024-09-13T15:08:15.433106+0000","last_clean":"2024-09-13T15:08:15.433106+0000","last_became_active":"2024-09-13T15:05:52.008634+0000","last_became_peered":"2024-09-13T15:05:52.008634+0000","last_unstale":"2024-09-13T15:08:15.433106+0000","last_undegraded":"2024-09-13T15:08:15.433106+0000","last_fullsized":"2024-09-13T15:08:15.433106+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:25.498296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"78'50","reported_seq":639,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.577939+0000","last_change":"2024-09-13T14:37:09.290510+0000","last_active":"2024-09-13T15:08:11.577939+0000","last_peered":"2024-09-13T15:08:11.577939+0000","last_clean":"2024-09-13T15:08:11.577939+0000","last_became_active":"2024-09-13T14:37:09.290095+0000","last_became_peered":"2024-09-13T14:37:09.290095+0000","last_unstale":"2024-09-13T15:08:11.577939+0000","last_undegraded":"2024-09-13T15:08:11.577939+0000","last_fullsized":"2024-09-13T15:08:11.577939+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:58.396788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035882600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":325,"num_read_kb":1313,"num_write":35,"num_write_kb":2065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":882,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260623+0000","last_change":"2024-09-13T14:55:59.245841+0000","last_active":"2024-09-13T15:08:15.260623+0000","last_peered":"2024-09-13T15:08:15.260623+0000","last_clean":"2024-09-13T15:08:15.260623+0000","last_became_active":"2024-09-13T14:55:51.159487+0000","last_became_peered":"2024-09-13T14:55:51.159487+0000","last_unstale":"2024-09-13T15:08:15.260623+0000","last_undegraded":"2024-09-13T15:08:15.260623+0000","last_fullsized":"2024-09-13T15:08:15.260623+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.245213+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.245213+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:16.152016+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":212,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600434+0000","last_change":"2024-09-13T14:48:32.205055+0000","last_active":"2024-09-13T15:08:11.600434+0000","last_peered":"2024-09-13T15:08:11.600434+0000","last_clean":"2024-09-13T15:08:11.600434+0000","last_became_active":"2024-09-13T14:48:32.204665+0000","last_became_peered":"2024-09-13T14:48:32.204665+0000","last_unstale":"2024-09-13T15:08:11.600434+0000","last_undegraded":"2024-09-13T15:08:11.600434+0000","last_fullsized":"2024-09-13T15:08:11.600434+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:38.024175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982110+0000","last_change":"2024-09-13T15:05:52.011318+0000","last_active":"2024-09-13T15:08:14.982110+0000","last_peered":"2024-09-13T15:08:14.982110+0000","last_clean":"2024-09-13T15:08:14.982110+0000","last_became_active":"2024-09-13T15:05:52.011038+0000","last_became_peered":"2024-09-13T15:05:52.011038+0000","last_unstale":"2024-09-13T15:08:14.982110+0000","last_undegraded":"2024-09-13T15:08:14.982110+0000","last_fullsized":"2024-09-13T15:08:14.982110+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:35.033102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'208","reported_seq":954,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578308+0000","last_change":"2024-09-13T14:37:40.924870+0000","last_active":"2024-09-13T15:08:11.578308+0000","last_peered":"2024-09-13T15:08:11.578308+0000","last_clean":"2024-09-13T15:08:11.578308+0000","last_became_active":"2024-09-13T14:37:40.924326+0000","last_became_peered":"2024-09-13T14:37:40.924326+0000","last_unstale":"2024-09-13T15:08:11.578308+0000","last_undegraded":"2024-09-13T15:08:11.578308+0000","last_fullsized":"2024-09-13T15:08:11.578308+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:26:55.098694+0000","last_clean_scrub_stamp":"2024-09-13T14:26:55.098694+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.414961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000498829,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":866,"num_read_kb":1066,"num_write":197,"num_write_kb":286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":576,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578023+0000","last_change":"2024-09-13T14:55:58.568852+0000","last_active":"2024-09-13T15:08:11.578023+0000","last_peered":"2024-09-13T15:08:11.578023+0000","last_clean":"2024-09-13T15:08:11.578023+0000","last_became_active":"2024-09-13T14:55:51.169987+0000","last_became_peered":"2024-09-13T14:55:51.169987+0000","last_unstale":"2024-09-13T15:08:11.578023+0000","last_undegraded":"2024-09-13T15:08:11.578023+0000","last_fullsized":"2024-09-13T15:08:11.578023+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.568750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.568750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:56.273429+0000","scrub_duration":0.0074886340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":728,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.261294+0000","last_change":"2024-09-13T15:05:52.007594+0000","last_active":"2024-09-13T15:08:15.261294+0000","last_peered":"2024-09-13T15:08:15.261294+0000","last_clean":"2024-09-13T15:08:15.261294+0000","last_became_active":"2024-09-13T15:05:52.007146+0000","last_became_peered":"2024-09-13T15:05:52.007146+0000","last_unstale":"2024-09-13T15:08:15.261294+0000","last_undegraded":"2024-09-13T15:08:15.261294+0000","last_fullsized":"2024-09-13T15:08:15.261294+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:45.283696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.981900+0000","last_change":"2024-09-13T15:05:51.991616+0000","last_active":"2024-09-13T15:08:14.981900+0000","last_peered":"2024-09-13T15:08:14.981900+0000","last_clean":"2024-09-13T15:08:14.981900+0000","last_became_active":"2024-09-13T15:05:51.991294+0000","last_became_peered":"2024-09-13T15:05:51.991294+0000","last_unstale":"2024-09-13T15:08:14.981900+0000","last_undegraded":"2024-09-13T15:08:14.981900+0000","last_fullsized":"2024-09-13T15:08:14.981900+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:05.060508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":358,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578640+0000","last_change":"2024-09-13T15:02:58.197427+0000","last_active":"2024-09-13T15:08:11.578640+0000","last_peered":"2024-09-13T15:08:11.578640+0000","last_clean":"2024-09-13T15:08:11.578640+0000","last_became_active":"2024-09-13T14:59:53.232987+0000","last_became_peered":"2024-09-13T14:59:53.232987+0000","last_unstale":"2024-09-13T15:08:11.578640+0000","last_undegraded":"2024-09-13T15:08:11.578640+0000","last_fullsized":"2024-09-13T15:08:11.578640+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:58.195731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:58.195731+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:51.565489+0000","scrub_duration":0.011077536000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":412,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418326+0000","last_change":"2024-09-13T14:48:32.204380+0000","last_active":"2024-09-13T15:08:12.418326+0000","last_peered":"2024-09-13T15:08:12.418326+0000","last_clean":"2024-09-13T15:08:12.418326+0000","last_became_active":"2024-09-13T14:48:32.203087+0000","last_became_peered":"2024-09-13T14:48:32.203087+0000","last_unstale":"2024-09-13T15:08:12.418326+0000","last_undegraded":"2024-09-13T15:08:12.418326+0000","last_fullsized":"2024-09-13T15:08:12.418326+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:21:02.119974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":712,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433214+0000","last_change":"2024-09-13T14:37:40.927288+0000","last_active":"2024-09-13T15:08:15.433214+0000","last_peered":"2024-09-13T15:08:15.433214+0000","last_clean":"2024-09-13T15:08:15.433214+0000","last_became_active":"2024-09-13T14:37:40.927005+0000","last_became_peered":"2024-09-13T14:37:40.927005+0000","last_unstale":"2024-09-13T15:08:15.433214+0000","last_undegraded":"2024-09-13T15:08:15.433214+0000","last_fullsized":"2024-09-13T15:08:15.433214+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:44.045843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":910,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260977+0000","last_change":"2024-09-13T15:00:10.659910+0000","last_active":"2024-09-13T15:08:15.260977+0000","last_peered":"2024-09-13T15:08:15.260977+0000","last_clean":"2024-09-13T15:08:15.260977+0000","last_became_active":"2024-09-13T14:59:53.216739+0000","last_became_peered":"2024-09-13T14:59:53.216739+0000","last_unstale":"2024-09-13T15:08:15.260977+0000","last_undegraded":"2024-09-13T15:08:15.260977+0000","last_fullsized":"2024-09-13T15:08:15.260977+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.659838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.659838+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:16.890653+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":409,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418396+0000","last_change":"2024-09-13T14:37:40.927490+0000","last_active":"2024-09-13T15:08:12.418396+0000","last_peered":"2024-09-13T15:08:12.418396+0000","last_clean":"2024-09-13T15:08:12.418396+0000","last_became_active":"2024-09-13T14:37:40.927226+0000","last_became_peered":"2024-09-13T14:37:40.927226+0000","last_unstale":"2024-09-13T15:08:12.418396+0000","last_undegraded":"2024-09-13T15:08:12.418396+0000","last_fullsized":"2024-09-13T15:08:12.418396+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:46.360432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":200,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600227+0000","last_change":"2024-09-13T15:08:03.399981+0000","last_active":"2024-09-13T15:08:11.600227+0000","last_peered":"2024-09-13T15:08:11.600227+0000","last_clean":"2024-09-13T15:08:11.600227+0000","last_became_active":"2024-09-13T15:08:03.399588+0000","last_became_peered":"2024-09-13T15:08:03.399588+0000","last_unstale":"2024-09-13T15:08:11.600227+0000","last_undegraded":"2024-09-13T15:08:11.600227+0000","last_fullsized":"2024-09-13T15:08:11.600227+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:49:52.771236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":379,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579244+0000","last_change":"2024-09-13T15:05:52.007408+0000","last_active":"2024-09-13T15:08:11.579244+0000","last_peered":"2024-09-13T15:08:11.579244+0000","last_clean":"2024-09-13T15:08:11.579244+0000","last_became_active":"2024-09-13T15:05:52.007032+0000","last_became_peered":"2024-09-13T15:05:52.007032+0000","last_unstale":"2024-09-13T15:08:11.579244+0000","last_undegraded":"2024-09-13T15:08:11.579244+0000","last_fullsized":"2024-09-13T15:08:11.579244+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:57.138789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:57.138789+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:28:20.106072+0000","scrub_duration":0.0036861670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":432,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418077+0000","last_change":"2024-09-13T15:05:51.998495+0000","last_active":"2024-09-13T15:08:12.418077+0000","last_peered":"2024-09-13T15:08:12.418077+0000","last_clean":"2024-09-13T15:08:12.418077+0000","last_became_active":"2024-09-13T15:05:51.998159+0000","last_became_peered":"2024-09-13T15:05:51.998159+0000","last_unstale":"2024-09-13T15:08:12.418077+0000","last_undegraded":"2024-09-13T15:08:12.418077+0000","last_fullsized":"2024-09-13T15:08:12.418077+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:23:05.948148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":105,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982168+0000","last_change":"2024-09-13T15:05:52.020378+0000","last_active":"2024-09-13T15:08:14.982168+0000","last_peered":"2024-09-13T15:08:14.982168+0000","last_clean":"2024-09-13T15:08:14.982168+0000","last_became_active":"2024-09-13T15:05:52.002686+0000","last_became_peered":"2024-09-13T15:05:52.002686+0000","last_unstale":"2024-09-13T15:08:14.982168+0000","last_undegraded":"2024-09-13T15:08:14.982168+0000","last_fullsized":"2024-09-13T15:08:14.982168+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.430732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":615,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.577870+0000","last_change":"2024-09-13T15:05:52.006880+0000","last_active":"2024-09-13T15:08:11.577870+0000","last_peered":"2024-09-13T15:08:11.577870+0000","last_clean":"2024-09-13T15:08:11.577870+0000","last_became_active":"2024-09-13T15:05:52.005704+0000","last_became_peered":"2024-09-13T15:05:52.005704+0000","last_unstale":"2024-09-13T15:08:11.577870+0000","last_undegraded":"2024-09-13T15:08:11.577870+0000","last_fullsized":"2024-09-13T15:08:11.577870+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":116,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:50.810045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:50.810045+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:39:30.729375+0000","scrub_duration":0.0042318529999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":725,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433543+0000","last_change":"2024-09-13T14:37:09.291674+0000","last_active":"2024-09-13T15:08:15.433543+0000","last_peered":"2024-09-13T15:08:15.433543+0000","last_clean":"2024-09-13T15:08:15.433543+0000","last_became_active":"2024-09-13T14:37:09.291339+0000","last_became_peered":"2024-09-13T14:37:09.291339+0000","last_unstale":"2024-09-13T15:08:15.433543+0000","last_undegraded":"2024-09-13T15:08:15.433543+0000","last_fullsized":"2024-09-13T15:08:15.433543+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:06.475668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":432,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579130+0000","last_change":"2024-09-13T15:05:52.015239+0000","last_active":"2024-09-13T15:08:11.579130+0000","last_peered":"2024-09-13T15:08:11.579130+0000","last_clean":"2024-09-13T15:08:11.579130+0000","last_became_active":"2024-09-13T15:05:52.014825+0000","last_became_peered":"2024-09-13T15:05:52.014825+0000","last_unstale":"2024-09-13T15:08:11.579130+0000","last_undegraded":"2024-09-13T15:08:11.579130+0000","last_fullsized":"2024-09-13T15:08:11.579130+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:47.548807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":198,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.418187+0000","last_change":"2024-09-13T15:05:51.996175+0000","last_active":"2024-09-13T15:08:12.418187+0000","last_peered":"2024-09-13T15:08:12.418187+0000","last_clean":"2024-09-13T15:08:12.418187+0000","last_became_active":"2024-09-13T15:05:51.995536+0000","last_became_peered":"2024-09-13T15:05:51.995536+0000","last_unstale":"2024-09-13T15:08:12.418187+0000","last_undegraded":"2024-09-13T15:08:12.418187+0000","last_fullsized":"2024-09-13T15:08:12.418187+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:55.358699+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:02:55.358699+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:58.331262+0000","scrub_duration":0.0023994049999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":232,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600153+0000","last_change":"2024-09-13T15:02:38.875727+0000","last_active":"2024-09-13T15:08:11.600153+0000","last_peered":"2024-09-13T15:08:11.600153+0000","last_clean":"2024-09-13T15:08:11.600153+0000","last_became_active":"2024-09-13T15:02:38.875372+0000","last_became_peered":"2024-09-13T15:02:38.875372+0000","last_unstale":"2024-09-13T15:08:11.600153+0000","last_undegraded":"2024-09-13T15:08:11.600153+0000","last_fullsized":"2024-09-13T15:08:11.600153+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:08.504810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":404,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578499+0000","last_change":"2024-09-13T14:48:32.212622+0000","last_active":"2024-09-13T15:08:11.578499+0000","last_peered":"2024-09-13T15:08:11.578499+0000","last_clean":"2024-09-13T15:08:11.578499+0000","last_became_active":"2024-09-13T14:48:32.203835+0000","last_became_peered":"2024-09-13T14:48:32.203835+0000","last_unstale":"2024-09-13T15:08:11.578499+0000","last_undegraded":"2024-09-13T15:08:11.578499+0000","last_fullsized":"2024-09-13T15:08:11.578499+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:34.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":261,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600524+0000","last_change":"2024-09-13T14:56:08.889974+0000","last_active":"2024-09-13T15:08:11.600524+0000","last_peered":"2024-09-13T15:08:11.600524+0000","last_clean":"2024-09-13T15:08:11.600524+0000","last_became_active":"2024-09-13T14:55:55.185755+0000","last_became_peered":"2024-09-13T14:55:55.185755+0000","last_unstale":"2024-09-13T15:08:11.600524+0000","last_undegraded":"2024-09-13T15:08:11.600524+0000","last_fullsized":"2024-09-13T15:08:11.600524+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:08.889814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:08.889814+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:02.911564+0000","scrub_duration":0.0053731569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":469,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:12.417839+0000","last_change":"2024-09-13T15:05:52.000473+0000","last_active":"2024-09-13T15:08:12.417839+0000","last_peered":"2024-09-13T15:08:12.417839+0000","last_clean":"2024-09-13T15:08:12.417839+0000","last_became_active":"2024-09-13T15:05:52.000152+0000","last_became_peered":"2024-09-13T15:05:52.000152+0000","last_unstale":"2024-09-13T15:08:12.417839+0000","last_undegraded":"2024-09-13T15:08:12.417839+0000","last_fullsized":"2024-09-13T15:08:12.417839+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:31.530855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":394,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578698+0000","last_change":"2024-09-13T14:37:09.291756+0000","last_active":"2024-09-13T15:08:11.578698+0000","last_peered":"2024-09-13T15:08:11.578698+0000","last_clean":"2024-09-13T15:08:11.578698+0000","last_became_active":"2024-09-13T14:37:09.291420+0000","last_became_peered":"2024-09-13T14:37:09.291420+0000","last_unstale":"2024-09-13T15:08:11.578698+0000","last_undegraded":"2024-09-13T15:08:11.578698+0000","last_fullsized":"2024-09-13T15:08:11.578698+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:45.516522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":106,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982342+0000","last_change":"2024-09-13T15:05:52.036891+0000","last_active":"2024-09-13T15:08:14.982342+0000","last_peered":"2024-09-13T15:08:14.982342+0000","last_clean":"2024-09-13T15:08:14.982342+0000","last_became_active":"2024-09-13T15:05:52.003251+0000","last_became_peered":"2024-09-13T15:05:52.003251+0000","last_unstale":"2024-09-13T15:08:14.982342+0000","last_undegraded":"2024-09-13T15:08:14.982342+0000","last_fullsized":"2024-09-13T15:08:14.982342+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:00:10.016260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T15:00:10.016260+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:42.653316+0000","scrub_duration":71,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":414,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579391+0000","last_change":"2024-09-13T14:51:44.187799+0000","last_active":"2024-09-13T15:08:11.579391+0000","last_peered":"2024-09-13T15:08:11.579391+0000","last_clean":"2024-09-13T15:08:11.579391+0000","last_became_active":"2024-09-13T14:51:44.187183+0000","last_became_peered":"2024-09-13T14:51:44.187183+0000","last_unstale":"2024-09-13T15:08:11.579391+0000","last_undegraded":"2024-09-13T15:08:11.579391+0000","last_fullsized":"2024-09-13T15:08:11.579391+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:07.940293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":716,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.592072+0000","last_change":"2024-09-13T14:37:40.927703+0000","last_active":"2024-09-13T15:08:14.592072+0000","last_peered":"2024-09-13T15:08:14.592072+0000","last_clean":"2024-09-13T15:08:14.592072+0000","last_became_active":"2024-09-13T14:37:40.927211+0000","last_became_peered":"2024-09-13T14:37:40.927211+0000","last_unstale":"2024-09-13T15:08:14.592072+0000","last_undegraded":"2024-09-13T15:08:14.592072+0000","last_fullsized":"2024-09-13T15:08:14.592072+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:07.941888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":178,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578278+0000","last_change":"2024-09-13T15:02:53.820519+0000","last_active":"2024-09-13T15:08:11.578278+0000","last_peered":"2024-09-13T15:08:11.578278+0000","last_clean":"2024-09-13T15:08:11.578278+0000","last_became_active":"2024-09-13T14:59:53.222306+0000","last_became_peered":"2024-09-13T14:59:53.222306+0000","last_unstale":"2024-09-13T15:08:11.578278+0000","last_undegraded":"2024-09-13T15:08:11.578278+0000","last_fullsized":"2024-09-13T15:08:11.578278+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T15:02:53.820435+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:40:30.500046+0000","last_clean_scrub_stamp":"2024-09-13T15:02:53.820435+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:40.108691+0000","scrub_duration":0.0030873739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":222,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.600183+0000","last_change":"2024-09-13T14:48:32.219381+0000","last_active":"2024-09-13T15:08:11.600183+0000","last_peered":"2024-09-13T15:08:11.600183+0000","last_clean":"2024-09-13T15:08:11.600183+0000","last_became_active":"2024-09-13T14:48:32.204701+0000","last_became_peered":"2024-09-13T14:48:32.204701+0000","last_unstale":"2024-09-13T15:08:11.600183+0000","last_undegraded":"2024-09-13T15:08:11.600183+0000","last_fullsized":"2024-09-13T15:08:11.600183+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:52.332692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982570+0000","last_change":"2024-09-13T15:05:51.999191+0000","last_active":"2024-09-13T15:08:14.982570+0000","last_peered":"2024-09-13T15:08:14.982570+0000","last_clean":"2024-09-13T15:08:14.982570+0000","last_became_active":"2024-09-13T15:05:51.998885+0000","last_became_peered":"2024-09-13T15:05:51.998885+0000","last_unstale":"2024-09-13T15:08:14.982570+0000","last_undegraded":"2024-09-13T15:08:14.982570+0000","last_fullsized":"2024-09-13T15:08:14.982570+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:52.548839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":158,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578818+0000","last_change":"2024-09-13T14:56:03.605447+0000","last_active":"2024-09-13T15:08:11.578818+0000","last_peered":"2024-09-13T15:08:11.578818+0000","last_clean":"2024-09-13T15:08:11.578818+0000","last_became_active":"2024-09-13T14:55:55.218939+0000","last_became_peered":"2024-09-13T14:55:55.218939+0000","last_unstale":"2024-09-13T15:08:11.578818+0000","last_undegraded":"2024-09-13T15:08:11.578818+0000","last_fullsized":"2024-09-13T15:08:11.578818+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.605356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.605356+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:12.467371+0000","scrub_duration":0.0020269620000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":395,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578112+0000","last_change":"2024-09-13T14:48:32.204086+0000","last_active":"2024-09-13T15:08:11.578112+0000","last_peered":"2024-09-13T15:08:11.578112+0000","last_clean":"2024-09-13T15:08:11.578112+0000","last_became_active":"2024-09-13T14:48:32.203755+0000","last_became_peered":"2024-09-13T14:48:32.203755+0000","last_unstale":"2024-09-13T15:08:11.578112+0000","last_undegraded":"2024-09-13T15:08:11.578112+0000","last_fullsized":"2024-09-13T15:08:11.578112+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:09.817189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":752,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433470+0000","last_change":"2024-09-13T15:05:52.008360+0000","last_active":"2024-09-13T15:08:15.433470+0000","last_peered":"2024-09-13T15:08:15.433470+0000","last_clean":"2024-09-13T15:08:15.433470+0000","last_became_active":"2024-09-13T15:05:52.007789+0000","last_became_peered":"2024-09-13T15:05:52.007789+0000","last_unstale":"2024-09-13T15:08:15.433470+0000","last_undegraded":"2024-09-13T15:08:15.433470+0000","last_fullsized":"2024-09-13T15:08:15.433470+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:12:26.443562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":901,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.260857+0000","last_change":"2024-09-13T15:05:52.009899+0000","last_active":"2024-09-13T15:08:15.260857+0000","last_peered":"2024-09-13T15:08:15.260857+0000","last_clean":"2024-09-13T15:08:15.260857+0000","last_became_active":"2024-09-13T15:05:52.009365+0000","last_became_peered":"2024-09-13T15:05:52.009365+0000","last_unstale":"2024-09-13T15:08:15.260857+0000","last_undegraded":"2024-09-13T15:08:15.260857+0000","last_fullsized":"2024-09-13T15:08:15.260857+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:03.240882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:03.240882+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:05:45.534365+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":759,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433208+0000","last_change":"2024-09-13T15:05:52.011548+0000","last_active":"2024-09-13T15:08:15.433208+0000","last_peered":"2024-09-13T15:08:15.433208+0000","last_clean":"2024-09-13T15:08:15.433208+0000","last_became_active":"2024-09-13T15:05:52.009924+0000","last_became_peered":"2024-09-13T15:05:52.009924+0000","last_unstale":"2024-09-13T15:08:15.433208+0000","last_undegraded":"2024-09-13T15:08:15.433208+0000","last_fullsized":"2024-09-13T15:08:15.433208+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:49.485860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":411,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578954+0000","last_change":"2024-09-13T15:05:52.014781+0000","last_active":"2024-09-13T15:08:11.578954+0000","last_peered":"2024-09-13T15:08:11.578954+0000","last_clean":"2024-09-13T15:08:11.578954+0000","last_became_active":"2024-09-13T15:05:52.014049+0000","last_became_peered":"2024-09-13T15:05:52.014049+0000","last_unstale":"2024-09-13T15:08:11.578954+0000","last_undegraded":"2024-09-13T15:08:11.578954+0000","last_fullsized":"2024-09-13T15:08:11.578954+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:33.528483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":158,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.579157+0000","last_change":"2024-09-13T14:56:04.643672+0000","last_active":"2024-09-13T15:08:11.579157+0000","last_peered":"2024-09-13T15:08:11.579157+0000","last_clean":"2024-09-13T15:08:11.579157+0000","last_became_active":"2024-09-13T14:55:55.185995+0000","last_became_peered":"2024-09-13T14:55:55.185995+0000","last_unstale":"2024-09-13T15:08:11.579157+0000","last_undegraded":"2024-09-13T15:08:11.579157+0000","last_fullsized":"2024-09-13T15:08:11.579157+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.643614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.643614+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:36.495546+0000","scrub_duration":0.001441681,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":394,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-13T15:08:11.578269+0000","last_change":"2024-09-13T14:37:40.925007+0000","last_active":"2024-09-13T15:08:11.578269+0000","last_peered":"2024-09-13T15:08:11.578269+0000","last_clean":"2024-09-13T15:08:11.578269+0000","last_became_active":"2024-09-13T14:37:40.924534+0000","last_became_peered":"2024-09-13T14:37:40.924534+0000","last_unstale":"2024-09-13T15:08:11.578269+0000","last_undegraded":"2024-09-13T15:08:11.578269+0000","last_fullsized":"2024-09-13T15:08:11.578269+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:09.478192+0000","last_clean_scrub_stamp":"2024-09-13T14:28:09.478192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:17.929630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":101,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:14.982372+0000","last_change":"2024-09-13T15:05:52.015281+0000","last_active":"2024-09-13T15:08:14.982372+0000","last_peered":"2024-09-13T15:08:14.982372+0000","last_clean":"2024-09-13T15:08:14.982372+0000","last_became_active":"2024-09-13T15:05:52.014517+0000","last_became_peered":"2024-09-13T15:05:52.014517+0000","last_unstale":"2024-09-13T15:08:14.982372+0000","last_undegraded":"2024-09-13T15:08:14.982372+0000","last_fullsized":"2024-09-13T15:08:14.982372+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:28:10.481124+0000","last_clean_scrub_stamp":"2024-09-13T14:28:10.481124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:10:37.226490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"82'100","reported_seq":469,"reported_epoch":428,"state":"active+clean","last_fresh":"2024-09-13T15:08:15.433226+0000","last_change":"2024-09-13T15:05:52.008756+0000","last_active":"2024-09-13T15:08:15.433226+0000","last_peered":"2024-09-13T15:08:15.433226+0000","last_clean":"2024-09-13T15:08:15.433226+0000","last_became_active":"2024-09-13T15:05:52.007609+0000","last_became_peered":"2024-09-13T15:05:52.007609+0000","last_unstale":"2024-09-13T15:08:15.433226+0000","last_undegraded":"2024-09-13T15:08:15.433226+0000","last_fullsized":"2024-09-13T15:08:15.433226+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":287,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.087994+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:20.875473+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.087994+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:33:22.602291+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":16262,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24498,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2797,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398022,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":887580,"kb_used_data":6480,"kb_used_omap":1,"kb_used_meta":881086,"kb_avail":92857572,"statfs":{"total":95995035648,"available":95086153728,"internally_reserved":0,"allocated":6635520,"data_stored":3696576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":902232408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594251,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":324624,"kb_used_data":10628,"kb_used_omap":1,"kb_used_meta":313982,"kb_avail":93420528,"statfs":{"total":95995035648,"available":95662620672,"internally_reserved":0,"allocated":10883072,"data_stored":7955444,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":321517912},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790480,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1195980,"kb_used_data":10560,"kb_used_omap":1,"kb_used_meta":1185406,"kb_avail":92549172,"statfs":{"total":95995035648,"available":94770352128,"internally_reserved":0,"allocated":10813440,"data_stored":7886349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1213856088},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986709,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907596,"kb_used_data":6208,"kb_used_omap":1,"kb_used_meta":901374,"kb_avail":92837556,"statfs":{"total":95995035648,"available":95065657344,"internally_reserved":0,"allocated":6356992,"data_stored":3429088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923007320},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":335,"seq":1438814044256,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":123904,"kb_used_data":6516,"kb_used_omap":10,"kb_used_meta":117365,"kb_avail":93621248,"statfs":{"total":95995035648,"available":95868157952,"internally_reserved":0,"allocated":6672384,"data_stored":3756496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10273,"internal_metadata":120182751},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":99,"seq":425201762683,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1051600,"kb_used_data":6464,"kb_used_omap":8,"kb_used_meta":1045111,"kb_avail":92693552,"statfs":{"total":95995035648,"available":94918197248,"internally_reserved":0,"allocated":6619136,"data_stored":3692040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8758,"internal_metadata":1070194122},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":94,"seq":403726926210,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1015364,"kb_used_data":10296,"kb_used_omap":7,"kb_used_meta":1005048,"kb_avail":92729788,"statfs":{"total":95995035648,"available":94955302912,"internally_reserved":0,"allocated":10543104,"data_stored":7623293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7479,"internal_metadata":1029169865},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":88,"seq":377957122449,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1035860,"kb_used_data":6536,"kb_used_omap":7,"kb_used_meta":1029304,"kb_avail":92709292,"statfs":{"total":95995035648,"available":94934315008,"internally_reserved":0,"allocated":6692864,"data_stored":3761986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7601,"internal_metadata":1054007887},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1494,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5897,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5775,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7266,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:09:03.770 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T15:09:03.771 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi096 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-13T15:09:03.960 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=215ccecffe754054bf79ded524059acc Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi096 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-13T15:09:03.963 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-13T15:09:03.972 INFO:tasks.cephadm:Teardown begin 2024-09-13T15:09:03.972 DEBUG:teuthology.orchestra.run.smithi096:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:09:03.990 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:09:03 smithi204 ceph-mon[87071]: pgmap v1442: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:09:03.990 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:09:03 smithi204 ceph-mon[87071]: from='client.37092 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:09:04.013 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:09:03 smithi096 ceph-mon[98901]: pgmap v1442: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:09:04.013 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:09:03 smithi096 ceph-mon[98901]: from='client.37092 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:09:04.013 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:09:03 smithi096 ceph-mon[102928]: pgmap v1442: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:09:04.013 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:09:03 smithi096 ceph-mon[102928]: from='client.37092 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:09:04.034 DEBUG:teuthology.orchestra.run.smithi204:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:09:04.080 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-13T15:09:04.081 DEBUG:teuthology.orchestra.run.smithi096:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T15:09:04.097 DEBUG:teuthology.orchestra.run.smithi204:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T15:09:04.112 INFO:tasks.cephadm:Stopping all daemons... 2024-09-13T15:09:04.112 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-13T15:09:04.112 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.a 2024-09-13T15:09:04.298 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:09:04 smithi096 systemd[1]: Stopping Ceph mon.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:09:04.958 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:09:04 smithi096 ceph-mon[98901]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:09:04.958 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:09:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a[98876]: 2024-09-13T15:09:04.908+0000 7f16b45af640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:09:04.958 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:09:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a[98876]: 2024-09-13T15:09:04.908+0000 7f16b45af640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-13T15:09:04.958 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:15:09:04] ENGINE Bus STOPPING 2024-09-13T15:09:04.959 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:15:09:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T15:09:04.959 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:15:09:04] ENGINE Bus STOPPED 2024-09-13T15:09:04.959 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:09:04 smithi096 ceph-mon[102928]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:09:04.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:09:04 smithi204 ceph-mon[87071]: from='mgr.26588 172.21.15.96:0/1489493814' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:09:05.323 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:09:05 smithi096 podman[324865]: 2024-09-13 15:09:05.091013246 +0000 UTC m=+0.311552573 container died 97a94128e048ed7eb609e53a28bf4a29c94b24310260d4b5b69fa0fcdae33191 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2) 2024-09-13T15:09:05.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:04 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:15:09:04] ENGINE Bus STARTING 2024-09-13T15:09:05.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:05 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:15:09:05] ENGINE Serving on http://:::9283 2024-09-13T15:09:05.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:05 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:15:09:05] ENGINE Bus STARTED 2024-09-13T15:09:05.777 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:09:05 smithi096 podman[324865]: 2024-09-13 15:09:05.591772003 +0000 UTC m=+0.812311327 container cleanup 97a94128e048ed7eb609e53a28bf4a29c94b24310260d4b5b69fa0fcdae33191 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T15:09:05.777 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:09:05 smithi096 bash[324865]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a 2024-09-13T15:09:06.073 INFO:journalctl@ceph.mon.a.smithi096.stdout:Sep 13 15:09:05 smithi096 podman[324980]: 2024-09-13 15:09:05.776336936 +0000 UTC m=+0.679647094 container remove 97a94128e048ed7eb609e53a28bf4a29c94b24310260d4b5b69fa0fcdae33191 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-a, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-13T15:09:06.283 DEBUG:teuthology.orchestra.run.smithi096:> sudo pkill -f 'journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.a.service' 2024-09-13T15:09:06.323 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:09:06.323 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-13T15:09:06.323 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-13T15:09:06.323 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.c 2024-09-13T15:09:06.823 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:09:06 smithi096 systemd[1]: Stopping Ceph mon.c for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:09:08.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:07 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: ::ffff:172.21.15.204 - - [13/Sep/2024:15:09:07] "GET /metrics HTTP/1.1" 200 34162 "" "Prometheus/2.51.0" 2024-09-13T15:09:08.339 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:08 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:15:09:08] ENGINE Bus STOPPING 2024-09-13T15:09:08.340 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:09:08 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c[102924]: 2024-09-13T15:09:08.182+0000 7f095b5cc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:09:08.340 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:09:08 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c[102924]: 2024-09-13T15:09:08.182+0000 7f095b5cc640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-13T15:09:08.625 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:09:08 smithi096 podman[325270]: 2024-09-13 15:09:08.355979802 +0000 UTC m=+0.921065929 container died babdd691d9b6f9db01e49db29e8b8f6650954acad4c80ce77f47b922f1e27943 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-13T15:09:08.625 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:08 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:15:09:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T15:09:08.625 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:08 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:15:09:08] ENGINE Bus STOPPED 2024-09-13T15:09:08.625 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:08 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:15:09:08] ENGINE Bus STARTING 2024-09-13T15:09:08.626 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:08 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:15:09:08] ENGINE Serving on http://:::9283 2024-09-13T15:09:08.626 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:08 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: [13/Sep/2024:15:09:08] ENGINE Bus STARTED 2024-09-13T15:09:09.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:09:08 smithi096 podman[325270]: 2024-09-13 15:09:08.824011465 +0000 UTC m=+1.389097592 container cleanup babdd691d9b6f9db01e49db29e8b8f6650954acad4c80ce77f47b922f1e27943 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0) 2024-09-13T15:09:09.073 INFO:journalctl@ceph.mon.c.smithi096.stdout:Sep 13 15:09:08 smithi096 bash[325270]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-c 2024-09-13T15:09:09.311 DEBUG:teuthology.orchestra.run.smithi096:> sudo pkill -f 'journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.c.service' 2024-09-13T15:09:09.349 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:09:09.350 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-13T15:09:09.350 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-13T15:09:09.350 DEBUG:teuthology.orchestra.run.smithi204:> sudo systemctl stop ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.b 2024-09-13T15:09:09.671 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:09:09 smithi204 systemd[1]: Stopping Ceph mon.b for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:09:09.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:09:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b[87067]: 2024-09-13T15:09:09.668+0000 7f9a99a60640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:09:09.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:09:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b[87067]: 2024-09-13T15:09:09.668+0000 7f9a99a60640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-13T15:09:09.969 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:09:09 smithi204 podman[230564]: 2024-09-13 15:09:09.740350599 +0000 UTC m=+0.192281374 container stop 4cc02299264a7d91ea242d2c1a08e54ed47cae79108e57893ff26235ae235224 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731) 2024-09-13T15:09:09.970 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:09:09 smithi204 podman[230564]: 2024-09-13 15:09:09.766247236 +0000 UTC m=+0.218178049 container died 4cc02299264a7d91ea242d2c1a08e54ed47cae79108e57893ff26235ae235224 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731) 2024-09-13T15:09:10.430 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:09:10 smithi204 podman[230564]: 2024-09-13 15:09:10.18584049 +0000 UTC m=+0.637771268 container cleanup 4cc02299264a7d91ea242d2c1a08e54ed47cae79108e57893ff26235ae235224 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:09:10.430 INFO:journalctl@ceph.mon.b.smithi204.stdout:Sep 13 15:09:10 smithi204 bash[230564]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mon-b 2024-09-13T15:09:10.430 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:09:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:09:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"51e50ed6-71db-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:09:10.644 DEBUG:teuthology.orchestra.run.smithi204:> sudo pkill -f 'journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mon.b.service' 2024-09-13T15:09:10.694 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:09:10.694 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-13T15:09:10.694 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-13T15:09:10.695 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.y 2024-09-13T15:09:11.013 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:10 smithi096 systemd[1]: Stopping Ceph mgr.y for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:09:11.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:11 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T15:09:11.010+0000 7ff0a8e3b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:09:11.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:11 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y[89381]: 2024-09-13T15:09:11.010+0000 7ff0a8e3b640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-09-13T15:09:11.323 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:11 smithi096 podman[325421]: 2024-09-13 15:09:11.05295361 +0000 UTC m=+0.164942924 container died a4bc51171f6e7979db2dd1de560dbf97688832d73e8b630979eba72f017e0e47 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:09:11.817 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:11 smithi096 podman[325421]: 2024-09-13 15:09:11.581137148 +0000 UTC m=+0.693126461 container cleanup a4bc51171f6e7979db2dd1de560dbf97688832d73e8b630979eba72f017e0e47 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T15:09:11.817 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:11 smithi096 bash[325421]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y 2024-09-13T15:09:12.068 INFO:journalctl@ceph.mgr.y.smithi096.stdout:Sep 13 15:09:11 smithi096 podman[325436]: 2024-09-13 15:09:11.816189352 +0000 UTC m=+0.756119609 container remove a4bc51171f6e7979db2dd1de560dbf97688832d73e8b630979eba72f017e0e47 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-y, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-13T15:09:12.073 DEBUG:teuthology.orchestra.run.smithi096:> sudo pkill -f 'journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.y.service' 2024-09-13T15:09:12.108 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:09:12.108 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-13T15:09:12.108 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-13T15:09:12.109 DEBUG:teuthology.orchestra.run.smithi204:> sudo systemctl stop ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.x 2024-09-13T15:09:12.466 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 15:09:12 smithi204 systemd[1]: Stopping Ceph mgr.x for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:09:12.719 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 15:09:12 smithi204 podman[230685]: 2024-09-13 15:09:12.464212917 +0000 UTC m=+0.168208581 container died 04572bea1e6fa024cb0842bda1f6b4e197695cca853dc44d9db2769b0895e14a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-13T15:09:13.127 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 15:09:12 smithi204 podman[230685]: 2024-09-13 15:09:12.900058667 +0000 UTC m=+0.604054318 container cleanup 04572bea1e6fa024cb0842bda1f6b4e197695cca853dc44d9db2769b0895e14a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-13T15:09:13.127 INFO:journalctl@ceph.mgr.x.smithi204.stdout:Sep 13 15:09:12 smithi204 bash[230685]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-mgr-x 2024-09-13T15:09:13.351 DEBUG:teuthology.orchestra.run.smithi204:> sudo pkill -f 'journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@mgr.x.service' 2024-09-13T15:09:13.386 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:09:13.387 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-13T15:09:13.387 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-13T15:09:13.387 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.0 2024-09-13T15:09:13.695 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 15:09:13 smithi096 systemd[1]: Stopping Ceph osd.0 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:09:14.073 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 15:09:13 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0[110036]: 2024-09-13T15:09:13.692+0000 7f1d2899e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:09:14.073 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 15:09:13 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0[110036]: 2024-09-13T15:09:13.692+0000 7f1d2899e640 -1 osd.0 428 *** Got signal Terminated *** 2024-09-13T15:09:14.073 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 15:09:13 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0[110036]: 2024-09-13T15:09:13.692+0000 7f1d2899e640 -1 osd.0 428 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:09:19.637 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 15:09:19 smithi096 podman[325542]: 2024-09-13 15:09:19.372395923 +0000 UTC m=+5.794334775 container died ab741f6f9e89b46014f5bad7029e73a146f897aa8195dd59deb225b0b80e6e97 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True) 2024-09-13T15:09:19.961 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 15:09:19 smithi096 podman[325542]: 2024-09-13 15:09:19.843632966 +0000 UTC m=+6.265571825 container cleanup ab741f6f9e89b46014f5bad7029e73a146f897aa8195dd59deb225b0b80e6e97 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True) 2024-09-13T15:09:19.961 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 15:09:19 smithi096 bash[325542]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0 2024-09-13T15:09:20.322 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 15:09:20 smithi096 podman[325558]: 2024-09-13 15:09:20.162212642 +0000 UTC m=+0.785959357 container remove ab741f6f9e89b46014f5bad7029e73a146f897aa8195dd59deb225b0b80e6e97 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-13T15:09:20.823 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 15:09:20 smithi096 podman[325653]: 2024-09-13 15:09:20.452583376 +0000 UTC m=+0.017947404 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:09:20.824 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 15:09:20 smithi096 podman[325653]: 2024-09-13 15:09:20.563164028 +0000 UTC m=+0.128528031 container create c13cb21e03d2f72e4dfc3804e57711a7ef80310d0bf9d6edd3fdb4cb983a96bf (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-13T15:09:20.824 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 15:09:20 smithi096 podman[325653]: 2024-09-13 15:09:20.790507965 +0000 UTC m=+0.355871973 container init c13cb21e03d2f72e4dfc3804e57711a7ef80310d0bf9d6edd3fdb4cb983a96bf (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True) 2024-09-13T15:09:20.824 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 15:09:20 smithi096 podman[325653]: 2024-09-13 15:09:20.793974207 +0000 UTC m=+0.359338213 container start c13cb21e03d2f72e4dfc3804e57711a7ef80310d0bf9d6edd3fdb4cb983a96bf (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:09:21.323 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 15:09:20 smithi096 podman[325653]: 2024-09-13 15:09:20.957539881 +0000 UTC m=+0.522903963 container attach c13cb21e03d2f72e4dfc3804e57711a7ef80310d0bf9d6edd3fdb4cb983a96bf (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-13T15:09:21.878 INFO:journalctl@ceph.osd.0.smithi096.stdout:Sep 13 15:09:21 smithi096 podman[325653]: 2024-09-13 15:09:21.58039515 +0000 UTC m=+1.145759159 container died c13cb21e03d2f72e4dfc3804e57711a7ef80310d0bf9d6edd3fdb4cb983a96bf (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-13T15:09:22.277 DEBUG:teuthology.orchestra.run.smithi096:> sudo pkill -f 'journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.0.service' 2024-09-13T15:09:22.321 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:09:22.321 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-13T15:09:22.322 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-13T15:09:22.322 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.1 2024-09-13T15:09:22.573 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 15:09:22 smithi096 systemd[1]: Stopping Ceph osd.1 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:09:23.073 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 15:09:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1[115304]: 2024-09-13T15:09:22.664+0000 7f1aa7650640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:09:23.073 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 15:09:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1[115304]: 2024-09-13T15:09:22.664+0000 7f1aa7650640 -1 osd.1 428 *** Got signal Terminated *** 2024-09-13T15:09:23.073 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 15:09:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1[115304]: 2024-09-13T15:09:22.664+0000 7f1aa7650640 -1 osd.1 428 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:09:28.468 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 15:09:28 smithi096 podman[325760]: 2024-09-13 15:09:28.175726636 +0000 UTC m=+5.620957651 container died bbf0998790c29e4d23f884f8270db9338a3cb8b8571e5d34badf28ca2ca470f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T15:09:28.823 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 15:09:28 smithi096 podman[325760]: 2024-09-13 15:09:28.647681687 +0000 UTC m=+6.092912703 container cleanup bbf0998790c29e4d23f884f8270db9338a3cb8b8571e5d34badf28ca2ca470f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T15:09:28.865 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 15:09:28 smithi096 bash[325760]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1 2024-09-13T15:09:29.242 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 15:09:28 smithi096 podman[325775]: 2024-09-13 15:09:28.932421663 +0000 UTC m=+0.752224698 container remove bbf0998790c29e4d23f884f8270db9338a3cb8b8571e5d34badf28ca2ca470f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-13T15:09:29.553 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 15:09:29 smithi096 podman[325864]: 2024-09-13 15:09:29.144735698 +0000 UTC m=+0.016484122 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:09:29.553 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 15:09:29 smithi096 podman[325864]: 2024-09-13 15:09:29.258226883 +0000 UTC m=+0.129975348 container create ff0885389bb9cfe2969b2e2886627b514d7ee9336ff1c97b8ac047bc20bf9326 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T15:09:29.553 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 15:09:29 smithi096 podman[325864]: 2024-09-13 15:09:29.4775917 +0000 UTC m=+0.349340124 container init ff0885389bb9cfe2969b2e2886627b514d7ee9336ff1c97b8ac047bc20bf9326 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-13T15:09:29.554 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 15:09:29 smithi096 podman[325864]: 2024-09-13 15:09:29.480862923 +0000 UTC m=+0.352611355 container start ff0885389bb9cfe2969b2e2886627b514d7ee9336ff1c97b8ac047bc20bf9326 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=) 2024-09-13T15:09:29.554 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 15:09:29 smithi096 podman[325864]: 2024-09-13 15:09:29.552326358 +0000 UTC m=+0.424074791 container attach ff0885389bb9cfe2969b2e2886627b514d7ee9336ff1c97b8ac047bc20bf9326 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T15:09:29.823 INFO:journalctl@ceph.osd.1.smithi096.stdout:Sep 13 15:09:29 smithi096 podman[325864]: 2024-09-13 15:09:29.667182049 +0000 UTC m=+0.538930478 container died ff0885389bb9cfe2969b2e2886627b514d7ee9336ff1c97b8ac047bc20bf9326 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-13T15:09:30.387 DEBUG:teuthology.orchestra.run.smithi096:> sudo pkill -f 'journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.1.service' 2024-09-13T15:09:30.426 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:09:30.426 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-13T15:09:30.426 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-13T15:09:30.426 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.2 2024-09-13T15:09:30.752 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 15:09:30 smithi096 systemd[1]: Stopping Ceph osd.2 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:09:31.028 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 15:09:30 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2[120548]: 2024-09-13T15:09:30.750+0000 7f230c646640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:09:31.029 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 15:09:30 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2[120548]: 2024-09-13T15:09:30.750+0000 7f230c646640 -1 osd.2 428 *** Got signal Terminated *** 2024-09-13T15:09:31.029 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 15:09:30 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2[120548]: 2024-09-13T15:09:30.750+0000 7f230c646640 -1 osd.2 428 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:09:36.510 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 15:09:36 smithi096 podman[325969]: 2024-09-13 15:09:36.198854767 +0000 UTC m=+5.538226373 container died eaf4f3b21ff8e9dda6f29cf9195bbdd44fb39f39136edae9fd9c90aee7453100 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T15:09:36.823 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 15:09:36 smithi096 podman[325969]: 2024-09-13 15:09:36.692314777 +0000 UTC m=+6.031686376 container cleanup eaf4f3b21ff8e9dda6f29cf9195bbdd44fb39f39136edae9fd9c90aee7453100 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:09:36.823 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 15:09:36 smithi096 bash[325969]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2 2024-09-13T15:09:37.322 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 15:09:36 smithi096 podman[325987]: 2024-09-13 15:09:36.94370064 +0000 UTC m=+0.741212021 container remove eaf4f3b21ff8e9dda6f29cf9195bbdd44fb39f39136edae9fd9c90aee7453100 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=) 2024-09-13T15:09:37.606 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 15:09:37 smithi096 podman[326076]: 2024-09-13 15:09:37.235315648 +0000 UTC m=+0.015859009 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:09:37.606 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 15:09:37 smithi096 podman[326076]: 2024-09-13 15:09:37.344387316 +0000 UTC m=+0.124930669 container create cf5a2d1c56b6d9b4f9e587f177ebc8cec7cb4eaaa5ff6111ddb0e04bacd08fa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-13T15:09:37.606 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 15:09:37 smithi096 podman[326076]: 2024-09-13 15:09:37.604966801 +0000 UTC m=+0.385510157 container init cf5a2d1c56b6d9b4f9e587f177ebc8cec7cb4eaaa5ff6111ddb0e04bacd08fa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:09:38.073 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 15:09:37 smithi096 podman[326076]: 2024-09-13 15:09:37.608614634 +0000 UTC m=+0.389157991 container start cf5a2d1c56b6d9b4f9e587f177ebc8cec7cb4eaaa5ff6111ddb0e04bacd08fa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-09-13T15:09:38.073 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 15:09:37 smithi096 podman[326076]: 2024-09-13 15:09:37.680000004 +0000 UTC m=+0.460543365 container attach cf5a2d1c56b6d9b4f9e587f177ebc8cec7cb4eaaa5ff6111ddb0e04bacd08fa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-13T15:09:38.073 INFO:journalctl@ceph.osd.2.smithi096.stdout:Sep 13 15:09:37 smithi096 podman[326076]: 2024-09-13 15:09:37.802506899 +0000 UTC m=+0.583050262 container died cf5a2d1c56b6d9b4f9e587f177ebc8cec7cb4eaaa5ff6111ddb0e04bacd08fa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-13T15:09:38.501 DEBUG:teuthology.orchestra.run.smithi096:> sudo pkill -f 'journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.2.service' 2024-09-13T15:09:38.543 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:09:38.543 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-13T15:09:38.543 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-13T15:09:38.544 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.3 2024-09-13T15:09:38.823 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:38 smithi096 systemd[1]: Stopping Ceph osd.3 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:09:39.322 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:38 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[276103]: 2024-09-13T15:09:38.878+0000 7fbe21f16640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:09:39.323 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:38 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[276103]: 2024-09-13T15:09:38.878+0000 7fbe21f16640 -1 osd.3 428 *** Got signal Terminated *** 2024-09-13T15:09:39.323 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:38 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[276103]: 2024-09-13T15:09:38.878+0000 7fbe21f16640 -1 osd.3 428 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:09:40.823 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:40 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[276103]: 2024-09-13T15:09:40.406+0000 7fbe1a523640 -1 osd.3 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:14.988835+0000 front 2024-09-13T15:09:14.988982+0000 (oldest deadline 2024-09-13T15:09:39.688916+0000) 2024-09-13T15:09:41.823 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:41 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[276103]: 2024-09-13T15:09:41.416+0000 7fbe1a523640 -1 osd.3 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:14.988835+0000 front 2024-09-13T15:09:14.988982+0000 (oldest deadline 2024-09-13T15:09:39.688916+0000) 2024-09-13T15:09:41.952 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:41 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:41.680+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:42.219 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:41 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T15:09:41.950+0000 7f75a4e67640 -1 osd.4 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:15.727349+0000 front 2024-09-13T15:09:15.727449+0000 (oldest deadline 2024-09-13T15:09:41.027228+0000) 2024-09-13T15:09:42.684 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:42 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:42.338+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:42.822 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:42 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[276103]: 2024-09-13T15:09:42.440+0000 7fbe1a523640 -1 osd.3 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:14.988835+0000 front 2024-09-13T15:09:14.988982+0000 (oldest deadline 2024-09-13T15:09:39.688916+0000) 2024-09-13T15:09:42.938 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:42 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:42.682+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:43.219 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:42 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T15:09:42.937+0000 7f75a4e67640 -1 osd.4 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:15.727349+0000 front 2024-09-13T15:09:15.727449+0000 (oldest deadline 2024-09-13T15:09:41.027228+0000) 2024-09-13T15:09:43.713 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:43.309+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:43.714 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:43.418+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:43.823 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:43 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3[276103]: 2024-09-13T15:09:43.440+0000 7fbe1a523640 -1 osd.3 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:14.988835+0000 front 2024-09-13T15:09:14.988982+0000 (oldest deadline 2024-09-13T15:09:39.688916+0000) 2024-09-13T15:09:43.969 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:43.712+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:43.969 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:43 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T15:09:43.933+0000 7f75a4e67640 -1 osd.4 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:15.727349+0000 front 2024-09-13T15:09:15.727449+0000 (oldest deadline 2024-09-13T15:09:41.027228+0000) 2024-09-13T15:09:44.664 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:44 smithi096 podman[326178]: 2024-09-13 15:09:44.365684046 +0000 UTC m=+5.600917668 container died 3dd8d5d5bb70f025ff9bff825c610f89affe973e5a1451657f1fa9b30194bf22 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T15:09:44.671 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:44.347+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:44.672 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:44.442+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:44.963 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:44.670+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:45.073 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:44 smithi096 podman[326178]: 2024-09-13 15:09:44.844472688 +0000 UTC m=+6.079706309 container cleanup 3dd8d5d5bb70f025ff9bff825c610f89affe973e5a1451657f1fa9b30194bf22 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:09:45.073 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:44 smithi096 bash[326178]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3 2024-09-13T15:09:45.219 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:44 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T15:09:44.962+0000 7f75a4e67640 -1 osd.4 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:15.727349+0000 front 2024-09-13T15:09:15.727449+0000 (oldest deadline 2024-09-13T15:09:41.027228+0000) 2024-09-13T15:09:45.452 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:45 smithi096 podman[326194]: 2024-09-13 15:09:45.105316092 +0000 UTC m=+0.732360603 container remove 3dd8d5d5bb70f025ff9bff825c610f89affe973e5a1451657f1fa9b30194bf22 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:09:45.670 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:45 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:45.363+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:45.670 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:45 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:45.489+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:45.766 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:45 smithi096 podman[326283]: 2024-09-13 15:09:45.354518497 +0000 UTC m=+0.015592110 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:09:45.766 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:45 smithi096 podman[326283]: 2024-09-13 15:09:45.471436694 +0000 UTC m=+0.132510373 container create fb03704233223c92a83ccc2d925b7b0bb100b35167a316cf19c6643defee5a0a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T15:09:45.766 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:45 smithi096 podman[326283]: 2024-09-13 15:09:45.681947651 +0000 UTC m=+0.343021264 container init fb03704233223c92a83ccc2d925b7b0bb100b35167a316cf19c6643defee5a0a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:09:45.767 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:45 smithi096 podman[326283]: 2024-09-13 15:09:45.685352501 +0000 UTC m=+0.346426118 container start fb03704233223c92a83ccc2d925b7b0bb100b35167a316cf19c6643defee5a0a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-13T15:09:45.767 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:45 smithi096 podman[326283]: 2024-09-13 15:09:45.765473045 +0000 UTC m=+0.426546676 container attach fb03704233223c92a83ccc2d925b7b0bb100b35167a316cf19c6643defee5a0a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T15:09:45.967 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:45 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:45.668+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:46.072 INFO:journalctl@ceph.osd.3.smithi096.stdout:Sep 13 15:09:45 smithi096 podman[326283]: 2024-09-13 15:09:45.875187067 +0000 UTC m=+0.536260681 container died fb03704233223c92a83ccc2d925b7b0bb100b35167a316cf19c6643defee5a0a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True) 2024-09-13T15:09:46.219 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:45 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T15:09:45.965+0000 7f75a4e67640 -1 osd.4 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:15.727349+0000 front 2024-09-13T15:09:15.727449+0000 (oldest deadline 2024-09-13T15:09:41.027228+0000) 2024-09-13T15:09:46.548 DEBUG:teuthology.orchestra.run.smithi096:> sudo pkill -f 'journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.3.service' 2024-09-13T15:09:46.586 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:09:46.587 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-13T15:09:46.587 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-13T15:09:46.587 DEBUG:teuthology.orchestra.run.smithi204:> sudo systemctl stop ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.4 2024-09-13T15:09:46.617 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:46.327+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:46.617 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:46.515+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:46.904 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:46 smithi204 systemd[1]: Stopping Ceph osd.4 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:09:46.904 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:46.693+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:47.219 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T15:09:46.902+0000 7f75a784d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:09:47.219 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T15:09:46.902+0000 7f75a784d640 -1 osd.4 428 *** Got signal Terminated *** 2024-09-13T15:09:47.219 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:46 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T15:09:46.902+0000 7f75a784d640 -1 osd.4 428 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:09:47.220 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:47 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T15:09:47.005+0000 7f75a4e67640 -1 osd.4 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:15.727349+0000 front 2024-09-13T15:09:15.727449+0000 (oldest deadline 2024-09-13T15:09:41.027228+0000) 2024-09-13T15:09:47.552 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:47 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:47.282+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:47.969 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:47 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:47.701+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:47.969 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:47 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:47.551+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:48.326 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:47 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T15:09:47.974+0000 7f75a4e67640 -1 osd.4 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:15.727349+0000 front 2024-09-13T15:09:15.727449+0000 (oldest deadline 2024-09-13T15:09:41.027228+0000) 2024-09-13T15:09:48.694 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:48.325+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:48.695 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:48.517+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:48.969 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:48 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:48.692+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:49.303 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T15:09:49.011+0000 7f75a4e67640 -1 osd.4 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:15.727349+0000 front 2024-09-13T15:09:15.727449+0000 (oldest deadline 2024-09-13T15:09:41.027228+0000) 2024-09-13T15:09:49.709 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:49.302+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:49.709 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:49.520+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:49.969 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:49 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:49.708+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:50.274 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T15:09:50.016+0000 7f75a4e67640 -1 osd.4 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:15.727349+0000 front 2024-09-13T15:09:15.727449+0000 (oldest deadline 2024-09-13T15:09:41.027228+0000) 2024-09-13T15:09:50.274 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T15:09:50.016+0000 7f75a4e67640 -1 osd.4 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:23.328268+0000 front 2024-09-13T15:09:23.328285+0000 (oldest deadline 2024-09-13T15:09:49.228123+0000) 2024-09-13T15:09:50.537 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:50.273+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:50.969 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:50.536+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:50.970 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:50 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:50.678+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:51.469 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:51 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:51.265+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:51.469 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:51 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T15:09:51.059+0000 7f75a4e67640 -1 osd.4 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:15.727349+0000 front 2024-09-13T15:09:15.727449+0000 (oldest deadline 2024-09-13T15:09:41.027228+0000) 2024-09-13T15:09:51.470 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:51 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4[37346]: 2024-09-13T15:09:51.059+0000 7f75a4e67640 -1 osd.4 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:23.328268+0000 front 2024-09-13T15:09:23.328285+0000 (oldest deadline 2024-09-13T15:09:49.228123+0000) 2024-09-13T15:09:51.969 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:51 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:51.567+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:51.969 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:51 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:51.691+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:51.970 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:51 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:51.691+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:09:52.532 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:52 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:52.232+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:52.533 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:52 smithi204 podman[230820]: 2024-09-13 15:09:52.331410451 +0000 UTC m=+5.537229882 container died b48920b2b9e743bbc18a85c64502168f1ff46babb194e4535be20e15ff8dcb41 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-13T15:09:52.795 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:52 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:52.530+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:52.796 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:52 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:52.699+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:52.796 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:52 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:52.699+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:09:53.219 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:52 smithi204 podman[230820]: 2024-09-13 15:09:52.79486812 +0000 UTC m=+6.000687550 container cleanup b48920b2b9e743bbc18a85c64502168f1ff46babb194e4535be20e15ff8dcb41 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-13T15:09:53.219 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:52 smithi204 bash[230820]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4 2024-09-13T15:09:53.219 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:53 smithi204 podman[230836]: 2024-09-13 15:09:53.034772364 +0000 UTC m=+0.698079449 container remove b48920b2b9e743bbc18a85c64502168f1ff46babb194e4535be20e15ff8dcb41 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:09:53.525 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:53 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:53.223+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:53.525 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:53 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:53.223+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:26.661092+0000 front 2024-09-13T15:09:26.661184+0000 (oldest deadline 2024-09-13T15:09:53.061487+0000) 2024-09-13T15:09:53.526 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:53 smithi204 podman[230925]: 2024-09-13 15:09:53.337394319 +0000 UTC m=+0.015853704 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T15:09:53.526 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:53 smithi204 podman[230925]: 2024-09-13 15:09:53.439212897 +0000 UTC m=+0.117672272 container create f5a7ae784b49c6b1d0bb4b9c80fde1494239ee04c0722672aec35cb6f1179ed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=quincy-bbdadfa, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-13T15:09:53.789 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:53 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:53.523+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:53.789 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:53 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:53.523+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:24.909085+0000 front 2024-09-13T15:09:24.909155+0000 (oldest deadline 2024-09-13T15:09:53.109496+0000) 2024-09-13T15:09:53.789 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:53 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:53.663+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:53.789 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:53 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:53.663+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:09:53.790 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:53 smithi204 podman[230925]: 2024-09-13 15:09:53.687452845 +0000 UTC m=+0.365912218 container init f5a7ae784b49c6b1d0bb4b9c80fde1494239ee04c0722672aec35cb6f1179ed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2) 2024-09-13T15:09:53.790 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:53 smithi204 podman[230925]: 2024-09-13 15:09:53.691758406 +0000 UTC m=+0.370217792 container start f5a7ae784b49c6b1d0bb4b9c80fde1494239ee04c0722672aec35cb6f1179ed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-13T15:09:53.790 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:53 smithi204 podman[230925]: 2024-09-13 15:09:53.787701459 +0000 UTC m=+0.466160900 container attach f5a7ae784b49c6b1d0bb4b9c80fde1494239ee04c0722672aec35cb6f1179ed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=quincy-bbdadfa, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:09:54.216 INFO:journalctl@ceph.osd.4.smithi204.stdout:Sep 13 15:09:53 smithi204 podman[230925]: 2024-09-13 15:09:53.930198837 +0000 UTC m=+0.608658220 container died f5a7ae784b49c6b1d0bb4b9c80fde1494239ee04c0722672aec35cb6f1179ed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-13T15:09:54.469 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:54 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:54.257+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:54.469 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:54 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:54.257+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:26.661092+0000 front 2024-09-13T15:09:26.661184+0000 (oldest deadline 2024-09-13T15:09:53.061487+0000) 2024-09-13T15:09:54.519 DEBUG:teuthology.orchestra.run.smithi204:> sudo pkill -f 'journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.4.service' 2024-09-13T15:09:54.600 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:09:54.600 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-13T15:09:54.600 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-13T15:09:54.601 DEBUG:teuthology.orchestra.run.smithi204:> sudo systemctl stop ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.5 2024-09-13T15:09:54.778 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:54 smithi204 systemd[1]: Stopping Ceph osd.5 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:09:54.779 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:54 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:54.514+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:54.779 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:54 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:54.514+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:24.909085+0000 front 2024-09-13T15:09:24.909155+0000 (oldest deadline 2024-09-13T15:09:53.109496+0000) 2024-09-13T15:09:54.779 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:54 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:54.678+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:54.780 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:54 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:54.678+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:09:55.219 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:54 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:54.936+0000 7fabc1677640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:09:55.219 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:54 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:54.936+0000 7fabc1677640 -1 osd.5 428 *** Got signal Terminated *** 2024-09-13T15:09:55.219 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:54 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:54.936+0000 7fabc1677640 -1 osd.5 428 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:09:55.490 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:55.222+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:55.490 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:55.222+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:26.661092+0000 front 2024-09-13T15:09:26.661184+0000 (oldest deadline 2024-09-13T15:09:53.061487+0000) 2024-09-13T15:09:55.750 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:55.488+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:55.752 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:55.488+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:24.909085+0000 front 2024-09-13T15:09:24.909155+0000 (oldest deadline 2024-09-13T15:09:53.109496+0000) 2024-09-13T15:09:55.752 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:55.713+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:55.753 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:55 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:55.713+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:09:56.500 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:56 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:56.236+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:56.501 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:56 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:56.236+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:26.661092+0000 front 2024-09-13T15:09:26.661184+0000 (oldest deadline 2024-09-13T15:09:53.061487+0000) 2024-09-13T15:09:56.969 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:56 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:56.711+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:56.969 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:56 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:56.711+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:09:56.970 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:56 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:56.499+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:56.970 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:56 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:56.499+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:24.909085+0000 front 2024-09-13T15:09:24.909155+0000 (oldest deadline 2024-09-13T15:09:53.109496+0000) 2024-09-13T15:09:57.469 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:57 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:57.209+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:57.469 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:57 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:57.209+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:26.661092+0000 front 2024-09-13T15:09:26.661184+0000 (oldest deadline 2024-09-13T15:09:53.061487+0000) 2024-09-13T15:09:57.469 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:57 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:57.459+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:57.470 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:57 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:57.459+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:24.909085+0000 front 2024-09-13T15:09:24.909155+0000 (oldest deadline 2024-09-13T15:09:53.109496+0000) 2024-09-13T15:09:57.969 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:57 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:57.705+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:57.969 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:57 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:57.705+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:09:58.469 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:58 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:58.209+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:58.469 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:58 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:58.209+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:26.661092+0000 front 2024-09-13T15:09:26.661184+0000 (oldest deadline 2024-09-13T15:09:53.061487+0000) 2024-09-13T15:09:58.469 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:58 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:58.441+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:58.470 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:58 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:58.441+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:24.909085+0000 front 2024-09-13T15:09:24.909155+0000 (oldest deadline 2024-09-13T15:09:53.109496+0000) 2024-09-13T15:09:58.969 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:58 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:58.715+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:09:58.969 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:58 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:58.715+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:09:59.433 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:59 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:59.164+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.259540+0000 front 2024-09-13T15:09:16.259502+0000 (oldest deadline 2024-09-13T15:09:42.159403+0000) 2024-09-13T15:09:59.433 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:09:59 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5[43316]: 2024-09-13T15:09:59.164+0000 7fabbc48c640 -1 osd.5 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:26.661092+0000 front 2024-09-13T15:09:26.661184+0000 (oldest deadline 2024-09-13T15:09:53.061487+0000) 2024-09-13T15:09:59.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:59 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:59.428+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:09:59.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:09:59 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:09:59.428+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:24.909085+0000 front 2024-09-13T15:09:24.909155+0000 (oldest deadline 2024-09-13T15:09:53.109496+0000) 2024-09-13T15:10:00.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:59 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:59.726+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:00.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:59 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:59.726+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:00.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:09:59 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:09:59.726+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:00.487 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:00.386+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:10:00.487 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:00.386+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:24.909085+0000 front 2024-09-13T15:09:24.909155+0000 (oldest deadline 2024-09-13T15:09:53.109496+0000) 2024-09-13T15:10:00.487 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:00.386+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.410676+0000 front 2024-09-13T15:09:35.410662+0000 (oldest deadline 2024-09-13T15:10:00.110592+0000) 2024-09-13T15:10:00.756 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:10:00 smithi204 podman[231027]: 2024-09-13 15:10:00.486179145 +0000 UTC m=+5.657420636 container died 0ed6e334d2ea12a0e68bdc2e3e1518a8d805d8805aa82eb2c4518f44465d417a (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:10:01.012 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:00.753+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:01.013 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:00.753+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:01.013 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:00 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:00.753+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:01.369 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:10:01 smithi204 podman[231027]: 2024-09-13 15:10:01.011831253 +0000 UTC m=+6.183072733 container cleanup 0ed6e334d2ea12a0e68bdc2e3e1518a8d805d8805aa82eb2c4518f44465d417a (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-13T15:10:01.370 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:10:01 smithi204 bash[231027]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5 2024-09-13T15:10:01.370 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:10:01 smithi204 podman[231054]: 2024-09-13 15:10:01.238575431 +0000 UTC m=+0.752376059 container remove 0ed6e334d2ea12a0e68bdc2e3e1518a8d805d8805aa82eb2c4518f44465d417a (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-13T15:10:01.673 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:01 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:01.366+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:10:01.673 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:01 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:01.366+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:24.909085+0000 front 2024-09-13T15:09:24.909155+0000 (oldest deadline 2024-09-13T15:09:53.109496+0000) 2024-09-13T15:10:01.673 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:01 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:01.366+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.410676+0000 front 2024-09-13T15:09:35.410662+0000 (oldest deadline 2024-09-13T15:10:00.110592+0000) 2024-09-13T15:10:01.969 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:01 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:01.729+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:01.970 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:01 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:01.729+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:01.970 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:01 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:01.729+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:01.970 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:10:01 smithi204 podman[231145]: 2024-09-13 15:10:01.574856893 +0000 UTC m=+0.015840048 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T15:10:01.970 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:10:01 smithi204 podman[231145]: 2024-09-13 15:10:01.681148944 +0000 UTC m=+0.122132148 container create b5b17a67f20e5284ad3a1229496816f48c6cb7058ee34cf5d0e57da45d4eb922 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-13T15:10:01.970 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:10:01 smithi204 podman[231145]: 2024-09-13 15:10:01.900582049 +0000 UTC m=+0.341565195 container init b5b17a67f20e5284ad3a1229496816f48c6cb7058ee34cf5d0e57da45d4eb922 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa) 2024-09-13T15:10:01.971 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:10:01 smithi204 podman[231145]: 2024-09-13 15:10:01.903878725 +0000 UTC m=+0.344861871 container start b5b17a67f20e5284ad3a1229496816f48c6cb7058ee34cf5d0e57da45d4eb922 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, ceph=True, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-13T15:10:02.376 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:10:01 smithi204 podman[231145]: 2024-09-13 15:10:01.993840233 +0000 UTC m=+0.434823380 container attach b5b17a67f20e5284ad3a1229496816f48c6cb7058ee34cf5d0e57da45d4eb922 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True) 2024-09-13T15:10:02.376 INFO:journalctl@ceph.osd.5.smithi204.stdout:Sep 13 15:10:02 smithi204 podman[231145]: 2024-09-13 15:10:02.077755116 +0000 UTC m=+0.518738261 container died b5b17a67f20e5284ad3a1229496816f48c6cb7058ee34cf5d0e57da45d4eb922 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=quincy-bbdadfa, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:10:02.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:02 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:02.374+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:10:02.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:02 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:02.374+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:24.909085+0000 front 2024-09-13T15:09:24.909155+0000 (oldest deadline 2024-09-13T15:09:53.109496+0000) 2024-09-13T15:10:02.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:02 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:02.374+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.410676+0000 front 2024-09-13T15:09:35.410662+0000 (oldest deadline 2024-09-13T15:10:00.110592+0000) 2024-09-13T15:10:02.764 DEBUG:teuthology.orchestra.run.smithi204:> sudo pkill -f 'journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.5.service' 2024-09-13T15:10:02.803 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:10:02.804 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-13T15:10:02.804 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-13T15:10:02.804 DEBUG:teuthology.orchestra.run.smithi204:> sudo systemctl stop ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.6 2024-09-13T15:10:03.128 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:02 smithi204 systemd[1]: Stopping Ceph osd.6 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:10:03.129 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:03 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:03.125+0000 7efeebe19640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:10:03.129 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:02 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:02.739+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:03.129 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:02 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:02.739+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:03.129 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:02 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:02.739+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:03.469 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:03 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:03.125+0000 7efeebe19640 -1 osd.6 428 *** Got signal Terminated *** 2024-09-13T15:10:03.469 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:03 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:03.125+0000 7efeebe19640 -1 osd.6 428 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:10:03.469 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:03 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:03.332+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:10:03.469 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:03 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:03.332+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:24.909085+0000 front 2024-09-13T15:09:24.909155+0000 (oldest deadline 2024-09-13T15:09:53.109496+0000) 2024-09-13T15:10:03.470 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:03 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:03.332+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.410676+0000 front 2024-09-13T15:09:35.410662+0000 (oldest deadline 2024-09-13T15:10:00.110592+0000) 2024-09-13T15:10:04.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:03 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:03.769+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:04.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:03 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:03.769+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:04.220 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:03 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:03.769+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:04.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:04 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:04.362+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:10:04.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:04 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:04.362+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:24.909085+0000 front 2024-09-13T15:09:24.909155+0000 (oldest deadline 2024-09-13T15:09:53.109496+0000) 2024-09-13T15:10:04.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:04 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:04.362+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.410676+0000 front 2024-09-13T15:09:35.410662+0000 (oldest deadline 2024-09-13T15:10:00.110592+0000) 2024-09-13T15:10:05.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:04 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:04.788+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:05.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:04 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:04.788+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:05.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:04 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:04.788+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:05.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:05 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:05.356+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:10:05.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:05 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:05.356+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:24.909085+0000 front 2024-09-13T15:09:24.909155+0000 (oldest deadline 2024-09-13T15:09:53.109496+0000) 2024-09-13T15:10:05.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:05 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:05.356+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.410676+0000 front 2024-09-13T15:09:35.410662+0000 (oldest deadline 2024-09-13T15:10:00.110592+0000) 2024-09-13T15:10:06.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:05 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:05.826+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:06.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:05 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:05.826+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:06.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:05 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:05.826+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:06.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:06.407+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:10:06.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:06.407+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:24.909085+0000 front 2024-09-13T15:09:24.909155+0000 (oldest deadline 2024-09-13T15:09:53.109496+0000) 2024-09-13T15:10:06.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:06.407+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.410676+0000 front 2024-09-13T15:09:35.410662+0000 (oldest deadline 2024-09-13T15:10:00.110592+0000) 2024-09-13T15:10:07.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:06.811+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:07.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:06.811+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:07.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:06 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:06.811+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:07.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:07 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:07.359+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:18.508120+0000 front 2024-09-13T15:09:18.508182+0000 (oldest deadline 2024-09-13T15:09:43.208021+0000) 2024-09-13T15:10:07.719 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:07 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:07.359+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:24.909085+0000 front 2024-09-13T15:09:24.909155+0000 (oldest deadline 2024-09-13T15:09:53.109496+0000) 2024-09-13T15:10:07.720 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:07 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6[49271]: 2024-09-13T15:10:07.359+0000 7efee9433640 -1 osd.6 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.410676+0000 front 2024-09-13T15:09:35.410662+0000 (oldest deadline 2024-09-13T15:10:00.110592+0000) 2024-09-13T15:10:08.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:07 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:07.852+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:08.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:07 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:07.852+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:08.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:07 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:07.852+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:09.138 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:08 smithi204 podman[231248]: 2024-09-13 15:10:08.829223903 +0000 UTC m=+5.802808580 container died 8f12f56d751c5e370c74b8e73cbd5aef82f3553940e94d5d8f19f1f5d67acb68 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:10:09.138 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:08.859+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:09.138 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:08.859+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:09.138 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:08.859+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:09.138 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:08 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:08.859+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6830 osd.3 since back 2024-09-13T15:09:39.188059+0000 front 2024-09-13T15:09:39.188005+0000 (oldest deadline 2024-09-13T15:10:07.988329+0000) 2024-09-13T15:10:09.469 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:09 smithi204 podman[231248]: 2024-09-13 15:10:09.31061732 +0000 UTC m=+6.284202010 container cleanup 8f12f56d751c5e370c74b8e73cbd5aef82f3553940e94d5d8f19f1f5d67acb68 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-13T15:10:09.469 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:09 smithi204 bash[231248]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6 2024-09-13T15:10:09.873 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:09 smithi204 podman[231264]: 2024-09-13 15:10:09.570624776 +0000 UTC m=+0.738332661 container remove 8f12f56d751c5e370c74b8e73cbd5aef82f3553940e94d5d8f19f1f5d67acb68 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-13T15:10:10.133 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:09.871+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:10.133 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:09.871+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:10.133 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:09.871+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:10.133 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:09 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:09.871+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6830 osd.3 since back 2024-09-13T15:09:39.188059+0000 front 2024-09-13T15:09:39.188005+0000 (oldest deadline 2024-09-13T15:10:07.988329+0000) 2024-09-13T15:10:10.134 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:09 smithi204 podman[231351]: 2024-09-13 15:10:09.819176748 +0000 UTC m=+0.015990032 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T15:10:10.134 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:09 smithi204 podman[231351]: 2024-09-13 15:10:09.921394708 +0000 UTC m=+0.118207984 container create 5a509b02753c8f9b2fe962aa6364d4a6c4eb184880ef4bf51855310806b90359 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=quincy-bbdadfa, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-13T15:10:10.134 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:10 smithi204 podman[231351]: 2024-09-13 15:10:10.132290851 +0000 UTC m=+0.329104127 container init 5a509b02753c8f9b2fe962aa6364d4a6c4eb184880ef4bf51855310806b90359 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T15:10:10.469 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:10 smithi204 podman[231351]: 2024-09-13 15:10:10.135637888 +0000 UTC m=+0.332451172 container start 5a509b02753c8f9b2fe962aa6364d4a6c4eb184880ef4bf51855310806b90359 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-13T15:10:10.469 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:10 smithi204 podman[231351]: 2024-09-13 15:10:10.223783266 +0000 UTC m=+0.420596566 container attach 5a509b02753c8f9b2fe962aa6364d4a6c4eb184880ef4bf51855310806b90359 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:10:10.470 INFO:journalctl@ceph.osd.6.smithi204.stdout:Sep 13 15:10:10 smithi204 podman[231351]: 2024-09-13 15:10:10.303831253 +0000 UTC m=+0.500644531 container died 5a509b02753c8f9b2fe962aa6364d4a6c4eb184880ef4bf51855310806b90359 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-13T15:10:10.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:10.370Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.96:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.96:8765: connect: connection refused" 2024-09-13T15:10:10.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:10.372Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.96:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.96:8765: connect: connection refused" 2024-09-13T15:10:10.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:10.372Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.96:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.96:8765: connect: connection refused" 2024-09-13T15:10:10.470 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:10.378Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.96:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.96:8765: connect: connection refused" 2024-09-13T15:10:10.471 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:10.379Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.96:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.96:8765: connect: connection refused" 2024-09-13T15:10:10.471 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:10.379Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.96:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.96:8765: connect: connection refused" 2024-09-13T15:10:10.877 DEBUG:teuthology.orchestra.run.smithi204:> sudo pkill -f 'journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.6.service' 2024-09-13T15:10:10.916 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:10:10.916 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-13T15:10:10.916 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-13T15:10:10.916 DEBUG:teuthology.orchestra.run.smithi204:> sudo systemctl stop ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.7 2024-09-13T15:10:11.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:10.916+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:11.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:10.916+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:11.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:10.916+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:11.220 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:10 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:10.916+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6830 osd.3 since back 2024-09-13T15:09:39.188059+0000 front 2024-09-13T15:09:39.188005+0000 (oldest deadline 2024-09-13T15:10:07.988329+0000) 2024-09-13T15:10:11.220 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:10 smithi204 systemd[1]: Stopping Ceph osd.7 for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:10:11.719 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:11 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:11.254+0000 7fe67e880640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:10:11.719 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:11 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:11.254+0000 7fe67e880640 -1 osd.7 428 *** Got signal Terminated *** 2024-09-13T15:10:11.719 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:11 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:11.254+0000 7fe67e880640 -1 osd.7 428 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:10:12.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:11 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:11.960+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:12.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:11 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:11.960+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:12.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:11 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:11.960+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:12.220 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:11 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:11.960+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6830 osd.3 since back 2024-09-13T15:09:39.188059+0000 front 2024-09-13T15:09:39.188005+0000 (oldest deadline 2024-09-13T15:10:07.988329+0000) 2024-09-13T15:10:13.439 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:12 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:12.985+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:13.439 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:12 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:12.985+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:13.440 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:12 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:12.985+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:13.440 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:12 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:12.985+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6830 osd.3 since back 2024-09-13T15:09:39.188059+0000 front 2024-09-13T15:09:39.188005+0000 (oldest deadline 2024-09-13T15:10:07.988329+0000) 2024-09-13T15:10:14.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:13 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:13.946+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:14.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:13 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:13.946+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:14.219 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:13 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:13.946+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:14.220 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:13 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:13.946+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6830 osd.3 since back 2024-09-13T15:09:39.188059+0000 front 2024-09-13T15:09:39.188005+0000 (oldest deadline 2024-09-13T15:10:07.988329+0000) 2024-09-13T15:10:14.220 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:13 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:13.946+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.204:6804 osd.4 since back 2024-09-13T15:09:51.290009+0000 front 2024-09-13T15:09:51.289974+0000 (oldest deadline 2024-09-13T15:10:12.989997+0000) 2024-09-13T15:10:15.469 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:14 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:14.990+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:15.469 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:14 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:14.990+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:15.469 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:14 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:14.990+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:15.470 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:14 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:14.990+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6830 osd.3 since back 2024-09-13T15:09:39.188059+0000 front 2024-09-13T15:09:39.188005+0000 (oldest deadline 2024-09-13T15:10:07.988329+0000) 2024-09-13T15:10:15.470 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:14 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:14.990+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.204:6804 osd.4 since back 2024-09-13T15:09:51.290009+0000 front 2024-09-13T15:09:51.289974+0000 (oldest deadline 2024-09-13T15:10:12.989997+0000) 2024-09-13T15:10:16.469 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:15 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:15.972+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6806 osd.0 since back 2024-09-13T15:09:16.985331+0000 front 2024-09-13T15:09:16.985381+0000 (oldest deadline 2024-09-13T15:09:41.085247+0000) 2024-09-13T15:10:16.469 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:15 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:15.972+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6814 osd.1 since back 2024-09-13T15:09:25.786162+0000 front 2024-09-13T15:09:25.786120+0000 (oldest deadline 2024-09-13T15:09:51.686076+0000) 2024-09-13T15:10:16.469 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:15 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:15.972+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6822 osd.2 since back 2024-09-13T15:09:35.687608+0000 front 2024-09-13T15:09:35.687582+0000 (oldest deadline 2024-09-13T15:09:59.187453+0000) 2024-09-13T15:10:16.470 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:15 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:15.972+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.96:6830 osd.3 since back 2024-09-13T15:09:39.188059+0000 front 2024-09-13T15:09:39.188005+0000 (oldest deadline 2024-09-13T15:10:07.988329+0000) 2024-09-13T15:10:16.470 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:15 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7[54829]: 2024-09-13T15:10:15.972+0000 7fe67be9a640 -1 osd.7 428 heartbeat_check: no reply from 172.21.15.204:6804 osd.4 since back 2024-09-13T15:09:51.290009+0000 front 2024-09-13T15:09:51.289974+0000 (oldest deadline 2024-09-13T15:10:12.989997+0000) 2024-09-13T15:10:17.150 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:16 smithi204 podman[231453]: 2024-09-13 15:10:16.89531695 +0000 UTC m=+5.757200912 container died ffd75c8be38a6d94eed9a4fa134f92b82ae33865cafc018355d03dfa4311bc15 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, RELEASE=quincy-bbdadfa, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:10:17.469 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:17 smithi204 podman[231453]: 2024-09-13 15:10:17.323884701 +0000 UTC m=+6.185768662 container cleanup ffd75c8be38a6d94eed9a4fa134f92b82ae33865cafc018355d03dfa4311bc15 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-13T15:10:17.469 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:17 smithi204 bash[231453]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7 2024-09-13T15:10:17.969 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:17 smithi204 podman[231480]: 2024-09-13 15:10:17.583556587 +0000 UTC m=+0.684699548 container remove ffd75c8be38a6d94eed9a4fa134f92b82ae33865cafc018355d03dfa4311bc15 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:10:18.271 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:17 smithi204 podman[231571]: 2024-09-13 15:10:17.884644544 +0000 UTC m=+0.015748405 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T15:10:18.271 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:17 smithi204 podman[231571]: 2024-09-13 15:10:17.98440699 +0000 UTC m=+0.115510896 container create 0b12255e458dc9d7a4f4659a75cd276f648b24476630531bac6be9a4b46dea84 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-13T15:10:18.271 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:18 smithi204 podman[231571]: 2024-09-13 15:10:18.195460738 +0000 UTC m=+0.326564598 container init 0b12255e458dc9d7a4f4659a75cd276f648b24476630531bac6be9a4b46dea84 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-13T15:10:18.272 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:18 smithi204 podman[231571]: 2024-09-13 15:10:18.198680066 +0000 UTC m=+0.329783935 container start 0b12255e458dc9d7a4f4659a75cd276f648b24476630531bac6be9a4b46dea84 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa) 2024-09-13T15:10:18.272 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:18 smithi204 podman[231571]: 2024-09-13 15:10:18.270318984 +0000 UTC m=+0.401422858 container attach 0b12255e458dc9d7a4f4659a75cd276f648b24476630531bac6be9a4b46dea84 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731) 2024-09-13T15:10:18.693 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:18 smithi204 podman[231594]: 2024-09-13 15:10:18.402096685 +0000 UTC m=+0.016537527 container died 0b12255e458dc9d7a4f4659a75cd276f648b24476630531bac6be9a4b46dea84 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-13T15:10:18.943 INFO:journalctl@ceph.osd.7.smithi204.stdout:Sep 13 15:10:18 smithi204 podman[231594]: 2024-09-13 15:10:18.940631933 +0000 UTC m=+0.555072809 container remove 0b12255e458dc9d7a4f4659a75cd276f648b24476630531bac6be9a4b46dea84 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-13T15:10:18.966 DEBUG:teuthology.orchestra.run.smithi204:> sudo pkill -f 'journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@osd.7.service' 2024-09-13T15:10:19.006 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:10:19.006 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-13T15:10:19.007 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-13T15:10:19.007 DEBUG:teuthology.orchestra.run.smithi204:> sudo systemctl stop ceph-51e50ed6-71db-11ef-bceb-c7b262605968@prometheus.a 2024-09-13T15:10:19.219 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 systemd[1]: Stopping Ceph prometheus.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:10:19.601 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:19.327Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T15:10:19.601 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:19.327Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-13T15:10:19.602 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:19.327Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-13T15:10:19.602 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:19.327Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T15:10:19.602 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:19.327Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-13T15:10:19.602 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:19.327Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-13T15:10:19.602 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:19.327Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T15:10:19.602 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:19.327Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-13T15:10:19.602 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:19.327Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-13T15:10:19.603 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:19.329Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T15:10:19.603 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:19.329Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-13T15:10:19.603 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a[77262]: ts=2024-09-13T15:10:19.329Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-13T15:10:19.603 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 podman[231673]: 2024-09-13 15:10:19.337622243 +0000 UTC m=+0.114412369 container died c1d744b6a96b23ffee75907c4e30283f63be4ba6613fc93534f5eb071315d6d7 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T15:10:19.962 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 podman[231673]: 2024-09-13 15:10:19.693858746 +0000 UTC m=+0.470648860 container cleanup c1d744b6a96b23ffee75907c4e30283f63be4ba6613fc93534f5eb071315d6d7 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T15:10:19.963 INFO:journalctl@ceph.prometheus.a.smithi204.stdout:Sep 13 15:10:19 smithi204 bash[231673]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T15:10:20.110 DEBUG:teuthology.orchestra.run.smithi204:> sudo pkill -f 'journalctl -f -n 0 -u ceph-51e50ed6-71db-11ef-bceb-c7b262605968@prometheus.a.service' 2024-09-13T15:10:20.145 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:10:20.145 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-13T15:10:20.146 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T15:10:22.070 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 15:10:21 smithi096 systemd[1]: Stopping Ceph alertmanager.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:10:22.323 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 15:10:22 smithi096 ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a[87717]: ts=2024-09-13T15:10:22.092Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T15:10:22.323 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 15:10:22 smithi096 podman[326686]: 2024-09-13 15:10:22.143311434 +0000 UTC m=+0.175294940 container died e04c10a0dca29b51d31413f06f949393267b8971acc674e6450577429d4584fd (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T15:10:22.785 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 15:10:22 smithi096 podman[326686]: 2024-09-13 15:10:22.575443718 +0000 UTC m=+0.607427269 container cleanup e04c10a0dca29b51d31413f06f949393267b8971acc674e6450577429d4584fd (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T15:10:22.786 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 15:10:22 smithi096 bash[326686]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T15:10:23.072 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 15:10:22 smithi096 podman[326699]: 2024-09-13 15:10:22.784866283 +0000 UTC m=+0.638510472 container remove e04c10a0dca29b51d31413f06f949393267b8971acc674e6450577429d4584fd (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T15:10:23.073 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 15:10:22 smithi096 podman[326699]: 2024-09-13 15:10:22.860872382 +0000 UTC m=+0.714516570 volume remove 8d0e8a14e572f640da0e9347ec1093af31e275f22776f36071fa49466c78834c 2024-09-13T15:10:23.573 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 15:10:23 smithi096 systemd[1]: Stopping Ceph node-exporter.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:10:23.573 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 15:10:23 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T15:10:23.573 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 15:10:23 smithi096 systemd[1]: Stopped Ceph alertmanager.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T15:10:23.573 INFO:journalctl@ceph.alertmanager.a.smithi096.stdout:Sep 13 15:10:23 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 2.641s CPU time. 2024-09-13T15:10:23.839 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 15:10:23 smithi096 podman[326808]: 2024-09-13 15:10:23.582083637 +0000 UTC m=+0.105999975 container died b02b57424615009493d722b9392c17ec710c4ab0d10ec1135a5c97a42c7c6c0f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T15:10:24.290 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 15:10:24 smithi096 podman[326808]: 2024-09-13 15:10:24.013179783 +0000 UTC m=+0.537096153 container cleanup b02b57424615009493d722b9392c17ec710c4ab0d10ec1135a5c97a42c7c6c0f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T15:10:24.291 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 15:10:24 smithi096 bash[326808]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T15:10:24.564 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 15:10:24 smithi096 podman[326820]: 2024-09-13 15:10:24.289665372 +0000 UTC m=+0.711342501 container remove b02b57424615009493d722b9392c17ec710c4ab0d10ec1135a5c97a42c7c6c0f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T15:10:24.564 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 15:10:24 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T15:10:25.019 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 15:10:24 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T15:10:25.019 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 15:10:24 smithi096 systemd[1]: Stopped Ceph node-exporter.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T15:10:25.019 INFO:journalctl@ceph.node-exporter.a.smithi096.stdout:Sep 13 15:10:24 smithi096 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 6.746s CPU time. 2024-09-13T15:10:34.194 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T15:10:35.969 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 15:10:35 smithi204 systemd[1]: Stopping Ceph grafana.a for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:10:36.282 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 15:10:35 smithi204 podman[232084]: 2024-09-13 15:10:35.983595263 +0000 UTC m=+0.113716606 container died 451523aebd73725094363334047a26fda8e2f23df04d92394ffac5bbd73e4ea6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, distribution-scope=public, version=8.5, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., release=239.1651231664, name=ubi8, description=Ceph Grafana Container, io.buildah.version=1.32.0, maintainer=Paul Cuzner ) 2024-09-13T15:10:36.719 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 15:10:36 smithi204 podman[232084]: 2024-09-13 15:10:36.464989627 +0000 UTC m=+0.595110968 container cleanup 451523aebd73725094363334047a26fda8e2f23df04d92394ffac5bbd73e4ea6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.buildah.version=1.32.0, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, name=ubi8, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T15:10:36.719 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 15:10:36 smithi204 bash[232084]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a 2024-09-13T15:10:37.035 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 15:10:36 smithi204 podman[232099]: 2024-09-13 15:10:36.724944799 +0000 UTC m=+0.740255058 container remove 451523aebd73725094363334047a26fda8e2f23df04d92394ffac5bbd73e4ea6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, vcs-type=git, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, name=ubi8, io.buildah.version=1.32.0) 2024-09-13T15:10:37.036 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 15:10:36 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T15:10:37.036 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 15:10:36 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-13T15:10:37.036 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 15:10:36 smithi204 systemd[1]: Stopped Ceph grafana.a for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T15:10:37.036 INFO:journalctl@ceph.grafana.a.smithi204.stdout:Sep 13 15:10:36 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@grafana.a.service: Consumed 25.321s CPU time. 2024-09-13T15:10:37.306 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 15:10:37 smithi204 systemd[1]: Stopping Ceph node-exporter.b for 51e50ed6-71db-11ef-bceb-c7b262605968... 2024-09-13T15:10:37.571 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 15:10:37 smithi204 podman[232203]: 2024-09-13 15:10:37.304812783 +0000 UTC m=+0.131158761 container died c58ad3b83def6d730f4ee7c03b308cc87251da76923f9d53da5ebedc4725659e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T15:10:37.938 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 15:10:37 smithi204 podman[232203]: 2024-09-13 15:10:37.669279864 +0000 UTC m=+0.495625841 container cleanup c58ad3b83def6d730f4ee7c03b308cc87251da76923f9d53da5ebedc4725659e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T15:10:37.938 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 15:10:37 smithi204 bash[232203]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T15:10:38.219 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 15:10:37 smithi204 podman[232214]: 2024-09-13 15:10:37.937250068 +0000 UTC m=+0.636103203 container remove c58ad3b83def6d730f4ee7c03b308cc87251da76923f9d53da5ebedc4725659e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-51e50ed6-71db-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T15:10:38.219 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 15:10:37 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T15:10:38.219 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 15:10:38 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T15:10:38.219 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 15:10:38 smithi204 systemd[1]: Stopped Ceph node-exporter.b for 51e50ed6-71db-11ef-bceb-c7b262605968. 2024-09-13T15:10:38.219 INFO:journalctl@ceph.node-exporter.b.smithi204.stdout:Sep 13 15:10:38 smithi204 systemd[1]: ceph-51e50ed6-71db-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 6.229s CPU time. 2024-09-13T15:10:43.983 DEBUG:teuthology.orchestra.run.smithi096:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:10:44.017 DEBUG:teuthology.orchestra.run.smithi204:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:10:44.240 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-13T15:10:44.242 DEBUG:teuthology.misc:Transferring archived files from smithi096:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903565/remote/smithi096/crash 2024-09-13T15:10:44.243 DEBUG:teuthology.orchestra.run.smithi096:> sudo tar c -f - -C /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/crash -- . 2024-09-13T15:10:44.283 INFO:teuthology.orchestra.run.smithi096.stderr:tar: /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T15:10:44.284 INFO:teuthology.orchestra.run.smithi096.stderr:tar: Error is not recoverable: exiting now 2024-09-13T15:10:44.286 DEBUG:teuthology.misc:Transferring archived files from smithi204:/var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903565/remote/smithi204/crash 2024-09-13T15:10:44.287 DEBUG:teuthology.orchestra.run.smithi204:> sudo tar c -f - -C /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/crash -- . 2024-09-13T15:10:44.321 INFO:teuthology.orchestra.run.smithi204.stderr:tar: /var/lib/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T15:10:44.321 INFO:teuthology.orchestra.run.smithi204.stderr:tar: Error is not recoverable: exiting now 2024-09-13T15:10:44.322 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-13T15:10:44.322 DEBUG:teuthology.orchestra.run.smithi096:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v OSD_DOWN | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v OSDMAP_FLAGS | egrep -v PG_AVAILABILITY | head -n 1 2024-09-13T15:10:44.419 INFO:teuthology.orchestra.run.smithi096.stdout:2024-09-13T14:40:00.000123+0000 mon.a (mon.0) 554 : cluster [WRN] Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T15:10:44.419 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-09-13T15:10:44.419 INFO:tasks.cephadm:Compressing logs... 2024-09-13T15:10:44.419 DEBUG:teuthology.orchestra.run.smithi096:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T15:10:44.422 DEBUG:teuthology.orchestra.run.smithi204:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T15:10:44.451 INFO:teuthology.orchestra.run.smithi204.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-09-13T15:10:44.455 INFO:teuthology.orchestra.run.smithi096.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose --: No such file or directory /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-13T15:10:44.456 INFO:teuthology.orchestra.run.smithi096.stderr: 2024-09-13T15:10:44.460 INFO:teuthology.orchestra.run.smithi096.stderr:gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-13T15:10:44.460 INFO:teuthology.orchestra.run.smithi096.stderr:gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T15:10:44.460 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.0.log: /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T15:10:44.460 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mon.c.log 2024-09-13T15:10:44.461 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T15:10:44.461 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mon.a.log 2024-09-13T15:10:44.461 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-13T15:10:44.461 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mgr.y.log 2024-09-13T15:10:44.461 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.log 2024-09-13T15:10:44.461 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mds.foo.smithi096.dqiqpz.log 2024-09-13T15:10:44.461 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-13T15:10:44.462 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mds.foo.smithi096.dqiqpz.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T15:10:44.462 INFO:teuthology.orchestra.run.smithi096.stderr: 81.8% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mds.foo.smithi096.dqiqpz.log.gz 2024-09-13T15:10:44.463 INFO:teuthology.orchestra.run.smithi204.stderr:gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-13T15:10:44.463 INFO:teuthology.orchestra.run.smithi204.stderr:gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T15:10:44.463 INFO:teuthology.orchestra.run.smithi204.stderr:gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-13T15:10:44.463 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.7.log: /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T15:10:44.463 INFO:teuthology.orchestra.run.smithi204.stderr:gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T15:10:44.464 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.4.log: /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mds.foo.smithi204.hbwbot.log 2024-09-13T15:10:44.465 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-13T15:10:44.465 INFO:teuthology.orchestra.run.smithi204.stderr:gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-13T15:10:44.465 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mds.foo.smithi204.hbwbot.log: /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.6.log: 62.4% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mds.foo.smithi204.hbwbot.log.gz 2024-09-13T15:10:44.466 INFO:teuthology.orchestra.run.smithi204.stderr:gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.log 2024-09-13T15:10:44.466 INFO:teuthology.orchestra.run.smithi204.stderr:gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mgr.x.log 2024-09-13T15:10:44.466 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.5.log: /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mon.b.log 2024-09-13T15:10:44.466 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T15:10:44.467 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.3.log: /var/log/ceph/cephadm.log: 92.4% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T15:10:44.653 INFO:teuthology.orchestra.run.smithi096.stderr: 91.0% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T15:10:44.685 INFO:teuthology.orchestra.run.smithi096.stderr: 93.9% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T15:10:44.725 INFO:teuthology.orchestra.run.smithi096.stderr: 95.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T15:10:44.771 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 91.3% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T15:10:44.811 INFO:teuthology.orchestra.run.smithi096.stderr: 91.2% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T15:10:44.837 INFO:teuthology.orchestra.run.smithi204.stderr: 84.1% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T15:10:44.856 INFO:teuthology.orchestra.run.smithi204.stderr: 86.5% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T15:10:44.864 INFO:teuthology.orchestra.run.smithi204.stderr: 92.0% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T15:10:44.965 INFO:teuthology.orchestra.run.smithi204.stderr: 89.9% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T15:10:45.326 INFO:teuthology.orchestra.run.smithi204.stderr: 90.2% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mgr.x.log.gz 2024-09-13T15:10:49.228 INFO:teuthology.orchestra.run.smithi096.stderr: 89.6% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mgr.y.log.gz 2024-09-13T15:10:51.339 INFO:teuthology.orchestra.run.smithi204.stderr: 92.2% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mon.b.log.gz 2024-09-13T15:10:57.100 INFO:teuthology.orchestra.run.smithi096.stderr: 91.7% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mon.c.log.gz 2024-09-13T15:11:34.817 INFO:teuthology.orchestra.run.smithi096.stderr: 90.2% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-mon.a.log.gz 2024-09-13T15:24:28.447 INFO:teuthology.orchestra.run.smithi204.stderr: 93.4% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-13T15:24:44.435 INFO:teuthology.orchestra.run.smithi204.stderr: 93.2% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-13T15:26:27.198 INFO:teuthology.orchestra.run.smithi096.stderr: 94.0% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-13T15:26:35.464 INFO:teuthology.orchestra.run.smithi096.stderr: 93.4% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-13T15:27:58.252 INFO:teuthology.orchestra.run.smithi204.stderr: 92.3% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-13T15:28:20.250 INFO:teuthology.orchestra.run.smithi204.stderr: 92.7% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-13T15:28:20.252 INFO:teuthology.orchestra.run.smithi204.stderr: 2024-09-13T15:28:20.252 INFO:teuthology.orchestra.run.smithi204.stderr:real 17m35.814s 2024-09-13T15:28:20.252 INFO:teuthology.orchestra.run.smithi204.stderr:user 6m8.895s 2024-09-13T15:28:20.252 INFO:teuthology.orchestra.run.smithi204.stderr:sys 0m19.773s 2024-09-13T15:30:18.905 INFO:teuthology.orchestra.run.smithi096.stderr: 92.8% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-13T15:30:40.535 INFO:teuthology.orchestra.run.smithi096.stderr: 93.3% -- replaced with /var/log/ceph/51e50ed6-71db-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-13T15:30:40.537 INFO:teuthology.orchestra.run.smithi096.stderr: 2024-09-13T15:30:40.537 INFO:teuthology.orchestra.run.smithi096.stderr:real 19m56.102s 2024-09-13T15:30:40.538 INFO:teuthology.orchestra.run.smithi096.stderr:user 6m44.257s 2024-09-13T15:30:40.538 INFO:teuthology.orchestra.run.smithi096.stderr:sys 0m20.788s 2024-09-13T15:30:40.538 INFO:tasks.cephadm:Archiving logs... 2024-09-13T15:30:40.539 DEBUG:teuthology.misc:Transferring archived files from smithi096:/var/log/ceph to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903565/remote/smithi096/log 2024-09-13T15:30:40.540 DEBUG:teuthology.orchestra.run.smithi096:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T15:31:40.073 DEBUG:teuthology.misc:Transferring archived files from smithi204:/var/log/ceph to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903565/remote/smithi204/log 2024-09-13T15:31:40.074 DEBUG:teuthology.orchestra.run.smithi204:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T15:32:36.209 INFO:tasks.cephadm:Removing cluster... 2024-09-13T15:32:36.210 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 --force 2024-09-13T15:32:37.144 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 51e50ed6-71db-11ef-bceb-c7b262605968 --force 2024-09-13T15:32:38.106 INFO:tasks.cephadm:Removing cephadm ... 2024-09-13T15:32:38.106 DEBUG:teuthology.orchestra.run.smithi096:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T15:32:38.126 DEBUG:teuthology.orchestra.run.smithi204:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T15:32:38.142 INFO:tasks.cephadm:Teardown complete 2024-09-13T15:32:38.142 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-13T15:32:38.157 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-13T15:32:38.158 DEBUG:teuthology.orchestra.run.smithi096:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-13T15:32:38.169 DEBUG:teuthology.orchestra.run.smithi204:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-13T15:32:38.302 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-13T15:32:38.312 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-13T15:32:38.313 DEBUG:teuthology.orchestra.run.smithi096:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T15:32:38.316 DEBUG:teuthology.orchestra.run.smithi204:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T15:32:38.331 INFO:teuthology.orchestra.run.smithi204.stderr:bash: line 1: ntpq: command not found 2024-09-13T15:32:38.331 INFO:teuthology.orchestra.run.smithi096.stderr:bash: line 1: ntpq: command not found 2024-09-13T15:32:38.347 INFO:teuthology.orchestra.run.smithi204.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T15:32:38.347 INFO:teuthology.orchestra.run.smithi204.stdout:=============================================================================== 2024-09-13T15:32:38.347 INFO:teuthology.orchestra.run.smithi204.stdout:^* hv01.front.sepia.ceph.com 3 6 377 15 +288us[ +293us] +/- 62ms 2024-09-13T15:32:38.347 INFO:teuthology.orchestra.run.smithi204.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 16 -852us[ -846us] +/- 77ms 2024-09-13T15:32:38.347 INFO:teuthology.orchestra.run.smithi204.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 15 +455us[ +460us] +/- 62ms 2024-09-13T15:32:38.347 INFO:teuthology.orchestra.run.smithi204.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T15:32:38.358 INFO:teuthology.orchestra.run.smithi096.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T15:32:38.358 INFO:teuthology.orchestra.run.smithi096.stdout:=============================================================================== 2024-09-13T15:32:38.358 INFO:teuthology.orchestra.run.smithi096.stdout:^* hv01.front.sepia.ceph.com 3 6 377 11 +282us[ +276us] +/- 62ms 2024-09-13T15:32:38.358 INFO:teuthology.orchestra.run.smithi096.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 11 -862us[ -868us] +/- 78ms 2024-09-13T15:32:38.359 INFO:teuthology.orchestra.run.smithi096.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 13 +453us[ +447us] +/- 62ms 2024-09-13T15:32:38.359 INFO:teuthology.orchestra.run.smithi096.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T15:32:38.359 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-13T15:32:38.372 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-13T15:32:38.373 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-13T15:32:38.393 DEBUG:teuthology.orchestra.run.smithi096:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T15:32:38.548 DEBUG:teuthology.orchestra.run.smithi204:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T15:32:38.663 DEBUG:teuthology.orchestra.run.smithi096:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-13T15:32:38.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:32:38.694 DEBUG:teuthology.orchestra.run.smithi204:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-13T15:32:38.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:32:38.722 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-13T15:32:38.732 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-13T15:32:38.744 INFO:teuthology.task.internal:Duration was 4705.732251 seconds 2024-09-13T15:32:38.744 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-13T15:32:38.759 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-13T15:32:38.759 DEBUG:teuthology.orchestra.run.smithi096:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T15:32:38.761 DEBUG:teuthology.orchestra.run.smithi204:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T15:32:38.825 INFO:teuthology.orchestra.run.smithi096.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T15:32:38.826 INFO:teuthology.orchestra.run.smithi204.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T15:32:39.133 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-13T15:32:39.133 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi096.front.sepia.ceph.com 2024-09-13T15:32:39.133 DEBUG:teuthology.orchestra.run.smithi096:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T15:32:39.163 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi204.front.sepia.ceph.com 2024-09-13T15:32:39.164 DEBUG:teuthology.orchestra.run.smithi204:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T15:32:39.189 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-13T15:32:39.190 DEBUG:teuthology.orchestra.run.smithi096:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T15:32:39.206 DEBUG:teuthology.orchestra.run.smithi204:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T15:32:39.270 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-13T15:32:39.271 DEBUG:teuthology.orchestra.run.smithi096:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T15:32:39.273 DEBUG:teuthology.orchestra.run.smithi204:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T15:32:40.807 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-13T15:32:40.822 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-13T15:32:40.822 DEBUG:teuthology.orchestra.run.smithi096:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T15:32:40.848 DEBUG:teuthology.orchestra.run.smithi204:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T15:32:40.892 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-13T15:32:40.904 DEBUG:teuthology.orchestra.run.smithi096:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:32:40.906 DEBUG:teuthology.orchestra.run.smithi204:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:32:40.936 INFO:teuthology.orchestra.run.smithi096.stdout:kernel.core_pattern = core 2024-09-13T15:32:40.964 INFO:teuthology.orchestra.run.smithi204.stdout:kernel.core_pattern = core 2024-09-13T15:32:40.979 DEBUG:teuthology.orchestra.run.smithi096:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:32:41.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:32:41.012 DEBUG:teuthology.orchestra.run.smithi204:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:32:41.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:32:41.031 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-13T15:32:41.041 INFO:teuthology.task.internal:Transferring archived files... 2024-09-13T15:32:41.042 DEBUG:teuthology.misc:Transferring archived files from smithi096:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903565/remote/smithi096 2024-09-13T15:32:41.043 DEBUG:teuthology.orchestra.run.smithi096:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T15:32:41.124 DEBUG:teuthology.misc:Transferring archived files from smithi204:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903565/remote/smithi204 2024-09-13T15:32:41.124 DEBUG:teuthology.orchestra.run.smithi204:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T15:32:41.168 INFO:teuthology.task.internal:Removing archive directory... 2024-09-13T15:32:41.169 DEBUG:teuthology.orchestra.run.smithi096:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T15:32:41.171 DEBUG:teuthology.orchestra.run.smithi204:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T15:32:41.216 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-13T15:32:41.228 INFO:teuthology.task.internal:Not uploading archives. 2024-09-13T15:32:41.229 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-13T15:32:41.240 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-13T15:32:41.241 DEBUG:teuthology.orchestra.run.smithi096:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T15:32:41.243 DEBUG:teuthology.orchestra.run.smithi204:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T15:32:41.264 INFO:teuthology.orchestra.run.smithi096.stdout: 262169 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 13 15:32 /home/ubuntu/cephtest 2024-09-13T15:32:41.264 INFO:teuthology.orchestra.run.smithi096.stdout: 554134 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 13 15:06 /home/ubuntu/cephtest/mnt.0 2024-09-13T15:32:41.264 INFO:teuthology.orchestra.run.smithi096.stdout: 554271 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 13 15:06 /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T15:32:41.264 INFO:teuthology.orchestra.run.smithi096.stdout: 658827 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 15:06 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T15:32:41.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:32:41.265 INFO:teuthology.orchestra.run.smithi096.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-09-13T15:32:41.266 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi096 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-13T15:32:41.272 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-13T15:32:41.284 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-13T15:32:41.297 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi096 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-13T15:32:41.301 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 4705.732250928879 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi096 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_akupczyk@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=215ccecffe754054bf79ded524059acc status: fail success: false 2024-09-13T15:32:41.301 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T15:32:41.326 INFO:teuthology.orchestra.run.smithi204.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 15:32 /home/ubuntu/cephtest 2024-09-13T15:32:41.441 INFO:teuthology.run:FAIL